From 6989dd5ddec7ea02144ea4f8cb53c39b0f6746bb Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Mon, 20 Feb 2023 21:06:28 +0800 Subject: [PATCH 01/94] add PG --- .github/actions/confWriter/action.yml | 15 +- .github/workflows/api-rest.yml | 4 +- .github/workflows/api-session.yml | 4 +- .github/workflows/capacity-expansion.yml | 4 +- .github/workflows/codeql-analysis.yml | 73 +++ .github/workflows/dependabot.yml | 11 + .github/workflows/ds-influxdb.yml | 21 +- .github/workflows/ds-iotdb.yml | 22 +- .github/workflows/ds-parquet.yml | 21 +- .github/workflows/func-transform.yml | 4 +- .github/workflows/func-udf.yml | 4 +- .github/workflows/scale-out.yml | 4 +- .github/workflows/unit-mds.yml | 4 +- .../iginx/postgresql/PostgreSQLStorage.java | 121 +++-- .../postgresql/tools/TagFilterUtils.java | 7 - ...tgreSQLHistoryDataCapacityExpansionIT.java | 486 ++++++++++++++++++ .../PostgreSQLHistoryDataGenerator.java | 87 ++++ 17 files changed, 781 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/dependabot.yml create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index b3500eeb71..146b1aea30 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -4,7 +4,7 @@ inputs: confFile: description: 'the conf that you want to write' required: false - default: "Conf" + default: "iginxConf" if-CapExp: description: 'if you need capacity expansion' required: false @@ -44,16 +44,3 @@ runs: shell: bash run: | echo "${{inputs.DB-name}}" > ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/DBConf.txt - - - if: inputs.confFile=='iginxConf' - name: Change conf/config.properties - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - sed -i "s/port=6888/port=7888/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties - sed -i "s/restPort=6666/restPort=7666/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - sed -i "" "s/port=6888/port=7888/" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties - sed -i "" "restPort=6666/restPort=7666/" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties - fi - diff --git a/.github/workflows/api-rest.yml b/.github/workflows/api-rest.yml index 99e853048e..0c794e33d6 100644 --- a/.github/workflows/api-rest.yml +++ b/.github/workflows/api-rest.yml @@ -1,8 +1,10 @@ name: "API-Test-RESTful" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/api-session.yml b/.github/workflows/api-session.yml index a55d04f5f9..08458a785a 100644 --- a/.github/workflows/api-session.yml +++ b/.github/workflows/api-session.yml @@ -1,8 +1,10 @@ name: "API-Test-SESSIONv2" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/capacity-expansion.yml b/.github/workflows/capacity-expansion.yml index 2f525fa04c..c5d7c111c5 100644 --- a/.github/workflows/capacity-expansion.yml +++ b/.github/workflows/capacity-expansion.yml @@ -1,7 +1,9 @@ name: "Capacity-Expansion-Test" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..d6fd0008b2 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,73 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: + - main + pull_request: + # The branches below must be a subset of the branches above + branches: + - main + schedule: + - cron: '22 22 * * 6' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'java' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml new file mode 100644 index 0000000000..23a3bbe5c3 --- /dev/null +++ b/.github/workflows/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "maven" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "monthly" diff --git a/.github/workflows/ds-influxdb.yml b/.github/workflows/ds-influxdb.yml index 4aca343ece..18be290c77 100644 --- a/.github/workflows/ds-influxdb.yml +++ b/.github/workflows/ds-influxdb.yml @@ -1,8 +1,10 @@ name: "ITTest-ds-InfluxDB" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: @@ -70,6 +72,7 @@ jobs: chmod +x "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" nohup "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" & + InfluxDB-SQL-ds: strategy: fail-fast: false @@ -190,19 +193,9 @@ jobs: - name: Install with Maven run: mvn clean package -DskipTests - name: Start IginX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - - - name: Change Config to Start IGinX_2 - uses: ./.github/actions/confWriter - with: - confFile: "iginxConf" - - - name: Start IginX_2 - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} + run: | + chmod +x "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" + nohup "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" & - name: A Lame Integration Test with Maven for SQL run: mvn test -q -Dtest=InfluxDBSQLSessionPoolIT -DfailIfNoTests=false - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/ds-iotdb.yml b/.github/workflows/ds-iotdb.yml index ad4290ea5e..cd58bf872e 100644 --- a/.github/workflows/ds-iotdb.yml +++ b/.github/workflows/ds-iotdb.yml @@ -1,8 +1,10 @@ name: "ITTest-ds-IoTDB" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: @@ -177,14 +179,6 @@ jobs: with: version: ${VERSION} - - name: Change Config to Start IGinX_2 - uses: ./.github/actions/confWriter - - - name: Start IginX_2 - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - - name: A Lame Integration Test with Maven for IoTDB run: | if [ ${{ matrix.iotdb-version }} == "iotdb12" ]; then @@ -232,16 +226,6 @@ jobs: with: version: ${VERSION} - - name: Change Config to Start IGinX_2 - uses: ./.github/actions/confWriter - with: - confFile: "iginxConf" - - - name: Start IginX_2 - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - - name: A Lame Integration Test with Maven for SQL run: mvn test -q -Dtest=SQLSessionPoolIT -DfailIfNoTests=false - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/ds-parquet.yml b/.github/workflows/ds-parquet.yml index 61740a1c00..53733a4efb 100644 --- a/.github/workflows/ds-parquet.yml +++ b/.github/workflows/ds-parquet.yml @@ -1,8 +1,10 @@ name: "ITTest-ds-Parquet" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: @@ -132,21 +134,10 @@ jobs: fi - name: Install with Maven run: mvn clean package -DskipTests - - name: Start IginX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - - - name: Change Config to Start IGinX_2 - uses: ./.github/actions/confWriter - with: - confFile: "iginxConf" - - - name: Start IginX_2 - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} + run: | + chmod +x "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" + nohup "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" & - name: A Lame Integration Test with Maven for SQL run: mvn test -q -Dtest=ParquetSQLSessionPoolIT -DfailIfNoTests=false - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/func-transform.yml b/.github/workflows/func-transform.yml index ac52471ebe..429da548b8 100644 --- a/.github/workflows/func-transform.yml +++ b/.github/workflows/func-transform.yml @@ -1,8 +1,10 @@ name: "Function-Test-Transform" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/func-udf.yml b/.github/workflows/func-udf.yml index 1f37a2bfe6..7c6b0f2faa 100644 --- a/.github/workflows/func-udf.yml +++ b/.github/workflows/func-udf.yml @@ -1,8 +1,10 @@ name: "Function-Test-UDF" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/scale-out.yml b/.github/workflows/scale-out.yml index 00e239f22c..2dc9985f99 100644 --- a/.github/workflows/scale-out.yml +++ b/.github/workflows/scale-out.yml @@ -1,7 +1,9 @@ name: "Scale-out-On-IoTDB" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/unit-mds.yml b/.github/workflows/unit-mds.yml index bd158d9b0f..129804190d 100644 --- a/.github/workflows/unit-mds.yml +++ b/.github/workflows/unit-mds.yml @@ -1,8 +1,10 @@ name: "Metadata-Service-Test" on: + push: + branches: + - main pull_request: - types: [opened, reopened] branches: - main diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 7066195491..292d34d038 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -57,10 +57,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +78,7 @@ public class PostgreSQLStorage implements IStorage { private static final String DEFAULT_USERNAME = "postgres"; - private static final String DEFAULT_PASSWORD = "123456"; + private static final String DEFAULT_PASSWORD = "postgres"; private static final String DEFAULT_DBNAME = "timeseries"; @@ -143,6 +140,8 @@ private boolean testConnection() { @Override public TaskExecuteResult execute(StoragePhysicalTask task) { + logger.info("test begin!"); + logger.info(task.toString()); List operators = task.getOperators(); if (operators.size() != 1) { return new TaskExecuteResult( @@ -164,6 +163,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); } + logger.info("t1"); return executeProjectTask(project, filter); } else if (op.getType() == OperatorType.Insert) { Insert insert = (Insert) op; @@ -180,25 +180,37 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { public List getTimeSeries() throws PhysicalException { List timeseries = new ArrayList<>(); try { - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - if (tableName.startsWith("unit")) { - tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); - } - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - timeseries.add(new Timeseries( - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取数据库名称 +// if (databaseName.startsWith(DATABASE_PREFIX)) { + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + +// DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + if (tableName.startsWith("unit")) { + tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); + } + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) + timeseries.add(new Timeseries( + databaseName+"."+ + tableName/*.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) */ + IGINX_SEPARATOR + + columnName/*.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)*/, + DataTypeTransformer.fromPostgreSQL(typeName))); + } } +// } } } catch (SQLException e) { - throw new PhysicalException(e); + throw new RuntimeException(e); } return timeseries; } @@ -254,6 +266,7 @@ public Pair getBoundaryOfStorage(String prefix) t private TaskExecuteResult executeProjectTask(Project project, Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 + logger.info("t2"); try { List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); @@ -263,28 +276,44 @@ private TaskExecuteResult executeProjectTask(Project project, String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); // 查询序列类型 + DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (columnSet.next()) { + + ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field); + if(field.equals("*")){ + columnSet = databaseMetaData.getColumns(null, null, table,null); + } + while (columnSet.next()) { + field=columnSet.getString("COLUMN_NAME"); + if(field.equals("time")){ + continue; + } + logger.info(field); + String statement=""; + logger.info("t70"); String typeName = columnSet.getString("TYPE_NAME");//列字段类型 fields - .add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - String statement = String - .format(QUERY_DATA, field, table, - TagFilterUtils.transformToFilterStr(project.getTagFilter()), - FilterTransformer.toString(filter)); - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(statement); - resultSets.add(rs); + .add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + logger.info("t69"); + + statement = String + .format("SELECT time, %s FROM %s", field, table); + logger.info(statement); + + + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(statement); + resultSets.add(rs); + } } RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); return new TaskExecuteResult(rowStream); } catch (SQLException e) { return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in timescaledb failure", + new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); } } @@ -304,7 +333,7 @@ private TaskExecuteResult executeInsertTask(Insert insert) { } if (e != null) { return new TaskExecuteResult(null, - new PhysicalException("execute insert task in iotdb12 failure", e)); + new PhysicalException("execute insert task in postgresql failure", e)); } return new TaskExecuteResult(null, null); } @@ -312,6 +341,11 @@ private TaskExecuteResult executeInsertTask(Insert insert) { private void createTimeSeriesIfNotExists(String table, String field, Map tags, DataType dataType) { try { + if (tags==null){ + tags=new HashMap<>(); + tags.put("id","32"); + } + DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); if (!tableSet.next()) { @@ -350,6 +384,7 @@ private void useDatabase(String dbname) { stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { logger.info("create database error", e); + logger.info(dbname); } try { Map extraParams = meta.getExtraParams(); @@ -359,12 +394,15 @@ private void useDatabase(String dbname) { .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), dbname, username, password); connection = DriverManager.getConnection(connUrl); + logger.info("change database success,the database is: ",dbname); + logger.info(dbname); } catch (SQLException e) { logger.info("change database error", e); } } private Exception insertRowRecords(RowDataView data) { + logger.info("t4"); int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = connection.createStatement(); @@ -403,6 +441,10 @@ private Exception insertRowRecords(RowDataView data) { stmt.addBatch(String .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, columnsKeys, time, columnValues)); + logger.info(table); + logger.info(columnsKeys.toString()); + logger.info(String.valueOf(time)); + logger.info(columnValues.toString()); if (index > 0 && (index + 1) % batchSize == 0) { stmt.executeBatch(); } @@ -420,6 +462,7 @@ private Exception insertRowRecords(RowDataView data) { } private Exception insertColumnRecords(ColumnDataView data) { + logger.info("t7"); int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = connection.createStatement(); @@ -431,9 +474,15 @@ private Exception insertColumnRecords(ColumnDataView data) { String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); Map tags = data.getTags(i); + if (tags==null){ + tags=new HashMap<>(); + } + logger.info("t8"); createTimeSeriesIfNotExists(table, field, tags, dataType); + logger.info("t11"); BitmapView bitmapView = data.getBitmapView(i); int index = 0; + logger.info("t10"); for (int j = 0; j < data.getTimeSize(); j++) { if (bitmapView.get(j)) { long time = data.getKey(j) / 1000; // timescaledb存10位时间戳，java为13位时间戳 @@ -447,6 +496,7 @@ private Exception insertColumnRecords(ColumnDataView data) { StringBuilder columnsKeys = new StringBuilder(); StringBuilder columnValues = new StringBuilder(); + logger.info("t9"); for (Entry tagEntry : tags.entrySet()) { columnsKeys.append(tagEntry.getValue()).append(" "); columnValues.append(tagEntry.getValue()).append(" "); @@ -467,6 +517,7 @@ private Exception insertColumnRecords(ColumnDataView data) { } } stmt.executeBatch(); + logger.info("t8"); } catch (SQLException e) { return e; } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index 44a52b01dc..8d92d3697f 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -43,13 +43,6 @@ private static void transformToFilterStr(TagFilter filter, StringBuilder builder builder.append("="); builder.append(baseFilter.getTagValue()); break; - // TODO: case label - case BasePrecise: - break; - case Precise: - break; - case WithoutTag: - break; } } } \ No newline at end of file diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java new file mode 100644 index 0000000000..18eabf69ad --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java @@ -0,0 +1,486 @@ +package cn.edu.tsinghua.iginx.integration.expansion.postgresql + +import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.integration.SQLSessionIT; +import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; +import cn.edu.tsinghua.iginx.integration.expansion.unit.SQLTestTools; +import cn.edu.tsinghua.iginx.session.Session; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreSQLHistoryDataCapacityExpansionIT implements BaseCapacityExpansionIT { + + private static final Logger logger = LoggerFactory.getLogger(SQLSessionIT.class); + + private static Connection connection; + + private String ENGINE_TYPE; + + public PostgreSQLHistoryDataCapacityExpansionIT(String engineType) { + this.ENGINE_TYPE = engineType; + } + + @BeforeClass + public static void setUp() { +// session = new Session("127.0.0.1", 6888, "root", "root"); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); +// Connection connection = DriverManager.getConnection(connUrl); +// Statement stmt = connection.createStatement(); + try { +// session.openSession(); + Connection connection = DriverManager.getConnection(connUrl); + } catch (SessionException e) { + logger.error(e.getMessage()); + } + } + + private static void test_compare(Connection connection,String statement,String expect){ + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); + connection = DriverManager.getConnection(connUrl); + Statement stmt = connection.createStatement(); + ResultSet rs=stmt.execute(expect); + boolean act_true=true; + while(rs.next()) { + act_true=false; + String real=rs.getString("2"); + if (expect.contains(real)){ + act_true=true; + } + else{ + break; + } + } + if(act_true){ + logger.info("testQueryHistoryDataFromInitialNode is ok!") + } + else{ + logger.info("testQueryHistoryDataFromInitialNode have some problems!") + } + } + + @AfterClass + public static void tearDown() { + try { + connection.close(); + } catch (SessionException e) { + logger.error(e.getMessage()); + } + } + + @Test + public void oriHasDataExpHasData() throws Exception { + testQueryHistoryDataFromInitialNode(); + testQueryAfterInsertNewData(); + testCapacityExpansion_oriHasDataExpHasData(); + testWriteAndQueryAfterCapacityExpansion_oriHasDataExpHasData(); + } + + @Test + public void oriHasDataExpNoData() throws Exception { + testQueryHistoryDataFromInitialNode(); + testQueryAfterInsertNewData(); + testCapacityExpansion_oriHasDataExpNoData(); + testWriteAndQueryAfterCapacityExpansion_oriHasDataExpNoData(); + } + + @Test + public void oriNoDataExpHasData() throws Exception { + testQueryHistoryDataFromNoInitialNode(); + testQueryAfterInsertNewDataFromNoInitialNode(); + testCapacityExpansion_oriNoDataExpHasData(); + testWriteAndQueryAfterCapacityExpansion_oriNoDataExpHasData(); + } + + @Test + public void oriNoDataExpNoData() throws Exception { + testQueryHistoryDataFromNoInitialNode(); + testQueryAfterInsertNewDataFromNoInitialNode(); + testCapacityExpansion_oriNoDataExpNoData(); + testWriteAndQueryAfterCapacityExpansion_oriNoDataExpNoData(); + } + + //@Test + public void testQueryHistoryDataFromInitialNode() throws Exception { + String statement = "select * from *"; + String expect = "ResultSets:\n" + + "+----+-------------------+------------------------+\n" + + "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|\n" + + "+----+-------------------+------------------------+\n" + + "| 100| true| null|\n" + + "| 200| false| 20.71|\n" + + "+----+-------------------+------------------------+\n" + + "Total line number = 2\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 3\n"; + test_compare(connection, statement, expect); + + statement = "select count(*) from ln.wf01"; + expect = "ResultSets:\n" + + "+--------------------------+-------------------------------+\n" + + "|count(ln.wf01.wt01.status)|count(ln.wf01.wt01.temperature)|\n" + + "+--------------------------+-------------------------------+\n" + + "| 2| 1|\n" + + "+--------------------------+-------------------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + + } + + public void testQueryHistoryDataFromNoInitialNode() throws Exception { + String statement = "select * from ln"; + String expect = "ResultSets:\n" + + "+----+\n" + + "|Time|\n" + + "+----+\n" + + "+----+\n" + + "Empty set.\n"; + test_compare(connection, statement, expect); + + + statement = "count points"; + expect = "Points num: 0\n"; + test_compare(connection, statement, expect); + } + + //@Test + public void testQueryAfterInsertNewData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("insert into ln.wf02 (time, status, version) values (100, true, \"v1\");"); + stmt.execute("insert into ln.wf02 (time, status, version) values (400, false, \"v4\");"); + stmt.execute("insert into ln.wf02 (time, version) values (800, \"v8\");"); + + String statement = "select * from ln"; + String expect = "ResultSets:\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "| 100| true| null| true| v1|\n" + + "| 200| false| 20.71| null| null|\n" + + "| 400| null| null| false| v4|\n" + + "| 800| null| null| null| v8|\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "Total line number = 4\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 8\n"; + test_compare(connection, statement, expect); + + statement = "select count(*) from ln.wf02"; + expect = "ResultSets:\n" + + "+---------------------+----------------------+\n" + + "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + + "+---------------------+----------------------+\n" + + "| 2| 3|\n" + + "+---------------------+----------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + } + + public void testQueryAfterInsertNewDataFromNoInitialNode() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("insert into ln.wf02 (time, status, version) values (100, true, \"v1\");"); + stmt.execute("insert into ln.wf02 (time, status, version) values (400, false, \"v4\");"); + stmt.execute("insert into ln.wf02 (time, version) values (800, \"v8\");"); + + String statement = "select * from ln"; + String expect = "ResultSets:\n" + + "+----+--------------+---------------+\n" + + "|Time|ln.wf02.status|ln.wf02.version|\n" + + "+----+--------------+---------------+\n" + + "| 100| true| v1|\n" + + "| 400| false| v4|\n" + + "| 800| null| v8|\n" + + "+----+--------------+---------------+\n" + + "Total line number = 3\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 5\n"; + test_compare(connection, statement, expect); + + statement = "select count(*) from ln.wf02"; + expect = "ResultSets:\n" + + "+---------------------+----------------------+\n" + + "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + + "+---------------------+----------------------+\n" + + "| 2| 3|\n" + + "+---------------------+----------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + } + + //@Test + public void testCapacityExpansion_oriHasDataExpNoData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:no, is_read_only:true\");"); + + String statement = "select * from ln.wf03"; + String expect = "ResultSets:\n" + + "+----+\n" + + "|Time|\n" + + "+----+\n" + + "+----+\n" + + "Empty set.\n"; + test_compare(connection, statement, expect); + + statement = "select * from ln"; + expect = "ResultSets:\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "| 100| true| null| true| v1|\n" + + "| 200| false| 20.71| null| null|\n" + + "| 400| null| null| false| v4|\n" + + "| 800| null| null| null| v8|\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "Total line number = 4\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 8\n"; + test_compare(connection, statement, expect); + + } + + public void testCapacityExpansion_oriHasDataExpHasData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:true, is_read_only:true\");"); + + String statement = "select * from ln.wf03"; + String expect = "ResultSets:\n" + + "+----+-------------------+------------------------+\n" + + "|Time|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + + "+----+-------------------+------------------------+\n" + + "| 77| true| null|\n" + + "| 200| false| 77.71|\n" + + "+----+-------------------+------------------------+\n" + + "Total line number = 2\n"; + test_compare(connection, statement, expect); + + statement = "select * from ln"; + expect = "ResultSets:\n" + + "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + + "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + + "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + + "| 77| null| null| null| null| true| null|\n" + + "| 100| true| null| true| v1| null| null|\n" + + "| 200| false| 20.71| null| null| false| 77.71|\n" + + "| 400| null| null| false| v4| null| null|\n" + + "| 800| null| null| null| v8| null| null|\n" + + "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + + "Total line number = 5\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 11\n"; + test_compare(connection, statement, expect); + + } + + public void testCapacityExpansion_oriNoDataExpHasData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:true, is_read_only:true\");"); + + String statement = "select * from ln.wf03"; + String expect = "ResultSets:\n" + + "+----+-------------------+------------------------+\n" + + "|Time|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + + "+----+-------------------+------------------------+\n" + + "| 77| true| null|\n" + + "| 200| false| 77.71|\n" + + "+----+-------------------+------------------------+\n" + + "Total line number = 2\n"; + test_compare(connection, statement, expect); + + statement = "select * from ln"; + expect = "ResultSets:\n" + + "+----+--------------+---------------+-------------------+------------------------+\n" + + "|Time|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + + "+----+--------------+---------------+-------------------+------------------------+\n" + + "| 77| null| null| true| null|\n" + + "| 100| true| v1| null| null|\n" + + "| 200| null| null| false| 77.71|\n" + + "| 400| false| v4| null| null|\n" + + "| 800| null| v8| null| null|\n" + + "+----+--------------+---------------+-------------------+------------------------+\n" + + "Total line number = 5\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 8\n"; + test_compare(connection, statement, expect); + + } + + public void testCapacityExpansion_oriNoDataExpNoData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:false, is_read_only:true\");"); + + String statement = "select * from ln.wf03"; + String expect = "ResultSets:\n" + + "+----+\n" + + "|Time|\n" + + "+----+\n" + + "+----+\n" + + "Empty set.\n"; + test_compare(connection, statement, expect); + + statement = "select * from ln"; + expect = "ResultSets:\n" + + "+----+--------------+---------------+\n" + + "|Time|ln.wf02.status|ln.wf02.version|\n" + + "+----+--------------+---------------+\n" + + "| 100| true| v1|\n" + + "| 400| false| v4|\n" + + "| 800| null| v8|\n" + + "+----+--------------+---------------+\n" + + "Total line number = 3\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 5\n"; + test_compare(connection, statement, expect); + + } + + //@Test + public void testWriteAndQueryAfterCapacityExpansion_oriHasDataExpHasData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); + + String statement = "select * from ln"; + String expect = "ResultSets:\n" + + "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + + "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + + "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + + "| 77| null| null| null| null| true| null|\n" + + "| 100| true| null| true| v1| null| null|\n" + + "| 200| false| 20.71| null| null| false| 77.71|\n" + + "| 400| null| null| false| v4| null| null|\n" + + "| 800| null| null| null| v8| null| null|\n" + + "|1600| null| null| null| v48| null| null|\n" + + "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + + "Total line number = 6\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 12\n"; + test_compare(connection, statement, expect); + + statement = "select count(*) from ln.wf02"; + expect = "ResultSets:\n" + + "+---------------------+----------------------+\n" + + "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + + "+---------------------+----------------------+\n" + + "| 2| 4|\n" + + "+---------------------+----------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + } + + public void testWriteAndQueryAfterCapacityExpansion_oriNoDataExpHasData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); + + String statement = "select * from ln"; + String expect = "ResultSets:\n" + + "+----+--------------+---------------+-------------------+------------------------+\n" + + "|Time|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + + "+----+--------------+---------------+-------------------+------------------------+\n" + + "| 77| null| null| true| null|\n" + + "| 100| true| v1| null| null|\n" + + "| 200| null| null| false| 77.71|\n" + + "| 400| false| v4| null| null|\n" + + "| 800| null| v8| null| null|\n" + + "|1600| null| v48| null| null|\n" + + "+----+--------------+---------------+-------------------+------------------------+\n" + + "Total line number = 6\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 9\n"; + test_compare(connection, statement, expect); + + statement = "select count(*) from ln.wf02"; + expect = "ResultSets:\n" + + "+---------------------+----------------------+\n" + + "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + + "+---------------------+----------------------+\n" + + "| 2| 4|\n" + + "+---------------------+----------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + } + + public void testWriteAndQueryAfterCapacityExpansion_oriHasDataExpNoData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); + + String statement = "select * from ln"; + String expect = "ResultSets:\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "| 100| true| null| true| v1|\n" + + "| 200| false| 20.71| null| null|\n" + + "| 400| null| null| false| v4|\n" + + "| 800| null| null| null| v8|\n" + + "|1600| null| null| null| v48|\n" + + "+----+-------------------+------------------------+--------------+---------------+\n" + + "Total line number = 5\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 9\n"; + test_compare(connection, statement, expect); + + statement = "select count(*) from ln.wf02"; + expect = "ResultSets:\n" + + "+---------------------+----------------------+\n" + + "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + + "+---------------------+----------------------+\n" + + "| 2| 4|\n" + + "+---------------------+----------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + } + + public void testWriteAndQueryAfterCapacityExpansion_oriNoDataExpNoData() throws Exception { + Statement stmt = connection.createStatement(); + stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); + + String statement = "select * from *"; + String expect = "ResultSets:\n" + + "+----+--------------+---------------+\n" + + "|Time|ln.wf02.status|ln.wf02.version|\n" + + "+----+--------------+---------------+\n" + + "| 100| true| v1|\n" + + "| 400| false| v4|\n" + + "| 800| null| v8|\n" + + "|1600| null| v48|\n" + + "+----+--------------+---------------+\n" + + "Total line number = 4\n"; + test_compare(connection, statement, expect); + + statement = "count points"; + expect = "Points num: 6\n"; + test_compare(connection, statement, expect); + statement = "select count(*) from ln.wf02"; + expect = "ResultSets:\n" + + "+---------------------+----------------------+\n" + + "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + + "+---------------------+----------------------+\n" + + "| 2| 4|\n" + + "+---------------------+----------------------+\n" + + "Total line number = 1\n"; + test_compare(connection, statement, expect); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java new file mode 100644 index 0000000000..a8842bece6 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -0,0 +1,87 @@ +package cn.edu.tsinghua.iginx.integration.expansion.postgresql; + +import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; +import org.apache.iotdb.session.Session; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreSQLHistoryDataGenerator implements BaseHistoryDataGenerator { + + private static final Logger logger = LoggerFactory.getLogger(IoTDBHistoryDataGenerator.class); + + @Test + public void oriHasDataExpHasData() throws Exception { + writeHistoryDataToA(); + writeHistoryDataToB(); + } + + @Test + public void oriHasDataExpNoData() throws Exception { + writeHistoryDataToA(); + } + + @Test + public void oriNoDataExpHasData() throws Exception { + writeHistoryDataToB(); + } + + @Test + public void oriNoDataExpNoData() throws Exception { + } + + @Test + public void clearData() { + try { +// Session sessionA = new Session("127.0.0.1", 6667, "root", "root"); +// sessionA.open(); +// sessionA.executeNonQueryStatement("DELETE STORAGE GROUP root.*"); +// sessionA.close(); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); + Connection connection = DriverManager.getConnection(connUrl); + Statement stmt = connection.createStatement(); + ResultSet rs=stmt.executeQuery("SELECT datname FROM pg_database"); + while(rs.next()){ + db=rs.next(); + if(db.contains("unit")) { + stmt.execute(String.format("drop database %s",db)); + } + } + connection.close(); + } catch (Exception e) { + logger.error(e.getMessage()); + } + + logger.info("clear data success!"); + } + + public void writeHistoryDataToA() throws Exception { +// Session session = new Session("127.0.0.1", 6667, "root", "root"); +// session.open(); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); + Connection connection = DriverManager.getConnection(connUrl); + Statement stmt = connection.createStatement(); + + stmt.execute("INSERT INTO root.ln.wf01.wt01(time,status) values(100,true);"); + stmt.execute("INSERT INTO root.ln.wf01.wt01(time,status,temperature) values(200,false,20.71);"); + + connection.close(); + + logger.info("write data to 127.0.0.1:5432 success!"); + } + + public void writeHistoryDataToB() throws Exception { +// Session session = new Session("127.0.0.1", 6668, "root", "root"); +// session.open(); +// +// session.executeNonQueryStatement("INSERT INTO root.ln.wf03.wt01(timestamp,status) values(77,true);"); +// session.executeNonQueryStatement("INSERT INTO root.ln.wf03.wt01(timestamp,status,temperature) values(200,false,77.71);"); +// +// session.close(); + + logger.info("write data to 127.0.0.1:6668 success!"); + } + +} From e1463003df49b4cbb8957c0de22ba3292e89d7a3 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 22 Feb 2023 14:12:25 +0800 Subject: [PATCH 02/94] fix bugs --- .../iginx/postgresql/PostgreSQLStorage.java | 101 +++++++++++------- .../entity/PostgreSQLQueryRowStream.java | 6 ++ 2 files changed, 69 insertions(+), 38 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 292d34d038..a8de2089e4 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -201,9 +201,9 @@ public List getTimeSeries() throws PhysicalException { String typeName = columnSet.getString("TYPE_NAME");//列字段类型 //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) timeseries.add(new Timeseries( - databaseName+"."+ - tableName/*.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) */ + IGINX_SEPARATOR - + columnName/*.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)*/, + databaseName.replace(POSTGRESQL_SEPARATOR,IGINX_SEPARATOR)+IGINX_SEPARATOR+ + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), DataTypeTransformer.fromPostgreSQL(typeName))); } } @@ -224,36 +224,49 @@ public Pair getBoundaryOfStorage(String prefix) t ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); while (databaseSet.next()) { String databaseName = databaseSet.getString(1);//获取表名称 - if (databaseName.startsWith(DATABASE_PREFIX)) { - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); - // 获取first - String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); - Statement firstQueryStmt = connection.createStatement(); +// if (databaseName.startsWith(DATABASE_PREFIX)) { + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); + // 获取first + String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); + Statement firstQueryStmt = connection.createStatement(); + try { ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); if (firstQuerySet.next()) { long currMinTime = firstQuerySet.getLong(1); minTime = Math.min(currMinTime, minTime); } - // 获取last - String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - Statement lastQueryStmt = connection.createStatement(); + }catch (Exception e){ + minTime=1; //没有时间列，按递增的序列排序,1表示first + } + // 获取last + String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); + String queryNumOfRow=String.format("select count(*) from %s",tableName); + Statement lastQueryStmt = connection.createStatement(); + try { ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); if (lastQuerySet.next()) { long currMaxTime = lastQuerySet.getLong(1); maxTime = Math.max(currMaxTime, maxTime); } + }catch (Exception e){ //没有时间戳的表用行数表示last + ResultSet lastQuerySet=lastQueryStmt.executeQuery(queryNumOfRow); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } } } } +// } } } catch (SQLException e) { throw new PhysicalException(e); @@ -271,11 +284,15 @@ private TaskExecuteResult executeProjectTask(Project project, List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); for (String path : project.getPatterns()) { - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + logger.info("path======"+path); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database=database_table.substring(0,database_table.lastIndexOf('.')); + String table=database_table.substring(database_table.lastIndexOf('.')+1); + database=database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); // 查询序列类型 + useDatabase(database); DatabaseMetaData databaseMetaData = connection.getMetaData(); @@ -292,21 +309,19 @@ private TaskExecuteResult executeProjectTask(Project project, String statement=""; logger.info("t70"); String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields - .add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); logger.info("t69"); statement = String - .format("SELECT time, %s FROM %s", field, table); + .format("SELECT time,%s FROM %s", field, table); logger.info(statement); - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(statement); - resultSets.add(rs); - + Statement stmt = connection.createStatement(); + ResultSet rs = stmt.executeQuery(statement); + resultSets.add(rs); } } RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); @@ -383,7 +398,7 @@ private void useDatabase(String dbname) { Statement stmt = connection.createStatement(); stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { - logger.info("create database error", e); + logger.info("database exist!" ); logger.info(dbname); } try { @@ -413,11 +428,16 @@ private Exception insertRowRecords(RowDataView data) { if (bitmapView.get(j)) { String path = data.getPath(j); DataType dataType = data.getDataType(j); - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database=database_table.substring(0,database_table.lastIndexOf('.')); + String table=database_table.substring(database_table.lastIndexOf('.')+1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); Map tags = data.getTags(i); + + useDatabase(database); + stmt = connection.createStatement(); createTimeSeriesIfNotExists(table, field, tags, dataType); long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 @@ -441,10 +461,10 @@ private Exception insertRowRecords(RowDataView data) { stmt.addBatch(String .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, columnsKeys, time, columnValues)); - logger.info(table); - logger.info(columnsKeys.toString()); - logger.info(String.valueOf(time)); - logger.info(columnValues.toString()); +// logger.info(table); +// logger.info(columnsKeys.toString()); +// logger.info(String.valueOf(time)); +// logger.info(columnValues.toString()); if (index > 0 && (index + 1) % batchSize == 0) { stmt.executeBatch(); } @@ -469,14 +489,19 @@ private Exception insertColumnRecords(ColumnDataView data) { for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); DataType dataType = data.getDataType(i); - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database=database_table.substring(0,database_table.lastIndexOf('.')); + String table=database_table.substring(database_table.lastIndexOf('.')+1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); Map tags = data.getTags(i); if (tags==null){ tags=new HashMap<>(); } + + useDatabase(database); + stmt = connection.createStatement(); logger.info("t8"); createTimeSeriesIfNotExists(table, field, tags, dataType); logger.info("t11"); diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index b0a0b26d97..25e6f33629 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -8,6 +8,7 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; public class PostgreSQLQueryRowStream implements RowStream { @@ -20,11 +21,16 @@ public class PostgreSQLQueryRowStream implements RowStream { private final Header header; +// private Object[] values; +// private Object[] value; +// private final Long timestamp; + public PostgreSQLQueryRowStream(List resultSets, List fields) { this.resultSets = resultSets; this.header = new Header(Field.KEY, fields); this.currTimestamps = new long[resultSets.size()]; this.currValues = new Object[resultSets.size()]; +// this.values=new ArrayList<>(); // 默认填充一下timestamp列表 try { for (int i = 0; i < this.currTimestamps.length; i++) { From b368ca88848efebe5462da33ba3f3e3b6d9d7604 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 24 Feb 2023 12:13:30 +0800 Subject: [PATCH 03/94] fix pg bugs --- dataSources/dmdb/test.java | 306 ++++++++++++++++++ .../iginx/influxdb/InfluxDBStorage.java | 4 +- .../influxdb/query/entity/InfluxDBSchema.java | 3 + .../iginx/influxdb/tools/TagFilterUtils.java | 9 +- .../query/entity/IoTDBQueryRowStream.java | 1 + .../iginx/iotdb/tools/TagKVUtils.java | 2 +- .../query/entity/OpenTSDBRowStream.java | 1 - .../iginx/opentsdb/tools/TagKVUtils.java | 9 +- .../iginx/parquet/ParquetStorage.java | 5 +- .../tsinghua/iginx/parquet/exec/Executor.java | 1 + .../iginx/parquet/server/ParquetServer.java | 7 +- .../iginx/parquet/tools/TagKVUtils.java | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 102 +++--- .../entity/PostgreSQLQueryRowStream.java | 28 +- .../edu/tsinghua/iginx/utils/ByteUtils.java | 2 +- 15 files changed, 396 insertions(+), 86 deletions(-) create mode 100644 dataSources/dmdb/test.java diff --git a/dataSources/dmdb/test.java b/dataSources/dmdb/test.java new file mode 100644 index 0000000000..6ea82351ee --- /dev/null +++ b/dataSources/dmdb/test.java @@ -0,0 +1,306 @@ +/*该例程实现插入数据，修改数据，删除数据，数据查询等基本操作。*/ +import java.awt.Color; +import java.awt.Font; +import java.awt.Graphics2D; +import java.awt.font.FontRenderContext; +import java.awt.geom.Rectangle2D; +import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.Date; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import javax.imageio.ImageIO; +public class BasicApp { + // 定义DM JDBC驱动串 + String jdbcString = "dm.jdbc.driver.DmDriver"; + // 定义DM URL连接串 + String urlString = "jdbc:dm://localhost:5236"; + // 定义连接用户名 + String userName = "SYSDBA"; + // 定义连接用户口令 + String password = "SYSDBA001"; + // 定义连接对象 + Connection conn = null; + /* 加载JDBC驱动程序 + * @throws SQLException 异常 */ + public void loadJdbcDriver() throws SQLException { + try { + System.out.println("Loading JDBC Driver..."); + // 加载JDBC驱动程序 + Class.forName(jdbcString); + } catch (ClassNotFoundException e) { + throw new SQLException("Load JDBC Driver Error : " + e.getMessage()); + } catch (Exception ex) { + throw new SQLException("Load JDBC Driver Error : " + + ex.getMessage()); + } + } + /* 连接DM数据库 + * @throws SQLException 异常 */ + public void connect() throws SQLException { + try { + System.out.println("Connecting to DM Server..."); + // 连接DM数据库 + conn = DriverManager.getConnection(urlString, userName, password); + } catch (SQLException e) { + throw new SQLException("Connect to DM Server Error : " + + e.getMessage()); + } + } + /* 关闭连接 + * @throws SQLException 异常 */ + public void disConnect() throws SQLException { + try { + // 关闭连接 + conn.close(); + } catch (SQLException e) { + throw new SQLException("close connection error : " + e.getMessage()); + } + } + /* 往产品信息表插入数据 + * @throws SQLException 异常 */ + public void insertTable() throws SQLException { + // 插入数据语句 + String sql = "INSERT INTO production.product(name,author,publisher,publishtime," + + "product_subcategoryid,productno,satetystocklevel,originalprice,nowprice,discount," + + "description,photo,type,papertotal,wordtotal,sellstarttime,sellendtime) " + + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);"; + // 创建语句对象 + PreparedStatement pstmt = conn.prepareStatement(sql); + // 为参数赋值 + pstmt.setString(1, "三国演义"); + pstmt.setString(2, "罗贯中"); + pstmt.setString(3, "中华书局"); + pstmt.setDate(4, Date.valueOf("2005-04-01")); + pstmt.setInt(5, 4); + pstmt.setString(6, "9787101046121"); + pstmt.setInt(7, 10); + pstmt.setBigDecimal(8, new BigDecimal(19.0000)); + pstmt.setBigDecimal(9, new BigDecimal(15.2000)); + pstmt.setBigDecimal(10, new BigDecimal(8.0)); + pstmt.setString(11, "《三国演义》是中国第一部长篇章回体小说，中国小说由短篇发展至长篇的原因与说书有关。"); + // 设置大字段参数 + try { + // 创建一个图片用于插入大字段 + String filePath = "c:\\三国演义.jpg"; + CreateImage(filePath); + File file = new File(filePath); + InputStream in = new BufferedInputStream(new FileInputStream(file)); + pstmt.setBinaryStream(12, in, (int) file.length()); + } catch (FileNotFoundException e) { + System.out.println(e.getMessage()); + // 如果没有图片设置为NULL + pstmt.setNull(12, java.sql.Types.BINARY); + } catch (IOException e) { + System.out.println(e.getMessage()); + } + pstmt.setString(13, "25"); + pstmt.setInt(14, 943); + pstmt.setInt(15, 93000); + pstmt.setDate(16, Date.valueOf("2006-03-20")); + pstmt.setDate(17, Date.valueOf("1900-01-01")); + // 执行语句 + pstmt.executeUpdate(); + // 关闭语句 + pstmt.close(); + } + /* 查询产品信息表 + * @throws SQLException 异常 */ + public void queryProduct() throws SQLException { + // 查询语句 + String sql = "SELECT productid,name,author,description,photo FROM production.product WHERE productid=11"; + // 创建语句对象 + Statement stmt = conn.createStatement(); + // 执行查询 + ResultSet rs = stmt.executeQuery(sql); + // 显示结果集 + displayResultSet(rs); + // 关闭结果集 + rs.close(); + // 关闭语句 + stmt.close(); + } + /* 修改产品信息表数据 + * @throws SQLException 异常 */ + public void updateTable() throws SQLException { + // 更新数据语句 + String sql = "UPDATE production.product SET name = ?" + + "WHERE productid = 11;"; + // 创建语句对象 + PreparedStatement pstmt = conn.prepareStatement(sql); + // 为参数赋值 + pstmt.setString(1, "三国演义（上）"); + // 执行语句 + pstmt.executeUpdate(); + // 关闭语句 + pstmt.close(); + } + /* 删除产品信息表数据 + * @throws SQLException 异常 */ + public void deleteTable() throws SQLException { + // 删除数据语句 + String sql = "DELETE FROM production.product WHERE productid = 11;"; + // 创建语句对象 + Statement stmt = conn.createStatement(); + // 执行语句 + stmt.executeUpdate(sql); + // 关闭语句 + stmt.close(); + } + /* 查询产品信息表 + * @throws SQLException 异常 */ + public void queryTable() throws SQLException { + // 查询语句 + String sql = "SELECT productid,name,author,publisher FROM production.product"; + // 创建语句对象 + Statement stmt = conn.createStatement(); + // 执行查询 + ResultSet rs = stmt.executeQuery(sql); + // 显示结果集 + displayResultSet(rs); + // 关闭结果集 + rs.close(); + // 关闭语句 + stmt.close(); + } + /* 调用存储过程修改产品信息表数据 + * @throws SQLException 异常 */ + public void updateProduct() throws SQLException { + // 更新数据语句 + String sql = "{ CALL production.updateProduct(?,?) }"; + // 创建语句对象 + CallableStatement cstmt = conn.prepareCall(sql); + // 为参数赋值 + cstmt.setInt(1, 1); + cstmt.setString(2, "红楼梦（上）"); + // 执行语句 + cstmt.execute(); + // 关闭语句 + cstmt.close(); + } + /* 显示结果集 + * @param rs 结果集对象 + * @throws SQLException 异常 */ + private void displayResultSet(ResultSet rs) throws SQLException { + // 取得结果集元数据 + ResultSetMetaData rsmd = rs.getMetaData(); + // 取得结果集所包含的列数 + int numCols = rsmd.getColumnCount(); + // 显示列标头 + for (int i = 1; i <= numCols; i++) { + if (i > 1) { + System.out.print(","); + } + System.out.print(rsmd.getColumnLabel(i)); + } + System.out.println(""); + // 显示结果集中所有数据 + while (rs.next()) { + for (int i = 1; i <= numCols; i++) { + if (i > 1) { + System.out.print(","); + } + // 处理大字段 + if ("IMAGE".equals(rsmd.getColumnTypeName(i))) { + byte[] data = rs.getBytes(i); + if (data != null && data.length > 0) { + FileOutputStream fos; + try { + fos = new FileOutputStream("c:\\三国演义1.jpg"); + fos.write(data); + fos.close(); + } catch (FileNotFoundException e) { + System.out.println(e.getMessage()); + } catch (IOException e) { + System.out.println(e.getMessage()); + } + } + System.out.print("字段内容已写入文件c:\\三国演义1.jpg，长度" + data.length); + } else { + // 普通字段 + System.out.print(rs.getString(i)); + } + } + System.out.println(""); + } + } + /* 创建一个图片用于插入大字段 + * @throws IOException 异常 */ + private void CreateImage(String path) throws IOException { + int width = 100; + int height = 100; + String s = "三国演义"; + File file = new File(path); + Font font = new Font("Serif", Font.BOLD, 10); + BufferedImage bi = new BufferedImage(width, height, + BufferedImage.TYPE_INT_RGB); + Graphics2D g2 = (Graphics2D) bi.getGraphics(); + g2.setBackground(Color.WHITE); + g2.clearRect(0, 0, width, height); + g2.setPaint(Color.RED); + FontRenderContext context = g2.getFontRenderContext(); + Rectangle2D bounds = font.getStringBounds(s, context); + double x = (width - bounds.getWidth()) / 2; + double y = (height - bounds.getHeight()) / 2; + double ascent = -bounds.getY(); + double baseY = y + ascent; + g2.drawString(s, (int) x, (int) baseY); + ImageIO.write(bi, "jpg", file); + } + + //类主方法 @param args 参数 + public static void main(String args[]) { + try { + // 定义类对象 + BasicApp basicApp = new BasicApp(); + // 加载驱动程序 + basicApp.loadJdbcDriver(); + // 连接DM数据库 + basicApp.connect(); + // 插入数据 + System.out.println("--- 插入产品信息 ---"); + basicApp.insertTable(); + // 查询含有大字段的产品信息 + System.out.println("--- 显示插入结果 ---"); + basicApp.queryProduct(); + // 在修改前查询产品信息表 + System.out.println("--- 在修改前查询产品信息 ---"); + basicApp.queryTable(); + // 修改产品信息表 + System.out.println("--- 修改产品信息 ---"); + basicApp.updateTable(); + // 在修改后查询产品信息表 + System.out.println("--- 在修改后查询产品信息 ---"); + basicApp.queryTable(); + // 删除产品信息表 + System.out.println("--- 删除产品信息 ---"); + basicApp.deleteTable(); + // 在删除后查询产品信息表 + System.out.println("--- 在删除后查询产品信息 ---"); + basicApp.queryTable(); + // 调用存储过程修改产品信息表 + System.out.println("--- 调用存储过程修改产品信息 ---"); + basicApp.updateProduct(); + // 在存储过程更新后查询产品信息表 + System.out.println("--- 调用存储过程后查询产品信息 ---"); + basicApp.queryTable(); + // 关闭连接 + basicApp.disConnect(); + } catch (SQLException e) { + System.out.println(e.getMessage()); + } + } +} diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java index 4553e08a9d..5ab00fa0bc 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java @@ -179,8 +179,8 @@ public Pair getBoundaryOfStorage(String dataPrefi bucket.getName(), 0L, Long.MAX_VALUE, - measurementPrefix.equals(MEASUREMENTALL) ? MEASUREMENTALL : "= \"" + measurementPrefix + "\"", - fieldPrefix.equals(FIELDALL) ? FIELDALL : "~ /" + fieldPrefix + ".*/" + measurementPrefix == MEASUREMENTALL ? MEASUREMENTALL : "= \"" + measurementPrefix + "\"", + fieldPrefix == FIELDALL ? FIELDALL : "~ /" + fieldPrefix + ".*/" ); logger.debug("execute statement: " + statement); // 查询 first diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java index b56d526e42..5e2b1177f9 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java @@ -18,7 +18,10 @@ */ package cn.edu.tsinghua.iginx.influxdb.query.entity; +import cn.edu.tsinghua.iginx.utils.Pair; + import java.util.Collections; +import java.util.HashMap; import java.util.Map; public class InfluxDBSchema { diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java index 3aa257b547..96acefffec 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java @@ -5,6 +5,8 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import java.util.List; + public class TagFilterUtils { public static String transformToFilterStr(TagFilter filter) { @@ -46,13 +48,6 @@ private static void transformToFilterStr(TagFilter filter, StringBuilder builder builder.append(" == \"").append(baseFilter.getTagValue()).append("\" "); } break; - // TODO: case label - case BasePrecise: - break; - case Precise: - break; - case WithoutTag: - break; } } diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java index e8caf6aed4..2d9de649dd 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java @@ -26,6 +26,7 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.iotdb.IoTDBStorage; import cn.edu.tsinghua.iginx.iotdb.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.iotdb.tools.TagKVUtils; import cn.edu.tsinghua.iginx.utils.Pair; diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java index 3ef9e8a5c7..a59f0112a5 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java @@ -33,7 +33,7 @@ import java.util.stream.Collectors; public class TagKVUtils { - @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(TagKVUtils.class); public static final String tagNameAnnotation = Config.tagNameAnnotation; diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java index 635d209894..22a6f03872 100644 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java +++ b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java @@ -35,7 +35,6 @@ public class OpenTSDBRowStream implements RowStream { private int hasMoreRecords; - @SuppressWarnings("unchecked") public OpenTSDBRowStream(List resultList, boolean trimStorageUnit) { this.resultList = resultList; this.trimStorageUnit = trimStorageUnit; diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java index 2c4e5d61c3..aa02bbc2cd 100644 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java +++ b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java @@ -36,7 +36,7 @@ import java.util.stream.Collectors; public class TagKVUtils { - @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(TagKVUtils.class); public static final String tagNameAnnotation = Config.tagNameAnnotation; @@ -82,13 +82,6 @@ public static boolean match(Map tags, TagFilter tagFilter) { return match(tags, (OrTagFilter) tagFilter); case Base: return match(tags, (BaseTagFilter) tagFilter); - // TODO: case label - case BasePrecise: - break; - case Precise: - break; - case WithoutTag: - break; } return false; } diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java index 42cca09cc5..4bcea67625 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java @@ -21,8 +21,11 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.parquet.exec.Executor; +import cn.edu.tsinghua.iginx.parquet.exec.LocalExecutor; import cn.edu.tsinghua.iginx.parquet.exec.NewExecutor; import cn.edu.tsinghua.iginx.parquet.exec.RemoteExecutor; +import cn.edu.tsinghua.iginx.parquet.policy.NaiveParquetStoragePolicy; +import cn.edu.tsinghua.iginx.parquet.policy.ParquetStoragePolicy; import cn.edu.tsinghua.iginx.parquet.server.ParquetServer; import cn.edu.tsinghua.iginx.parquet.tools.FilterTransformer; import cn.edu.tsinghua.iginx.utils.Pair; @@ -41,7 +44,7 @@ import org.slf4j.LoggerFactory; public class ParquetStorage implements IStorage { - @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(ParquetStorage.class); private static final Config config = ConfigDescriptor.getInstance().getConfig(); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java index 9642d81370..7f3d472502 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java @@ -7,6 +7,7 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.TimeInterval; +import cn.edu.tsinghua.iginx.metadata.entity.TimeSeriesInterval; import cn.edu.tsinghua.iginx.metadata.entity.TimeSeriesRange; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java index 84321042f6..ce413a7987 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java @@ -1,7 +1,5 @@ package cn.edu.tsinghua.iginx.parquet.server; -import cn.edu.tsinghua.iginx.conf.Config; -import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; import cn.edu.tsinghua.iginx.parquet.exec.Executor; import cn.edu.tsinghua.iginx.parquet.thrift.ParquetService; import org.apache.thrift.TProcessor; @@ -17,8 +15,6 @@ public class ParquetServer implements Runnable { private static final Logger logger = LoggerFactory.getLogger(ParquetServer.class); - private static final Config config = ConfigDescriptor.getInstance().getConfig(); - private final int port; private final Executor executor; @@ -34,8 +30,7 @@ private void startServer() throws TTransportException { TThreadPoolServer.Args args = new TThreadPoolServer .Args(serverTransport) .processor(processor) - .minWorkerThreads(config.getMinThriftWorkerThreadNum()) - .maxWorkerThreads(config.getMaxThriftWrokerThreadNum()); + .minWorkerThreads(20); args.protocolFactory(new TBinaryProtocol.Factory()); TServer server = new TThreadPoolServer(args); logger.info("parquet service starts successfully!"); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java index b4f7046bc5..69b66a2bea 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java @@ -16,7 +16,7 @@ import java.util.stream.Collectors; public class TagKVUtils { - @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(TagKVUtils.class); public static final String tagNameAnnotation = Config.tagNameAnnotation; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index a8de2089e4..40fee21573 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -140,8 +140,6 @@ private boolean testConnection() { @Override public TaskExecuteResult execute(StoragePhysicalTask task) { - logger.info("test begin!"); - logger.info(task.toString()); List operators = task.getOperators(); if (operators.size() != 1) { return new TaskExecuteResult( @@ -163,7 +161,6 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); } - logger.info("t1"); return executeProjectTask(project, filter); } else if (op.getType() == OperatorType.Insert) { Insert insert = (Insert) op; @@ -245,7 +242,7 @@ public Pair getBoundaryOfStorage(String prefix) t minTime = Math.min(currMinTime, minTime); } }catch (Exception e){ - minTime=1; //没有时间列，按递增的序列排序,1表示first + minTime=1000; //没有时间列，按递增的序列排序,1000表示first } // 获取last String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); @@ -260,7 +257,7 @@ public Pair getBoundaryOfStorage(String prefix) t }catch (Exception e){ //没有时间戳的表用行数表示last ResultSet lastQuerySet=lastQueryStmt.executeQuery(queryNumOfRow); if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); + long currMaxTime = lastQuerySet.getLong(1)*1000; maxTime = Math.max(currMaxTime, maxTime); } } @@ -279,18 +276,34 @@ public Pair getBoundaryOfStorage(String prefix) t private TaskExecuteResult executeProjectTask(Project project, Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 - logger.info("t2"); + String filter1=filter.toString().replace("key","time").replace("&&",filter.getType().toString()); + String[] filter_list=filter1.split("\\s"); + long ft=Long.parseLong(filter_list[6].substring(0,filter_list[6].indexOf(")"))); + while(ft>9223372036854L){ + ft=ft/10; + } + filter1=String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", + filter_list[1],Long.parseLong(filter_list[2]),filter.getType().toString(),filter_list[5],ft); try { List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); +// String db=""; + String db=""; + String t=""; + String c=""; for (String path : project.getPatterns()) { - logger.info("path======"+path); String database_table = path.substring(0, path.lastIndexOf('.')); String database=database_table.substring(0,database_table.lastIndexOf('.')); String table=database_table.substring(database_table.lastIndexOf('.')+1); database=database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + if(db.equals(database) && t.equals(table) && c.equals(field)){ + continue; + } + db=database; + t=table; + c=field; // 查询序列类型 useDatabase(database); @@ -305,22 +318,23 @@ private TaskExecuteResult executeProjectTask(Project project, if(field.equals("time")){ continue; } - logger.info(field); String statement=""; - logger.info("t70"); String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR,IGINX_SEPARATOR)+IGINX_SEPARATOR+table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); - logger.info("t69"); - - statement = String - .format("SELECT time,%s FROM %s", field, table); - logger.info(statement); - - - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(statement); + ResultSet rs; + try { + statement = String + .format("SELECT time,%s FROM %s where %s", field, table, filter1); + Statement stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + }catch (Exception e){ + statement = String + .format("SELECT time,%s FROM %s", field, table); + Statement stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } resultSets.add(rs); } } @@ -417,7 +431,6 @@ private void useDatabase(String dbname) { } private Exception insertRowRecords(RowDataView data) { - logger.info("t4"); int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = connection.createStatement(); @@ -461,10 +474,6 @@ private Exception insertRowRecords(RowDataView data) { stmt.addBatch(String .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, columnsKeys, time, columnValues)); -// logger.info(table); -// logger.info(columnsKeys.toString()); -// logger.info(String.valueOf(time)); -// logger.info(columnValues.toString()); if (index > 0 && (index + 1) % batchSize == 0) { stmt.executeBatch(); } @@ -482,16 +491,20 @@ private Exception insertRowRecords(RowDataView data) { } private Exception insertColumnRecords(ColumnDataView data) { - logger.info("t7"); int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = connection.createStatement(); + String columnsKeys=""; + String columnValues=""; + String table=""; + long time=0; for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); + logger.info("insert path====="+path); DataType dataType = data.getDataType(i); String database_table = path.substring(0, path.lastIndexOf('.')); String database=database_table.substring(0,database_table.lastIndexOf('.')); - String table=database_table.substring(database_table.lastIndexOf('.')+1); + table=database_table.substring(database_table.lastIndexOf('.')+1); database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); @@ -502,15 +515,12 @@ private Exception insertColumnRecords(ColumnDataView data) { useDatabase(database); stmt = connection.createStatement(); - logger.info("t8"); createTimeSeriesIfNotExists(table, field, tags, dataType); - logger.info("t11"); BitmapView bitmapView = data.getBitmapView(i); int index = 0; - logger.info("t10"); for (int j = 0; j < data.getTimeSize(); j++) { if (bitmapView.get(j)) { - long time = data.getKey(j) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + time = data.getKey(j) / 1000; // 时间戳 String value; if (data.getDataType(i) == DataType.BINARY) { value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) @@ -518,31 +528,25 @@ private Exception insertColumnRecords(ColumnDataView data) { } else { value = data.getValue(i, index).toString(); } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - logger.info("t9"); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, - time, - columnValues)); + columnsKeys=columnsKeys+","+field; + columnValues=columnValues+","+value; if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); + stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues)); + columnsKeys=""; + columnValues=""; } index++; } } } - stmt.executeBatch(); - logger.info("t8"); + String s=String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues); + stmt.execute(s); } catch (SQLException e) { return e; } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 25e6f33629..1756fe7af0 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -8,12 +8,16 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; +import cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class PostgreSQLQueryRowStream implements RowStream { private final List resultSets; + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); private final long[] currTimestamps; @@ -21,10 +25,6 @@ public class PostgreSQLQueryRowStream implements RowStream { private final Header header; -// private Object[] values; -// private Object[] value; -// private final Long timestamp; - public PostgreSQLQueryRowStream(List resultSets, List fields) { this.resultSets = resultSets; this.header = new Header(Field.KEY, fields); @@ -33,10 +33,15 @@ public PostgreSQLQueryRowStream(List resultSets, List fields) // this.values=new ArrayList<>(); // 默认填充一下timestamp列表 try { + long j=1; for (int i = 0; i < this.currTimestamps.length; i++) { ResultSet resultSet = this.resultSets.get(i); if (resultSet.next()) { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + }catch (Exception e){ + this.currTimestamps[i]=j++; + } this.currValues[i] = resultSet.getObject(2); } } @@ -65,7 +70,7 @@ public void close() { @Override public boolean hasNext() throws PhysicalException { for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { + if (currTimestamp != Long.MIN_VALUE && currTimestamp!=0) { return true; } } @@ -82,13 +87,18 @@ public Row next() throws PhysicalException { timestamp = Math.min(timestamp, currTimestamp); } } - + long j=1; for (int i = 0; i < this.currTimestamps.length; i++) { if (this.currTimestamps[i] == timestamp) { values[i] = this.currValues[i]; ResultSet resultSet = this.resultSets.get(i); if (resultSet.next()) { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + }catch (Exception e){ + //this.currTimestamps[i]=j++; + logger.info("have no timestamp,set default timestamp!"); + } this.currValues[i] = resultSet.getObject(2); } else { // 值已经取完 diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java index 829bf422f9..5e170d4d67 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java @@ -239,7 +239,7 @@ public static ByteBuffer getRowByteBuffer(Object[] values, List dataTy buffer.putInt((int) value); break; case LONG: - buffer.putLong((long) value); + buffer.putLong(((Number)value).longValue()); break; case FLOAT: buffer.putFloat((float) value); From acd1a5f1c44a09423db9024c962d5654f09e4fa7 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 24 Feb 2023 12:28:26 +0800 Subject: [PATCH 04/94] fix bugs --- dataSources/dmdb/test.java | 306 ------------------ .../iginx/influxdb/InfluxDBStorage.java | 4 +- .../influxdb/query/entity/InfluxDBSchema.java | 3 - .../iginx/influxdb/tools/TagFilterUtils.java | 9 +- .../query/entity/IoTDBQueryRowStream.java | 1 - .../iginx/iotdb/tools/TagKVUtils.java | 2 +- .../query/entity/OpenTSDBRowStream.java | 1 + .../iginx/opentsdb/tools/TagKVUtils.java | 9 +- .../iginx/parquet/ParquetStorage.java | 5 +- .../tsinghua/iginx/parquet/exec/Executor.java | 1 - .../iginx/parquet/server/ParquetServer.java | 7 +- .../iginx/parquet/tools/TagKVUtils.java | 2 +- .../entity/PostgreSQLQueryRowStream.java | 1 - .../postgresql/tools/TagFilterUtils.java | 8 +- 14 files changed, 34 insertions(+), 325 deletions(-) delete mode 100644 dataSources/dmdb/test.java diff --git a/dataSources/dmdb/test.java b/dataSources/dmdb/test.java deleted file mode 100644 index 6ea82351ee..0000000000 --- a/dataSources/dmdb/test.java +++ /dev/null @@ -1,306 +0,0 @@ -/*该例程实现插入数据，修改数据，删除数据，数据查询等基本操作。*/ -import java.awt.Color; -import java.awt.Font; -import java.awt.Graphics2D; -import java.awt.font.FontRenderContext; -import java.awt.geom.Rectangle2D; -import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.math.BigDecimal; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.Date; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import javax.imageio.ImageIO; -public class BasicApp { - // 定义DM JDBC驱动串 - String jdbcString = "dm.jdbc.driver.DmDriver"; - // 定义DM URL连接串 - String urlString = "jdbc:dm://localhost:5236"; - // 定义连接用户名 - String userName = "SYSDBA"; - // 定义连接用户口令 - String password = "SYSDBA001"; - // 定义连接对象 - Connection conn = null; - /* 加载JDBC驱动程序 - * @throws SQLException 异常 */ - public void loadJdbcDriver() throws SQLException { - try { - System.out.println("Loading JDBC Driver..."); - // 加载JDBC驱动程序 - Class.forName(jdbcString); - } catch (ClassNotFoundException e) { - throw new SQLException("Load JDBC Driver Error : " + e.getMessage()); - } catch (Exception ex) { - throw new SQLException("Load JDBC Driver Error : " - + ex.getMessage()); - } - } - /* 连接DM数据库 - * @throws SQLException 异常 */ - public void connect() throws SQLException { - try { - System.out.println("Connecting to DM Server..."); - // 连接DM数据库 - conn = DriverManager.getConnection(urlString, userName, password); - } catch (SQLException e) { - throw new SQLException("Connect to DM Server Error : " - + e.getMessage()); - } - } - /* 关闭连接 - * @throws SQLException 异常 */ - public void disConnect() throws SQLException { - try { - // 关闭连接 - conn.close(); - } catch (SQLException e) { - throw new SQLException("close connection error : " + e.getMessage()); - } - } - /* 往产品信息表插入数据 - * @throws SQLException 异常 */ - public void insertTable() throws SQLException { - // 插入数据语句 - String sql = "INSERT INTO production.product(name,author,publisher,publishtime," - + "product_subcategoryid,productno,satetystocklevel,originalprice,nowprice,discount," - + "description,photo,type,papertotal,wordtotal,sellstarttime,sellendtime) " - + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);"; - // 创建语句对象 - PreparedStatement pstmt = conn.prepareStatement(sql); - // 为参数赋值 - pstmt.setString(1, "三国演义"); - pstmt.setString(2, "罗贯中"); - pstmt.setString(3, "中华书局"); - pstmt.setDate(4, Date.valueOf("2005-04-01")); - pstmt.setInt(5, 4); - pstmt.setString(6, "9787101046121"); - pstmt.setInt(7, 10); - pstmt.setBigDecimal(8, new BigDecimal(19.0000)); - pstmt.setBigDecimal(9, new BigDecimal(15.2000)); - pstmt.setBigDecimal(10, new BigDecimal(8.0)); - pstmt.setString(11, "《三国演义》是中国第一部长篇章回体小说，中国小说由短篇发展至长篇的原因与说书有关。"); - // 设置大字段参数 - try { - // 创建一个图片用于插入大字段 - String filePath = "c:\\三国演义.jpg"; - CreateImage(filePath); - File file = new File(filePath); - InputStream in = new BufferedInputStream(new FileInputStream(file)); - pstmt.setBinaryStream(12, in, (int) file.length()); - } catch (FileNotFoundException e) { - System.out.println(e.getMessage()); - // 如果没有图片设置为NULL - pstmt.setNull(12, java.sql.Types.BINARY); - } catch (IOException e) { - System.out.println(e.getMessage()); - } - pstmt.setString(13, "25"); - pstmt.setInt(14, 943); - pstmt.setInt(15, 93000); - pstmt.setDate(16, Date.valueOf("2006-03-20")); - pstmt.setDate(17, Date.valueOf("1900-01-01")); - // 执行语句 - pstmt.executeUpdate(); - // 关闭语句 - pstmt.close(); - } - /* 查询产品信息表 - * @throws SQLException 异常 */ - public void queryProduct() throws SQLException { - // 查询语句 - String sql = "SELECT productid,name,author,description,photo FROM production.product WHERE productid=11"; - // 创建语句对象 - Statement stmt = conn.createStatement(); - // 执行查询 - ResultSet rs = stmt.executeQuery(sql); - // 显示结果集 - displayResultSet(rs); - // 关闭结果集 - rs.close(); - // 关闭语句 - stmt.close(); - } - /* 修改产品信息表数据 - * @throws SQLException 异常 */ - public void updateTable() throws SQLException { - // 更新数据语句 - String sql = "UPDATE production.product SET name = ?" - + "WHERE productid = 11;"; - // 创建语句对象 - PreparedStatement pstmt = conn.prepareStatement(sql); - // 为参数赋值 - pstmt.setString(1, "三国演义（上）"); - // 执行语句 - pstmt.executeUpdate(); - // 关闭语句 - pstmt.close(); - } - /* 删除产品信息表数据 - * @throws SQLException 异常 */ - public void deleteTable() throws SQLException { - // 删除数据语句 - String sql = "DELETE FROM production.product WHERE productid = 11;"; - // 创建语句对象 - Statement stmt = conn.createStatement(); - // 执行语句 - stmt.executeUpdate(sql); - // 关闭语句 - stmt.close(); - } - /* 查询产品信息表 - * @throws SQLException 异常 */ - public void queryTable() throws SQLException { - // 查询语句 - String sql = "SELECT productid,name,author,publisher FROM production.product"; - // 创建语句对象 - Statement stmt = conn.createStatement(); - // 执行查询 - ResultSet rs = stmt.executeQuery(sql); - // 显示结果集 - displayResultSet(rs); - // 关闭结果集 - rs.close(); - // 关闭语句 - stmt.close(); - } - /* 调用存储过程修改产品信息表数据 - * @throws SQLException 异常 */ - public void updateProduct() throws SQLException { - // 更新数据语句 - String sql = "{ CALL production.updateProduct(?,?) }"; - // 创建语句对象 - CallableStatement cstmt = conn.prepareCall(sql); - // 为参数赋值 - cstmt.setInt(1, 1); - cstmt.setString(2, "红楼梦（上）"); - // 执行语句 - cstmt.execute(); - // 关闭语句 - cstmt.close(); - } - /* 显示结果集 - * @param rs 结果集对象 - * @throws SQLException 异常 */ - private void displayResultSet(ResultSet rs) throws SQLException { - // 取得结果集元数据 - ResultSetMetaData rsmd = rs.getMetaData(); - // 取得结果集所包含的列数 - int numCols = rsmd.getColumnCount(); - // 显示列标头 - for (int i = 1; i <= numCols; i++) { - if (i > 1) { - System.out.print(","); - } - System.out.print(rsmd.getColumnLabel(i)); - } - System.out.println(""); - // 显示结果集中所有数据 - while (rs.next()) { - for (int i = 1; i <= numCols; i++) { - if (i > 1) { - System.out.print(","); - } - // 处理大字段 - if ("IMAGE".equals(rsmd.getColumnTypeName(i))) { - byte[] data = rs.getBytes(i); - if (data != null && data.length > 0) { - FileOutputStream fos; - try { - fos = new FileOutputStream("c:\\三国演义1.jpg"); - fos.write(data); - fos.close(); - } catch (FileNotFoundException e) { - System.out.println(e.getMessage()); - } catch (IOException e) { - System.out.println(e.getMessage()); - } - } - System.out.print("字段内容已写入文件c:\\三国演义1.jpg，长度" + data.length); - } else { - // 普通字段 - System.out.print(rs.getString(i)); - } - } - System.out.println(""); - } - } - /* 创建一个图片用于插入大字段 - * @throws IOException 异常 */ - private void CreateImage(String path) throws IOException { - int width = 100; - int height = 100; - String s = "三国演义"; - File file = new File(path); - Font font = new Font("Serif", Font.BOLD, 10); - BufferedImage bi = new BufferedImage(width, height, - BufferedImage.TYPE_INT_RGB); - Graphics2D g2 = (Graphics2D) bi.getGraphics(); - g2.setBackground(Color.WHITE); - g2.clearRect(0, 0, width, height); - g2.setPaint(Color.RED); - FontRenderContext context = g2.getFontRenderContext(); - Rectangle2D bounds = font.getStringBounds(s, context); - double x = (width - bounds.getWidth()) / 2; - double y = (height - bounds.getHeight()) / 2; - double ascent = -bounds.getY(); - double baseY = y + ascent; - g2.drawString(s, (int) x, (int) baseY); - ImageIO.write(bi, "jpg", file); - } - - //类主方法 @param args 参数 - public static void main(String args[]) { - try { - // 定义类对象 - BasicApp basicApp = new BasicApp(); - // 加载驱动程序 - basicApp.loadJdbcDriver(); - // 连接DM数据库 - basicApp.connect(); - // 插入数据 - System.out.println("--- 插入产品信息 ---"); - basicApp.insertTable(); - // 查询含有大字段的产品信息 - System.out.println("--- 显示插入结果 ---"); - basicApp.queryProduct(); - // 在修改前查询产品信息表 - System.out.println("--- 在修改前查询产品信息 ---"); - basicApp.queryTable(); - // 修改产品信息表 - System.out.println("--- 修改产品信息 ---"); - basicApp.updateTable(); - // 在修改后查询产品信息表 - System.out.println("--- 在修改后查询产品信息 ---"); - basicApp.queryTable(); - // 删除产品信息表 - System.out.println("--- 删除产品信息 ---"); - basicApp.deleteTable(); - // 在删除后查询产品信息表 - System.out.println("--- 在删除后查询产品信息 ---"); - basicApp.queryTable(); - // 调用存储过程修改产品信息表 - System.out.println("--- 调用存储过程修改产品信息 ---"); - basicApp.updateProduct(); - // 在存储过程更新后查询产品信息表 - System.out.println("--- 调用存储过程后查询产品信息 ---"); - basicApp.queryTable(); - // 关闭连接 - basicApp.disConnect(); - } catch (SQLException e) { - System.out.println(e.getMessage()); - } - } -} diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java index 5ab00fa0bc..4553e08a9d 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java @@ -179,8 +179,8 @@ public Pair getBoundaryOfStorage(String dataPrefi bucket.getName(), 0L, Long.MAX_VALUE, - measurementPrefix == MEASUREMENTALL ? MEASUREMENTALL : "= \"" + measurementPrefix + "\"", - fieldPrefix == FIELDALL ? FIELDALL : "~ /" + fieldPrefix + ".*/" + measurementPrefix.equals(MEASUREMENTALL) ? MEASUREMENTALL : "= \"" + measurementPrefix + "\"", + fieldPrefix.equals(FIELDALL) ? FIELDALL : "~ /" + fieldPrefix + ".*/" ); logger.debug("execute statement: " + statement); // 查询 first diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java index 5e2b1177f9..b56d526e42 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/query/entity/InfluxDBSchema.java @@ -18,10 +18,7 @@ */ package cn.edu.tsinghua.iginx.influxdb.query.entity; -import cn.edu.tsinghua.iginx.utils.Pair; - import java.util.Collections; -import java.util.HashMap; import java.util.Map; public class InfluxDBSchema { diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java index 96acefffec..3aa257b547 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/tools/TagFilterUtils.java @@ -5,8 +5,6 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import java.util.List; - public class TagFilterUtils { public static String transformToFilterStr(TagFilter filter) { @@ -48,6 +46,13 @@ private static void transformToFilterStr(TagFilter filter, StringBuilder builder builder.append(" == \"").append(baseFilter.getTagValue()).append("\" "); } break; + // TODO: case label + case BasePrecise: + break; + case Precise: + break; + case WithoutTag: + break; } } diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java index 2d9de649dd..e8caf6aed4 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/query/entity/IoTDBQueryRowStream.java @@ -26,7 +26,6 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.Project; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.iotdb.IoTDBStorage; import cn.edu.tsinghua.iginx.iotdb.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.iotdb.tools.TagKVUtils; import cn.edu.tsinghua.iginx.utils.Pair; diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java index a59f0112a5..3ef9e8a5c7 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/TagKVUtils.java @@ -33,7 +33,7 @@ import java.util.stream.Collectors; public class TagKVUtils { - + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(TagKVUtils.class); public static final String tagNameAnnotation = Config.tagNameAnnotation; diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java index 22a6f03872..635d209894 100644 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java +++ b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java @@ -35,6 +35,7 @@ public class OpenTSDBRowStream implements RowStream { private int hasMoreRecords; + @SuppressWarnings("unchecked") public OpenTSDBRowStream(List resultList, boolean trimStorageUnit) { this.resultList = resultList; this.trimStorageUnit = trimStorageUnit; diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java index aa02bbc2cd..2c4e5d61c3 100644 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java +++ b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java @@ -36,7 +36,7 @@ import java.util.stream.Collectors; public class TagKVUtils { - + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(TagKVUtils.class); public static final String tagNameAnnotation = Config.tagNameAnnotation; @@ -82,6 +82,13 @@ public static boolean match(Map tags, TagFilter tagFilter) { return match(tags, (OrTagFilter) tagFilter); case Base: return match(tags, (BaseTagFilter) tagFilter); + // TODO: case label + case BasePrecise: + break; + case Precise: + break; + case WithoutTag: + break; } return false; } diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java index 4bcea67625..42cca09cc5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/ParquetStorage.java @@ -21,11 +21,8 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.parquet.exec.Executor; -import cn.edu.tsinghua.iginx.parquet.exec.LocalExecutor; import cn.edu.tsinghua.iginx.parquet.exec.NewExecutor; import cn.edu.tsinghua.iginx.parquet.exec.RemoteExecutor; -import cn.edu.tsinghua.iginx.parquet.policy.NaiveParquetStoragePolicy; -import cn.edu.tsinghua.iginx.parquet.policy.ParquetStoragePolicy; import cn.edu.tsinghua.iginx.parquet.server.ParquetServer; import cn.edu.tsinghua.iginx.parquet.tools.FilterTransformer; import cn.edu.tsinghua.iginx.utils.Pair; @@ -44,7 +41,7 @@ import org.slf4j.LoggerFactory; public class ParquetStorage implements IStorage { - + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(ParquetStorage.class); private static final Config config = ConfigDescriptor.getInstance().getConfig(); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java index 7f3d472502..9642d81370 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/Executor.java @@ -7,7 +7,6 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.TimeInterval; -import cn.edu.tsinghua.iginx.metadata.entity.TimeSeriesInterval; import cn.edu.tsinghua.iginx.metadata.entity.TimeSeriesRange; import cn.edu.tsinghua.iginx.utils.Pair; import java.util.List; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java index ce413a7987..84321042f6 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/server/ParquetServer.java @@ -1,5 +1,7 @@ package cn.edu.tsinghua.iginx.parquet.server; +import cn.edu.tsinghua.iginx.conf.Config; +import cn.edu.tsinghua.iginx.conf.ConfigDescriptor; import cn.edu.tsinghua.iginx.parquet.exec.Executor; import cn.edu.tsinghua.iginx.parquet.thrift.ParquetService; import org.apache.thrift.TProcessor; @@ -15,6 +17,8 @@ public class ParquetServer implements Runnable { private static final Logger logger = LoggerFactory.getLogger(ParquetServer.class); + private static final Config config = ConfigDescriptor.getInstance().getConfig(); + private final int port; private final Executor executor; @@ -30,7 +34,8 @@ private void startServer() throws TTransportException { TThreadPoolServer.Args args = new TThreadPoolServer .Args(serverTransport) .processor(processor) - .minWorkerThreads(20); + .minWorkerThreads(config.getMinThriftWorkerThreadNum()) + .maxWorkerThreads(config.getMaxThriftWrokerThreadNum()); args.protocolFactory(new TBinaryProtocol.Factory()); TServer server = new TThreadPoolServer(args); logger.info("parquet service starts successfully!"); diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java index 69b66a2bea..b4f7046bc5 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/TagKVUtils.java @@ -16,7 +16,7 @@ import java.util.stream.Collectors; public class TagKVUtils { - + @SuppressWarnings("unused") private static final Logger logger = LoggerFactory.getLogger(TagKVUtils.class); public static final String tagNameAnnotation = Config.tagNameAnnotation; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 1756fe7af0..37b9c9fafe 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -96,7 +96,6 @@ public Row next() throws PhysicalException { try { this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); }catch (Exception e){ - //this.currTimestamps[i]=j++; logger.info("have no timestamp,set default timestamp!"); } this.currValues[i] = resultSet.getObject(2); diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index 8d92d3697f..f9f6df9f5b 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -6,7 +6,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; public class TagFilterUtils { - + @SuppressWarnings("unused") public static String transformToFilterStr(TagFilter filter) { StringBuilder builder = new StringBuilder(); transformToFilterStr(filter, builder); @@ -43,6 +43,12 @@ private static void transformToFilterStr(TagFilter filter, StringBuilder builder builder.append("="); builder.append(baseFilter.getTagValue()); break; + // TODO: case label + case BasePrecise: + break; + case Precise: + case WithoutTag: + break; } } } \ No newline at end of file From 14453f7156f5bf229700ee710c50ee3b24fa22b5 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 24 Feb 2023 12:33:10 +0800 Subject: [PATCH 05/94] fixx --- .github/actions/confWriter/action.yml | 15 ++++- .github/workflows/api-rest.yml | 4 +- .github/workflows/api-session.yml | 4 +- .github/workflows/capacity-expansion.yml | 4 +- .github/workflows/codeql-analysis.yml | 73 ------------------------ .github/workflows/dependabot.yml | 11 ---- .github/workflows/ds-influxdb.yml | 21 ++++--- .github/workflows/ds-iotdb.yml | 22 ++++++- .github/workflows/ds-parquet.yml | 21 +++++-- .github/workflows/func-transform.yml | 4 +- .github/workflows/func-udf.yml | 4 +- .github/workflows/scale-out.yml | 4 +- .github/workflows/unit-mds.yml | 4 +- 13 files changed, 69 insertions(+), 122 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/dependabot.yml diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 146b1aea30..b3500eeb71 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -4,7 +4,7 @@ inputs: confFile: description: 'the conf that you want to write' required: false - default: "iginxConf" + default: "Conf" if-CapExp: description: 'if you need capacity expansion' required: false @@ -44,3 +44,16 @@ runs: shell: bash run: | echo "${{inputs.DB-name}}" > ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/DBConf.txt + + - if: inputs.confFile=='iginxConf' + name: Change conf/config.properties + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sed -i "s/port=6888/port=7888/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + sed -i "s/restPort=6666/restPort=7666/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + sed -i "" "s/port=6888/port=7888/" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + sed -i "" "restPort=6666/restPort=7666/" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + fi + diff --git a/.github/workflows/api-rest.yml b/.github/workflows/api-rest.yml index 0c794e33d6..99e853048e 100644 --- a/.github/workflows/api-rest.yml +++ b/.github/workflows/api-rest.yml @@ -1,10 +1,8 @@ name: "API-Test-RESTful" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/api-session.yml b/.github/workflows/api-session.yml index 08458a785a..a55d04f5f9 100644 --- a/.github/workflows/api-session.yml +++ b/.github/workflows/api-session.yml @@ -1,10 +1,8 @@ name: "API-Test-SESSIONv2" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/capacity-expansion.yml b/.github/workflows/capacity-expansion.yml index c5d7c111c5..2f525fa04c 100644 --- a/.github/workflows/capacity-expansion.yml +++ b/.github/workflows/capacity-expansion.yml @@ -1,9 +1,7 @@ name: "Capacity-Expansion-Test" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index d6fd0008b2..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,73 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: - - main - pull_request: - # The branches below must be a subset of the branches above - branches: - - main - schedule: - - cron: '22 22 * * 6' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: [ 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml deleted file mode 100644 index 23a3bbe5c3..0000000000 --- a/.github/workflows/dependabot.yml +++ /dev/null @@ -1,11 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "maven" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "monthly" diff --git a/.github/workflows/ds-influxdb.yml b/.github/workflows/ds-influxdb.yml index 18be290c77..4aca343ece 100644 --- a/.github/workflows/ds-influxdb.yml +++ b/.github/workflows/ds-influxdb.yml @@ -1,10 +1,8 @@ name: "ITTest-ds-InfluxDB" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: @@ -72,7 +70,6 @@ jobs: chmod +x "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" nohup "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" & - InfluxDB-SQL-ds: strategy: fail-fast: false @@ -193,9 +190,19 @@ jobs: - name: Install with Maven run: mvn clean package -DskipTests - name: Start IginX - run: | - chmod +x "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" - nohup "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" & + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + + - name: Change Config to Start IGinX_2 + uses: ./.github/actions/confWriter + with: + confFile: "iginxConf" + + - name: Start IginX_2 + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} - name: A Lame Integration Test with Maven for SQL run: mvn test -q -Dtest=InfluxDBSQLSessionPoolIT -DfailIfNoTests=false - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/ds-iotdb.yml b/.github/workflows/ds-iotdb.yml index cd58bf872e..ad4290ea5e 100644 --- a/.github/workflows/ds-iotdb.yml +++ b/.github/workflows/ds-iotdb.yml @@ -1,10 +1,8 @@ name: "ITTest-ds-IoTDB" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: @@ -179,6 +177,14 @@ jobs: with: version: ${VERSION} + - name: Change Config to Start IGinX_2 + uses: ./.github/actions/confWriter + + - name: Start IginX_2 + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + - name: A Lame Integration Test with Maven for IoTDB run: | if [ ${{ matrix.iotdb-version }} == "iotdb12" ]; then @@ -226,6 +232,16 @@ jobs: with: version: ${VERSION} + - name: Change Config to Start IGinX_2 + uses: ./.github/actions/confWriter + with: + confFile: "iginxConf" + + - name: Start IginX_2 + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + - name: A Lame Integration Test with Maven for SQL run: mvn test -q -Dtest=SQLSessionPoolIT -DfailIfNoTests=false - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/ds-parquet.yml b/.github/workflows/ds-parquet.yml index 53733a4efb..61740a1c00 100644 --- a/.github/workflows/ds-parquet.yml +++ b/.github/workflows/ds-parquet.yml @@ -1,10 +1,8 @@ name: "ITTest-ds-Parquet" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: @@ -134,10 +132,21 @@ jobs: fi - name: Install with Maven run: mvn clean package -DskipTests + - name: Start IginX - run: | - chmod +x "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" - nohup "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/sbin/start_iginx.sh" & + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + + - name: Change Config to Start IGinX_2 + uses: ./.github/actions/confWriter + with: + confFile: "iginxConf" + + - name: Start IginX_2 + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} - name: A Lame Integration Test with Maven for SQL run: mvn test -q -Dtest=ParquetSQLSessionPoolIT -DfailIfNoTests=false - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/func-transform.yml b/.github/workflows/func-transform.yml index 429da548b8..ac52471ebe 100644 --- a/.github/workflows/func-transform.yml +++ b/.github/workflows/func-transform.yml @@ -1,10 +1,8 @@ name: "Function-Test-Transform" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/func-udf.yml b/.github/workflows/func-udf.yml index 7c6b0f2faa..1f37a2bfe6 100644 --- a/.github/workflows/func-udf.yml +++ b/.github/workflows/func-udf.yml @@ -1,10 +1,8 @@ name: "Function-Test-UDF" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/scale-out.yml b/.github/workflows/scale-out.yml index 2dc9985f99..00e239f22c 100644 --- a/.github/workflows/scale-out.yml +++ b/.github/workflows/scale-out.yml @@ -1,9 +1,7 @@ name: "Scale-out-On-IoTDB" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main env: diff --git a/.github/workflows/unit-mds.yml b/.github/workflows/unit-mds.yml index 129804190d..bd158d9b0f 100644 --- a/.github/workflows/unit-mds.yml +++ b/.github/workflows/unit-mds.yml @@ -1,10 +1,8 @@ name: "Metadata-Service-Test" on: - push: - branches: - - main pull_request: + types: [opened, reopened] branches: - main From fb918d6ff784bc5c406c7642379ec7122b02a904 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 24 Feb 2023 16:02:38 +0800 Subject: [PATCH 06/94] fix bug --- .../PostgreSQLHistoryDataCapacityExpansionIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java index 18eabf69ad..84e5d7a70e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java @@ -1,4 +1,4 @@ -package cn.edu.tsinghua.iginx.integration.expansion.postgresql +package cn.edu.tsinghua.iginx.integration.expansion.postgresql; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.SQLSessionIT; @@ -56,10 +56,10 @@ private static void test_compare(Connection connection,String statement,String e } } if(act_true){ - logger.info("testQueryHistoryDataFromInitialNode is ok!") + logger.info("testQueryHistoryDataFromInitialNode is ok!"); } else{ - logger.info("testQueryHistoryDataFromInitialNode have some problems!") + logger.info("testQueryHistoryDataFromInitialNode have some problems!"); } } From fb9482a0502da77a56ece5dee621de598189793f Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 24 Feb 2023 16:15:20 +0800 Subject: [PATCH 07/94] fix test --- ...tgreSQLHistoryDataCapacityExpansionIT.java | 486 ------------------ .../PostgreSQLHistoryDataGenerator.java | 87 ---- 2 files changed, 573 deletions(-) delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java deleted file mode 100644 index 84e5d7a70e..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java +++ /dev/null @@ -1,486 +0,0 @@ -package cn.edu.tsinghua.iginx.integration.expansion.postgresql; - -import cn.edu.tsinghua.iginx.exceptions.SessionException; -import cn.edu.tsinghua.iginx.integration.SQLSessionIT; -import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; -import cn.edu.tsinghua.iginx.integration.expansion.unit.SQLTestTools; -import cn.edu.tsinghua.iginx.session.Session; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PostgreSQLHistoryDataCapacityExpansionIT implements BaseCapacityExpansionIT { - - private static final Logger logger = LoggerFactory.getLogger(SQLSessionIT.class); - - private static Connection connection; - - private String ENGINE_TYPE; - - public PostgreSQLHistoryDataCapacityExpansionIT(String engineType) { - this.ENGINE_TYPE = engineType; - } - - @BeforeClass - public static void setUp() { -// session = new Session("127.0.0.1", 6888, "root", "root"); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); -// Connection connection = DriverManager.getConnection(connUrl); -// Statement stmt = connection.createStatement(); - try { -// session.openSession(); - Connection connection = DriverManager.getConnection(connUrl); - } catch (SessionException e) { - logger.error(e.getMessage()); - } - } - - private static void test_compare(Connection connection,String statement,String expect){ - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); - connection = DriverManager.getConnection(connUrl); - Statement stmt = connection.createStatement(); - ResultSet rs=stmt.execute(expect); - boolean act_true=true; - while(rs.next()) { - act_true=false; - String real=rs.getString("2"); - if (expect.contains(real)){ - act_true=true; - } - else{ - break; - } - } - if(act_true){ - logger.info("testQueryHistoryDataFromInitialNode is ok!"); - } - else{ - logger.info("testQueryHistoryDataFromInitialNode have some problems!"); - } - } - - @AfterClass - public static void tearDown() { - try { - connection.close(); - } catch (SessionException e) { - logger.error(e.getMessage()); - } - } - - @Test - public void oriHasDataExpHasData() throws Exception { - testQueryHistoryDataFromInitialNode(); - testQueryAfterInsertNewData(); - testCapacityExpansion_oriHasDataExpHasData(); - testWriteAndQueryAfterCapacityExpansion_oriHasDataExpHasData(); - } - - @Test - public void oriHasDataExpNoData() throws Exception { - testQueryHistoryDataFromInitialNode(); - testQueryAfterInsertNewData(); - testCapacityExpansion_oriHasDataExpNoData(); - testWriteAndQueryAfterCapacityExpansion_oriHasDataExpNoData(); - } - - @Test - public void oriNoDataExpHasData() throws Exception { - testQueryHistoryDataFromNoInitialNode(); - testQueryAfterInsertNewDataFromNoInitialNode(); - testCapacityExpansion_oriNoDataExpHasData(); - testWriteAndQueryAfterCapacityExpansion_oriNoDataExpHasData(); - } - - @Test - public void oriNoDataExpNoData() throws Exception { - testQueryHistoryDataFromNoInitialNode(); - testQueryAfterInsertNewDataFromNoInitialNode(); - testCapacityExpansion_oriNoDataExpNoData(); - testWriteAndQueryAfterCapacityExpansion_oriNoDataExpNoData(); - } - - //@Test - public void testQueryHistoryDataFromInitialNode() throws Exception { - String statement = "select * from *"; - String expect = "ResultSets:\n" + - "+----+-------------------+------------------------+\n" + - "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|\n" + - "+----+-------------------+------------------------+\n" + - "| 100| true| null|\n" + - "| 200| false| 20.71|\n" + - "+----+-------------------+------------------------+\n" + - "Total line number = 2\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 3\n"; - test_compare(connection, statement, expect); - - statement = "select count(*) from ln.wf01"; - expect = "ResultSets:\n" + - "+--------------------------+-------------------------------+\n" + - "|count(ln.wf01.wt01.status)|count(ln.wf01.wt01.temperature)|\n" + - "+--------------------------+-------------------------------+\n" + - "| 2| 1|\n" + - "+--------------------------+-------------------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - - } - - public void testQueryHistoryDataFromNoInitialNode() throws Exception { - String statement = "select * from ln"; - String expect = "ResultSets:\n" + - "+----+\n" + - "|Time|\n" + - "+----+\n" + - "+----+\n" + - "Empty set.\n"; - test_compare(connection, statement, expect); - - - statement = "count points"; - expect = "Points num: 0\n"; - test_compare(connection, statement, expect); - } - - //@Test - public void testQueryAfterInsertNewData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("insert into ln.wf02 (time, status, version) values (100, true, \"v1\");"); - stmt.execute("insert into ln.wf02 (time, status, version) values (400, false, \"v4\");"); - stmt.execute("insert into ln.wf02 (time, version) values (800, \"v8\");"); - - String statement = "select * from ln"; - String expect = "ResultSets:\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "| 100| true| null| true| v1|\n" + - "| 200| false| 20.71| null| null|\n" + - "| 400| null| null| false| v4|\n" + - "| 800| null| null| null| v8|\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "Total line number = 4\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 8\n"; - test_compare(connection, statement, expect); - - statement = "select count(*) from ln.wf02"; - expect = "ResultSets:\n" + - "+---------------------+----------------------+\n" + - "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + - "+---------------------+----------------------+\n" + - "| 2| 3|\n" + - "+---------------------+----------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - } - - public void testQueryAfterInsertNewDataFromNoInitialNode() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("insert into ln.wf02 (time, status, version) values (100, true, \"v1\");"); - stmt.execute("insert into ln.wf02 (time, status, version) values (400, false, \"v4\");"); - stmt.execute("insert into ln.wf02 (time, version) values (800, \"v8\");"); - - String statement = "select * from ln"; - String expect = "ResultSets:\n" + - "+----+--------------+---------------+\n" + - "|Time|ln.wf02.status|ln.wf02.version|\n" + - "+----+--------------+---------------+\n" + - "| 100| true| v1|\n" + - "| 400| false| v4|\n" + - "| 800| null| v8|\n" + - "+----+--------------+---------------+\n" + - "Total line number = 3\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 5\n"; - test_compare(connection, statement, expect); - - statement = "select count(*) from ln.wf02"; - expect = "ResultSets:\n" + - "+---------------------+----------------------+\n" + - "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + - "+---------------------+----------------------+\n" + - "| 2| 3|\n" + - "+---------------------+----------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - } - - //@Test - public void testCapacityExpansion_oriHasDataExpNoData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:no, is_read_only:true\");"); - - String statement = "select * from ln.wf03"; - String expect = "ResultSets:\n" + - "+----+\n" + - "|Time|\n" + - "+----+\n" + - "+----+\n" + - "Empty set.\n"; - test_compare(connection, statement, expect); - - statement = "select * from ln"; - expect = "ResultSets:\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "| 100| true| null| true| v1|\n" + - "| 200| false| 20.71| null| null|\n" + - "| 400| null| null| false| v4|\n" + - "| 800| null| null| null| v8|\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "Total line number = 4\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 8\n"; - test_compare(connection, statement, expect); - - } - - public void testCapacityExpansion_oriHasDataExpHasData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:true, is_read_only:true\");"); - - String statement = "select * from ln.wf03"; - String expect = "ResultSets:\n" + - "+----+-------------------+------------------------+\n" + - "|Time|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + - "+----+-------------------+------------------------+\n" + - "| 77| true| null|\n" + - "| 200| false| 77.71|\n" + - "+----+-------------------+------------------------+\n" + - "Total line number = 2\n"; - test_compare(connection, statement, expect); - - statement = "select * from ln"; - expect = "ResultSets:\n" + - "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + - "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + - "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + - "| 77| null| null| null| null| true| null|\n" + - "| 100| true| null| true| v1| null| null|\n" + - "| 200| false| 20.71| null| null| false| 77.71|\n" + - "| 400| null| null| false| v4| null| null|\n" + - "| 800| null| null| null| v8| null| null|\n" + - "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + - "Total line number = 5\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 11\n"; - test_compare(connection, statement, expect); - - } - - public void testCapacityExpansion_oriNoDataExpHasData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:true, is_read_only:true\");"); - - String statement = "select * from ln.wf03"; - String expect = "ResultSets:\n" + - "+----+-------------------+------------------------+\n" + - "|Time|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + - "+----+-------------------+------------------------+\n" + - "| 77| true| null|\n" + - "| 200| false| 77.71|\n" + - "+----+-------------------+------------------------+\n" + - "Total line number = 2\n"; - test_compare(connection, statement, expect); - - statement = "select * from ln"; - expect = "ResultSets:\n" + - "+----+--------------+---------------+-------------------+------------------------+\n" + - "|Time|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + - "+----+--------------+---------------+-------------------+------------------------+\n" + - "| 77| null| null| true| null|\n" + - "| 100| true| v1| null| null|\n" + - "| 200| null| null| false| 77.71|\n" + - "| 400| false| v4| null| null|\n" + - "| 800| null| v8| null| null|\n" + - "+----+--------------+---------------+-------------------+------------------------+\n" + - "Total line number = 5\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 8\n"; - test_compare(connection, statement, expect); - - } - - public void testCapacityExpansion_oriNoDataExpNoData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("ADD STORAGEENGINE (\"127.0.0.1\", 6668, \"" + ENGINE_TYPE + "\", \"username:root, password:root, sessionPoolSize:20, has_data:false, is_read_only:true\");"); - - String statement = "select * from ln.wf03"; - String expect = "ResultSets:\n" + - "+----+\n" + - "|Time|\n" + - "+----+\n" + - "+----+\n" + - "Empty set.\n"; - test_compare(connection, statement, expect); - - statement = "select * from ln"; - expect = "ResultSets:\n" + - "+----+--------------+---------------+\n" + - "|Time|ln.wf02.status|ln.wf02.version|\n" + - "+----+--------------+---------------+\n" + - "| 100| true| v1|\n" + - "| 400| false| v4|\n" + - "| 800| null| v8|\n" + - "+----+--------------+---------------+\n" + - "Total line number = 3\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 5\n"; - test_compare(connection, statement, expect); - - } - - //@Test - public void testWriteAndQueryAfterCapacityExpansion_oriHasDataExpHasData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); - - String statement = "select * from ln"; - String expect = "ResultSets:\n" + - "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + - "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + - "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + - "| 77| null| null| null| null| true| null|\n" + - "| 100| true| null| true| v1| null| null|\n" + - "| 200| false| 20.71| null| null| false| 77.71|\n" + - "| 400| null| null| false| v4| null| null|\n" + - "| 800| null| null| null| v8| null| null|\n" + - "|1600| null| null| null| v48| null| null|\n" + - "+----+-------------------+------------------------+--------------+---------------+-------------------+------------------------+\n" + - "Total line number = 6\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 12\n"; - test_compare(connection, statement, expect); - - statement = "select count(*) from ln.wf02"; - expect = "ResultSets:\n" + - "+---------------------+----------------------+\n" + - "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + - "+---------------------+----------------------+\n" + - "| 2| 4|\n" + - "+---------------------+----------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - } - - public void testWriteAndQueryAfterCapacityExpansion_oriNoDataExpHasData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); - - String statement = "select * from ln"; - String expect = "ResultSets:\n" + - "+----+--------------+---------------+-------------------+------------------------+\n" + - "|Time|ln.wf02.status|ln.wf02.version|ln.wf03.wt01.status|ln.wf03.wt01.temperature|\n" + - "+----+--------------+---------------+-------------------+------------------------+\n" + - "| 77| null| null| true| null|\n" + - "| 100| true| v1| null| null|\n" + - "| 200| null| null| false| 77.71|\n" + - "| 400| false| v4| null| null|\n" + - "| 800| null| v8| null| null|\n" + - "|1600| null| v48| null| null|\n" + - "+----+--------------+---------------+-------------------+------------------------+\n" + - "Total line number = 6\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 9\n"; - test_compare(connection, statement, expect); - - statement = "select count(*) from ln.wf02"; - expect = "ResultSets:\n" + - "+---------------------+----------------------+\n" + - "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + - "+---------------------+----------------------+\n" + - "| 2| 4|\n" + - "+---------------------+----------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - } - - public void testWriteAndQueryAfterCapacityExpansion_oriHasDataExpNoData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); - - String statement = "select * from ln"; - String expect = "ResultSets:\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "|Time|ln.wf01.wt01.status|ln.wf01.wt01.temperature|ln.wf02.status|ln.wf02.version|\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "| 100| true| null| true| v1|\n" + - "| 200| false| 20.71| null| null|\n" + - "| 400| null| null| false| v4|\n" + - "| 800| null| null| null| v8|\n" + - "|1600| null| null| null| v48|\n" + - "+----+-------------------+------------------------+--------------+---------------+\n" + - "Total line number = 5\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 9\n"; - test_compare(connection, statement, expect); - - statement = "select count(*) from ln.wf02"; - expect = "ResultSets:\n" + - "+---------------------+----------------------+\n" + - "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + - "+---------------------+----------------------+\n" + - "| 2| 4|\n" + - "+---------------------+----------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - } - - public void testWriteAndQueryAfterCapacityExpansion_oriNoDataExpNoData() throws Exception { - Statement stmt = connection.createStatement(); - stmt.execute("insert into ln.wf02 (time, version) values (1600, \"v48\");"); - - String statement = "select * from *"; - String expect = "ResultSets:\n" + - "+----+--------------+---------------+\n" + - "|Time|ln.wf02.status|ln.wf02.version|\n" + - "+----+--------------+---------------+\n" + - "| 100| true| v1|\n" + - "| 400| false| v4|\n" + - "| 800| null| v8|\n" + - "|1600| null| v48|\n" + - "+----+--------------+---------------+\n" + - "Total line number = 4\n"; - test_compare(connection, statement, expect); - - statement = "count points"; - expect = "Points num: 6\n"; - test_compare(connection, statement, expect); - statement = "select count(*) from ln.wf02"; - expect = "ResultSets:\n" + - "+---------------------+----------------------+\n" + - "|count(ln.wf02.status)|count(ln.wf02.version)|\n" + - "+---------------------+----------------------+\n" + - "| 2| 4|\n" + - "+---------------------+----------------------+\n" + - "Total line number = 1\n"; - test_compare(connection, statement, expect); - } -} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java deleted file mode 100644 index a8842bece6..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ /dev/null @@ -1,87 +0,0 @@ -package cn.edu.tsinghua.iginx.integration.expansion.postgresql; - -import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; -import org.apache.iotdb.session.Session; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PostgreSQLHistoryDataGenerator implements BaseHistoryDataGenerator { - - private static final Logger logger = LoggerFactory.getLogger(IoTDBHistoryDataGenerator.class); - - @Test - public void oriHasDataExpHasData() throws Exception { - writeHistoryDataToA(); - writeHistoryDataToB(); - } - - @Test - public void oriHasDataExpNoData() throws Exception { - writeHistoryDataToA(); - } - - @Test - public void oriNoDataExpHasData() throws Exception { - writeHistoryDataToB(); - } - - @Test - public void oriNoDataExpNoData() throws Exception { - } - - @Test - public void clearData() { - try { -// Session sessionA = new Session("127.0.0.1", 6667, "root", "root"); -// sessionA.open(); -// sessionA.executeNonQueryStatement("DELETE STORAGE GROUP root.*"); -// sessionA.close(); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); - Connection connection = DriverManager.getConnection(connUrl); - Statement stmt = connection.createStatement(); - ResultSet rs=stmt.executeQuery("SELECT datname FROM pg_database"); - while(rs.next()){ - db=rs.next(); - if(db.contains("unit")) { - stmt.execute(String.format("drop database %s",db)); - } - } - connection.close(); - } catch (Exception e) { - logger.error(e.getMessage()); - } - - logger.info("clear data success!"); - } - - public void writeHistoryDataToA() throws Exception { -// Session session = new Session("127.0.0.1", 6667, "root", "root"); -// session.open(); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=postgres&password=postgres", meta.getIp(), meta.getPort()); - Connection connection = DriverManager.getConnection(connUrl); - Statement stmt = connection.createStatement(); - - stmt.execute("INSERT INTO root.ln.wf01.wt01(time,status) values(100,true);"); - stmt.execute("INSERT INTO root.ln.wf01.wt01(time,status,temperature) values(200,false,20.71);"); - - connection.close(); - - logger.info("write data to 127.0.0.1:5432 success!"); - } - - public void writeHistoryDataToB() throws Exception { -// Session session = new Session("127.0.0.1", 6668, "root", "root"); -// session.open(); -// -// session.executeNonQueryStatement("INSERT INTO root.ln.wf03.wt01(timestamp,status) values(77,true);"); -// session.executeNonQueryStatement("INSERT INTO root.ln.wf03.wt01(timestamp,status,temperature) values(200,false,77.71);"); -// -// session.close(); - - logger.info("write data to 127.0.0.1:6668 success!"); - } - -} From 8cb7100efde56fd920c8e444e11f6e383b2dbb8e Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 24 Feb 2023 21:01:18 +0800 Subject: [PATCH 08/94] fix bugs --- .../engine/physical/storage/IStorage.java | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 141 ++++++++++++------ .../entity/PostgreSQLQueryRowStream.java | 5 +- 3 files changed, 103 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java index 015a142fa4..f1e9756347 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java @@ -36,7 +36,7 @@ public interface IStorage { Pair getBoundaryOfStorage(String prefix) throws PhysicalException; - default Pair getBoundaryOfStorage() throws PhysicalException { + default Pair getBoundaryOfStorage() throws PhysicalException, SQLException { return getBoundaryOfStorage(null); } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 40fee21573..087a83341b 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -235,18 +235,23 @@ public Pair getBoundaryOfStorage(String prefix) t // 获取first String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); Statement firstQueryStmt = connection.createStatement(); + String execIfNoTimestamp_first=String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)",columnName,columnName,tableName); + String execIfNoTimestamp_last=String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)",columnName,columnName,tableName); try { ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); if (firstQuerySet.next()) { long currMinTime = firstQuerySet.getLong(1); minTime = Math.min(currMinTime, minTime); } - }catch (Exception e){ - minTime=1000; //没有时间列，按递增的序列排序,1000表示first + }catch (Exception e) { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 + } } // 获取last String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - String queryNumOfRow=String.format("select count(*) from %s",tableName); Statement lastQueryStmt = connection.createStatement(); try { ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); @@ -254,10 +259,10 @@ public Pair getBoundaryOfStorage(String prefix) t long currMaxTime = lastQuerySet.getLong(1); maxTime = Math.max(currMaxTime, maxTime); } - }catch (Exception e){ //没有时间戳的表用行数表示last - ResultSet lastQuerySet=lastQueryStmt.executeQuery(queryNumOfRow); + }catch (Exception e){ //没有时间戳执行该部分 + ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1)*1000; + long currMaxTime = lastQuerySet.getLong(1); maxTime = Math.max(currMaxTime, maxTime); } } @@ -292,50 +297,100 @@ private TaskExecuteResult executeProjectTask(Project project, String t=""; String c=""; for (String path : project.getPatterns()) { + ArrayList allDatabase=new ArrayList<>(); + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + allDatabase.add(databaseSet.getString(1)); + } + ArrayList databases=new ArrayList<>(); String database_table = path.substring(0, path.lastIndexOf('.')); - String database=database_table.substring(0,database_table.lastIndexOf('.')); - String table=database_table.substring(database_table.lastIndexOf('.')+1); - database=database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - if(db.equals(database) && t.equals(table) && c.equals(field)){ - continue; + String tableName=""; + String field1=""; + boolean allTable=false; + + if(path.equals("*.*")){ + stmt = connection.createStatement(); + databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + databases.add(databaseSet.getString(1)); + } + allTable=true; } - db=database; - t=table; - c=field; - // 查询序列类型 - useDatabase(database); - - DatabaseMetaData databaseMetaData = connection.getMetaData(); - - ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field); - if(field.equals("*")){ - columnSet = databaseMetaData.getColumns(null, null, table,null); + else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { + allTable=true; } - while (columnSet.next()) { - field=columnSet.getString("COLUMN_NAME"); - if(field.equals("time")){ - continue; + else { +// String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + tableName = database_table.substring(database_table.lastIndexOf('.') + 1); + if(tableName.equals("*")){ + allTable=true; } - String statement=""; - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR,IGINX_SEPARATOR)+IGINX_SEPARATOR+table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + databases.add(database); + field1 = path.substring(path.lastIndexOf('.') + 1); + field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); +// if (db.equals(database) && t.equals(table) && c.equals(field)) { +// continue; +// } +// db = database; +// t = table; +// c = field; + } + + for(int j=0;j resultSets, List fields) try { long j=1; for (int i = 0; i < this.currTimestamps.length; i++) { + j=1; ResultSet resultSet = this.resultSets.get(i); if (resultSet.next()) { try { this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); }catch (Exception e){ - this.currTimestamps[i]=j++; + this.currTimestamps[i] = j++; } this.currValues[i] = resultSet.getObject(2); } } + j++; } catch (SQLException e) { e.printStackTrace(); // pass } + } @Override From 74a12a6aa13372041efb33fe67db0bc78302bf1a Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Sat, 25 Feb 2023 14:45:59 +0800 Subject: [PATCH 09/94] fix bug --- .../iginx/postgresql/PostgreSQLStorage.java | 107 ++++++++---------- .../edu/tsinghua/iginx/utils/ByteUtils.java | 7 +- 2 files changed, 54 insertions(+), 60 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 087a83341b..c4e43f73aa 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -139,7 +139,7 @@ private boolean testConnection() { } @Override - public TaskExecuteResult execute(StoragePhysicalTask task) { + public TaskExecuteResult execute(StoragePhysicalTask task) throws SQLException { List operators = task.getOperators(); if (operators.size() != 1) { return new TaskExecuteResult( @@ -149,6 +149,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { Operator op = operators.get(0); String storageUnit = task.getStorageUnit(); // 先切换数据库 + //connection.close(); useDatabase(storageUnit); if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 @@ -182,6 +183,7 @@ public List getTimeSeries() throws PhysicalException { while (databaseSet.next()) { String databaseName = databaseSet.getString(1);//获取数据库名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { + //connection.close(); useDatabase(databaseName); DatabaseMetaData databaseMetaData = connection.getMetaData(); @@ -222,6 +224,7 @@ public Pair getBoundaryOfStorage(String prefix) t while (databaseSet.next()) { String databaseName = databaseSet.getString(1);//获取表名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { + //connection.close(); useDatabase(databaseName); DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); @@ -331,71 +334,56 @@ else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) databases.add(database); field1 = path.substring(path.lastIndexOf('.') + 1); field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); -// if (db.equals(database) && t.equals(table) && c.equals(field)) { -// continue; -// } -// db = database; -// t = table; -// c = field; } - for(int j=0;j(); - tags.put("id","32"); } - DatabaseMetaData databaseMetaData = connection.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); if (!tableSet.next()) { @@ -462,7 +448,7 @@ private void createTimeSeriesIfNotExists(String table, String field, } } - private void useDatabase(String dbname) { + private void useDatabase(String dbname) throws SQLException { try { Statement stmt = connection.createStatement(); stmt.execute(String.format("create database %s", dbname)); @@ -483,6 +469,7 @@ private void useDatabase(String dbname) { } catch (SQLException e) { logger.info("change database error", e); } + } private Exception insertRowRecords(RowDataView data) { @@ -504,6 +491,7 @@ private Exception insertRowRecords(RowDataView data) { field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); Map tags = data.getTags(i); + //connection.close(); useDatabase(database); stmt = connection.createStatement(); createTimeSeriesIfNotExists(table, field, tags, dataType); @@ -568,6 +556,7 @@ private Exception insertColumnRecords(ColumnDataView data) { tags=new HashMap<>(); } + // connection.close(); useDatabase(database); stmt = connection.createStatement(); createTimeSeriesIfNotExists(table, field, tags, dataType); diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java index 5e170d4d67..6e21cecebf 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java @@ -239,7 +239,12 @@ public static ByteBuffer getRowByteBuffer(Object[] values, List dataTy buffer.putInt((int) value); break; case LONG: - buffer.putLong(((Number)value).longValue()); + try{ + buffer.putLong((long) value); + } + catch(Exception e){ + buffer.putLong(((Number)value).longValue()); + } break; case FLOAT: buffer.putFloat((float) value); From 2f532fb7441aedc716d89dec1b1c0a5db5471f60 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Mon, 27 Feb 2023 10:58:55 +0800 Subject: [PATCH 10/94] fix insert and select --- conf/config.properties | 2 +- .../engine/physical/storage/IStorage.java | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 46 +++++++++++++------ .../session/PostgreSQLSessionExample.java | 26 ++++++----- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index f08f9c9e17..7e12032f40 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -25,7 +25,7 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool replicaNum=0 # 底层数据库类名 -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage # 内存任务执行线程池 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java index f1e9756347..015a142fa4 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/IStorage.java @@ -36,7 +36,7 @@ public interface IStorage { Pair getBoundaryOfStorage(String prefix) throws PhysicalException; - default Pair getBoundaryOfStorage() throws PhysicalException, SQLException { + default Pair getBoundaryOfStorage() throws PhysicalException { return getBoundaryOfStorage(null); } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index c4e43f73aa..dc39dd4c98 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -139,7 +139,7 @@ private boolean testConnection() { } @Override - public TaskExecuteResult execute(StoragePhysicalTask task) throws SQLException { + public TaskExecuteResult execute(StoragePhysicalTask task) { List operators = task.getOperators(); if (operators.size() != 1) { return new TaskExecuteResult( @@ -150,7 +150,11 @@ public TaskExecuteResult execute(StoragePhysicalTask task) throws SQLException { String storageUnit = task.getStorageUnit(); // 先切换数据库 //connection.close(); - useDatabase(storageUnit); + try { + useDatabase(storageUnit); + }catch (Exception e){ + logger.info("pass"); + } if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 Project project = (Project) op; @@ -307,6 +311,10 @@ private TaskExecuteResult executeProjectTask(Project project, allDatabase.add(databaseSet.getString(1)); } ArrayList databases=new ArrayList<>(); + String[] path_l=path.split("\\."); + if (path_l.length<3){ + path="postgres."+path; + } String database_table = path.substring(0, path.lastIndexOf('.')); String tableName=""; String field1=""; @@ -599,31 +607,43 @@ private Exception insertColumnRecords(ColumnDataView data) { } private TaskExecuteResult executeDeleteTask(Delete delete) { - // only support to the level of device now - // TODO support the delete to the level of sensor try { for (int i = 0; i < delete.getPatterns().size(); i++) { String path = delete.getPatterns().get(i); TimeRange timeRange = delete.getTimeRanges().get(i); - String table = path.substring(0, path.lastIndexOf('.')); + String db_ta=path.substring(0,path.lastIndexOf('.')); + String database = db_ta.substring(0, db_ta.lastIndexOf('.')); + String table=db_ta.substring(db_ta.lastIndexOf(',')+1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); // 查询序列类型 + useDatabase(database); DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (columnSet.next()) { - String statement = String - .format(DELETE_DATA, table, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = connection.createStatement(); - stmt.execute(statement); + ResultSet tableSet=null; + if(table.equals("*")){ + tableSet=databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } + else{ + tableSet=databaseMetaData.getTables(null,"%",table,new String[]{"table"}); + } + while(tableSet.next()) { + String tableName=tableSet.getString(3); + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); + if (columnSet.next()) { + String statement = String + .format(DELETE_DATA, tableName, + timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); + Statement stmt = connection.createStatement(); + stmt.execute(statement); + } } } return new TaskExecuteResult(null, null); } catch (SQLException e) { return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in timescaledb failure", + new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", e)); } } diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java index e942b6cbb9..5b285dee97 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java @@ -21,6 +21,8 @@ import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.thrift.DataType; +import org.apache.commons.lang3.RandomStringUtils; + import java.util.ArrayList; import java.util.List; @@ -45,9 +47,9 @@ public static void main(String[] args) throws SessionException, ExecutionExcepti session = new Session("127.0.0.1", 6888, "root", "root"); // 打开 Session session.openSession(); - // 行式插入对齐数据 -// insertRowRecords(); -// queryData(); +// 行式插入对齐数据 + insertRowRecords(); + queryData(); deleteDataInColumns(); // 关闭 Session session.closeSession(); @@ -57,20 +59,20 @@ private static void insertRowRecords() throws SessionException, ExecutionExcepti List paths = new ArrayList<>(); paths.add(S1); paths.add(S2); -// paths.add(S3); -// paths.add(S4); + paths.add(S3); + paths.add(S4); int size = (int) (ROW_END_TIMESTAMP - ROW_START_TIMESTAMP + 1); long[] timestamps = new long[size]; Object[] valuesList = new Object[size]; for (long i = 0; i < size; i++) { timestamps[(int) i] = ROW_START_TIMESTAMP + i; - Object[] values = new Object[2]; + Object[] values = new Object[4]; for (long j = 0; j < 4; j++) { if (j < 2) { values[(int) j] = i + j; } else { -// values[(int) j] = RandomStringUtils.randomAlphanumeric(10).getBytes(); + values[(int) j] = RandomStringUtils.randomAlphanumeric(10).getBytes(); } } valuesList[(int) i] = values; @@ -80,9 +82,9 @@ private static void insertRowRecords() throws SessionException, ExecutionExcepti for (int i = 0; i < 2; i++) { dataTypeList.add(DataType.LONG); } -// for (int i = 0; i < 2; i++) { -// dataTypeList.add(DataType.BINARY); -// } + for (int i = 0; i < 2; i++) { + dataTypeList.add(DataType.BINARY); + } session.insertRowRecords(paths, timestamps, valuesList, dataTypeList, null); } @@ -91,8 +93,8 @@ private static void queryData() throws SessionException, ExecutionException { List paths = new ArrayList<>(); paths.add(S1); paths.add(S2); -// paths.add(S3); -// paths.add(S4); + paths.add(S3); + paths.add(S4); long startTime = NON_ALIGNED_COLUMN_END_TIMESTAMP - 100L; long endTime = ROW_START_TIMESTAMP + 100L; From 9bdf794b7b63d31dc2667decb3a38a113411be59 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Mon, 27 Feb 2023 11:19:08 +0800 Subject: [PATCH 11/94] fix some error --- .../iginx/postgresql/PostgreSQLStorage.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index dc39dd4c98..16075d017e 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -299,10 +299,6 @@ private TaskExecuteResult executeProjectTask(Project project, try { List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); -// String db=""; - String db=""; - String t=""; - String c=""; for (String path : project.getPatterns()) { ArrayList allDatabase=new ArrayList<>(); Statement stmt = connection.createStatement(); @@ -313,7 +309,10 @@ private TaskExecuteResult executeProjectTask(Project project, ArrayList databases=new ArrayList<>(); String[] path_l=path.split("\\."); if (path_l.length<3){ - path="postgres."+path; + if(path.equals("*.*")){} + else { + path = "postgres." + path; + } } String database_table = path.substring(0, path.lastIndexOf('.')); String tableName=""; @@ -610,6 +609,13 @@ private TaskExecuteResult executeDeleteTask(Delete delete) { try { for (int i = 0; i < delete.getPatterns().size(); i++) { String path = delete.getPatterns().get(i); + String[] path_l=path.split("\\."); + if(path_l.length<3){ + if(path.equals("*.*")){} + else { + path = "postgres." + path; + } + } TimeRange timeRange = delete.getTimeRanges().get(i); String db_ta=path.substring(0,path.lastIndexOf('.')); String database = db_ta.substring(0, db_ta.lastIndexOf('.')); From 33d3291d597e63981d65361f50b2f1387e7a3f62 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 1 Mar 2023 10:24:21 +0800 Subject: [PATCH 12/94] fix code style and bugs --- dataSources/postgresql/pom.xml | 143 +- .../iginx/postgresql/PostgreSQLStorage.java | 1217 ++++++++--------- .../entity/PostgreSQLQueryRowStream.java | 203 ++- .../postgresql/tools/DataTypeTransformer.java | 67 +- .../postgresql/tools/FilterTransformer.java | 98 +- .../postgresql/tools/TagFilterUtils.java | 90 +- 6 files changed, 904 insertions(+), 914 deletions(-) diff --git a/dataSources/postgresql/pom.xml b/dataSources/postgresql/pom.xml index 9ca5c070ba..78da27ffc1 100644 --- a/dataSources/postgresql/pom.xml +++ b/dataSources/postgresql/pom.xml @@ -2,81 +2,82 @@ - 4.0.0 + 4.0.0 - - iginx - cn.edu.tsinghua - ${revision} - ../../pom.xml - + + iginx + cn.edu.tsinghua + ${revision} + ../../pom.xml + - postgresql - IGinX PostgreSQL + postgresql + IGinX PostgreSQL - - 8 - 8 - + + 8 + 8 + - - - cn.edu.tsinghua - iginx-core - ${project.version} - provided - - - org.postgresql - postgresql - 42.4.3 - - + + + cn.edu.tsinghua + iginx-core + ${project.version} + provided + + + org.postgresql + postgresql + 42.4.3 + + - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - package - - copy-dependencies - - - ../../core/target/iginx-core-${project.version}/driver/postgresql/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - package - - run - - - - - - - - - - - - - - - + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies + package + + copy-dependencies + + + ../../core/target/iginx-core-${project.version}/driver/postgresql/ + + provided + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + copy-native-libraries + package + + run + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 16075d017e..aad4ec9cdd 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -33,633 +33,632 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; -import cn.edu.tsinghua.iginx.engine.shared.operator.Delete; -import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; -import cn.edu.tsinghua.iginx.engine.shared.operator.Project; -import cn.edu.tsinghua.iginx.engine.shared.operator.Select; +import cn.edu.tsinghua.iginx.engine.shared.operator.*; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.postgresql.entity.PostgreSQLQueryRowStream; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; -import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; -import cn.edu.tsinghua.iginx.postgresql.tools.TagFilterUtils; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.*; import java.util.Map.Entry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class PostgreSQLStorage implements IStorage { - - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - - private static final int BATCH_SIZE = 10000; - - private static final String STORAGE_ENGINE = "postgresql"; - - private static final String USERNAME = "username"; - - private static final String PASSWORD = "password"; - - private static final String DBNAME = "dbname"; - - private static final String DEFAULT_USERNAME = "postgres"; - - private static final String DEFAULT_PASSWORD = "postgres"; - - private static final String DEFAULT_DBNAME = "timeseries"; - - private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; - - private static final String FIRST_QUERY = "select first(%s, time) from %s"; - - private static final String LAST_QUERY = "select last(%s, time) from %s"; - - private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; - - private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - - private static final String IGINX_SEPARATOR = "."; - - private static final String POSTGRESQL_SEPARATOR = "$"; - - private static final String DATABASE_PREFIX = "unit"; - - private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - private final StorageEngineMeta meta; - - private Connection connection; - - public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { - this.meta = meta; - if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - connection = DriverManager.getConnection(connUrl); - } catch (SQLException e) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - } - - private boolean testConnection() { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - Class.forName("org.postgresql.Driver"); - DriverManager.getConnection(connUrl); - return true; - } catch (SQLException | ClassNotFoundException e) { - return false; - } - } - - @Override - public TaskExecuteResult execute(StoragePhysicalTask task) { - List operators = task.getOperators(); - if (operators.size() != 1) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - FragmentMeta fragment = task.getTargetFragment(); - Operator op = operators.get(0); - String storageUnit = task.getStorageUnit(); - // 先切换数据库 - //connection.close(); - try { - useDatabase(storageUnit); - }catch (Exception e){ - logger.info("pass"); - } - - if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 - Project project = (Project) op; - Filter filter; - if (operators.size() == 2) { - filter = ((Select) operators.get(1)).getFilter(); - } else { - filter = new AndFilter(Arrays - .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), - new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); - } - return executeProjectTask(project, filter); - } else if (op.getType() == OperatorType.Insert) { - Insert insert = (Insert) op; - return executeInsertTask(insert); - } else if (op.getType() == OperatorType.Delete) { - Delete delete = (Delete) op; - return executeDeleteTask(delete); - } - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - - @Override - public List getTimeSeries() throws PhysicalException { - List timeseries = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取数据库名称 + + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); + + private static final int BATCH_SIZE = 10000; + + private static final String STORAGE_ENGINE = "postgresql"; + + private static final String USERNAME = "username"; + + private static final String PASSWORD = "password"; + + private static final String DBNAME = "dbname"; + + private static final String DEFAULT_USERNAME = "postgres"; + + private static final String DEFAULT_PASSWORD = "postgres"; + + private static final String DEFAULT_DBNAME = "timeseries"; + + private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; + + private static final String FIRST_QUERY = "select first(%s, time) from %s"; + + private static final String LAST_QUERY = "select last(%s, time) from %s"; + + private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; + + private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; + + private static final String IGINX_SEPARATOR = "."; + + private static final String POSTGRESQL_SEPARATOR = "$"; + + private static final String DATABASE_PREFIX = "unit"; + + private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + + private final StorageEngineMeta meta; + + private Connection connection; + + public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { + this.meta = meta; + if (!testConnection()) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), + username, password); + try { + connection = DriverManager.getConnection(connUrl); + } catch (SQLException e) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } + } + + private boolean testConnection() { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), + username, password); + try { + Class.forName("org.postgresql.Driver"); + DriverManager.getConnection(connUrl); + return true; + } catch (SQLException | ClassNotFoundException e) { + return false; + } + } + + @Override + public TaskExecuteResult execute(StoragePhysicalTask task) { + List operators = task.getOperators(); + if (operators.size() != 1) { + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); + } + FragmentMeta fragment = task.getTargetFragment(); + Operator op = operators.get(0); + String storageUnit = task.getStorageUnit(); + // 先切换数据库 + //connection.close(); + try { + useDatabase(storageUnit); + } catch (Exception e) { + logger.info("pass"); + } + + if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 + Project project = (Project) op; + Filter filter; + if (operators.size() == 2) { + filter = ((Select) operators.get(1)).getFilter(); + } else { + filter = new AndFilter(Arrays + .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), + new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); + } + return executeProjectTask(project, filter); + } else if (op.getType() == OperatorType.Insert) { + Insert insert = (Insert) op; + return executeInsertTask(insert); + } else if (op.getType() == OperatorType.Delete) { + Delete delete = (Delete) op; + return executeDeleteTask(delete); + } + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); + } + + @Override + public List getTimeSeries() throws PhysicalException { + List timeseries = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取数据库名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); + //connection.close(); + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); // DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - if (tableName.startsWith("unit")) { - tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); - } - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) - timeseries.add(new Timeseries( - databaseName.replace(POSTGRESQL_SEPARATOR,IGINX_SEPARATOR)+IGINX_SEPARATOR+ - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); - } - } + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + if (tableName.startsWith("unit")) { + tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); + } + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) + timeseries.add(new Timeseries( + databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName))); + } + } // } - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - return timeseries; - } - - @Override - public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { - long minTime = Long.MAX_VALUE, maxTime = 0; - List paths = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取表名称 + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return timeseries; + } + + @Override + public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { + long minTime = Long.MAX_VALUE, maxTime = 0; + List paths = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取表名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); - // 获取first - String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); - Statement firstQueryStmt = connection.createStatement(); - String execIfNoTimestamp_first=String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)",columnName,columnName,tableName); - String execIfNoTimestamp_last=String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)",columnName,columnName,tableName); - try { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); - } - }catch (Exception e) { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 - } - } - // 获取last - String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - Statement lastQueryStmt = connection.createStatement(); - try { - ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - }catch (Exception e){ //没有时间戳执行该部分 - ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } - } - } + //connection.close(); + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); + // 获取first + String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); + Statement firstQueryStmt = connection.createStatement(); + String execIfNoTimestamp_first = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); + String execIfNoTimestamp_last = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); + try { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); + } + } catch (Exception e) { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 + } + } + // 获取last + String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); + Statement lastQueryStmt = connection.createStatement(); + try { + ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } + } catch (Exception e) { //没有时间戳执行该部分 + ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } + } + } + } // } - } - } catch (SQLException e) { - throw new PhysicalException(e); - } - paths.sort(String::compareTo); - - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), - new TimeInterval(minTime, maxTime + 1)); - } - - private TaskExecuteResult executeProjectTask(Project project, - Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 - String filter1=filter.toString().replace("key","time").replace("&&",filter.getType().toString()); - String[] filter_list=filter1.split("\\s"); - long ft=Long.parseLong(filter_list[6].substring(0,filter_list[6].indexOf(")"))); - while(ft>9223372036854L){ - ft=ft/10; - } - filter1=String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", - filter_list[1],Long.parseLong(filter_list[2]),filter.getType().toString(),filter_list[5],ft); - try { - List resultSets = new ArrayList<>(); - List fields = new ArrayList<>(); - for (String path : project.getPatterns()) { - ArrayList allDatabase=new ArrayList<>(); - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - allDatabase.add(databaseSet.getString(1)); - } - ArrayList databases=new ArrayList<>(); - String[] path_l=path.split("\\."); - if (path_l.length<3){ - if(path.equals("*.*")){} - else { - path = "postgres." + path; - } - } - String database_table = path.substring(0, path.lastIndexOf('.')); - String tableName=""; - String field1=""; - boolean allTable=false; - - if(path.equals("*.*")){ - stmt = connection.createStatement(); - databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - databases.add(databaseSet.getString(1)); - } - allTable=true; - } - else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { - allTable=true; - } - else { + } + } catch (SQLException e) { + throw new PhysicalException(e); + } + paths.sort(String::compareTo); + + return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), + new TimeInterval(minTime, maxTime + 1)); + } + + private TaskExecuteResult executeProjectTask(Project project, + Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 + String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); + String[] filter_list = filter1.split("\\s"); + long ft = Long.parseLong(filter_list[6].substring(0, filter_list[6].indexOf(")"))); + while (ft > 9223372036854L) { + ft = ft / 10; + } + filter1 = String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", + filter_list[1], Long.parseLong(filter_list[2]), filter.getType().toString(), filter_list[5], ft); + try { + List resultSets = new ArrayList<>(); + List fields = new ArrayList<>(); + for (String path : project.getPatterns()) { + ArrayList allDatabase = new ArrayList<>(); + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + allDatabase.add(databaseSet.getString(1)); + } + ArrayList databases = new ArrayList<>(); + String[] path_l = path.split("\\."); + if (path_l.length < 3) { + if (path.equals("*.*")) { + } else { + path = "postgres." + path; + } + } + String database_table = path.substring(0, path.lastIndexOf('.')); + String tableName = ""; + String field1 = ""; + boolean allTable = false; + + if (path.equals("*.*")) { + stmt = connection.createStatement(); + databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + databases.add(databaseSet.getString(1)); + } + allTable = true; + } else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { + allTable = true; + } else { // String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - tableName = database_table.substring(database_table.lastIndexOf('.') + 1); - if(tableName.equals("*")){ - allTable=true; - } - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - databases.add(database); - field1 = path.substring(path.lastIndexOf('.') + 1); - field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - } - - for (int i = 0; i < databases.size(); i++) { - String database = (String) databases.get(i); - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = null; - if (allTable) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); - } - while (tableSet.next()) { - String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); - if (field1.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, table, null); - } - while (columnSet.next()) { - String field = columnSet.getString("COLUMN_NAME"); - if (field.equals("time")) { - continue; - } - String statement = ""; - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - ResultSet rs; - try { - statement = String - .format("SELECT time,%s FROM %s where %s", field, table, filter1); - stmt = connection.createStatement(); - rs = stmt.executeQuery(statement); - } catch (Exception e) { - statement = String - .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); - stmt = connection.createStatement(); - rs = stmt.executeQuery(statement); - } - resultSets.add(rs); - } - } - } - } - RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); - return new TaskExecuteResult(rowStream); - } - catch (SQLException e) { - logger.info("error: ",e); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", - e)); - } - } - - private TaskExecuteResult executeInsertTask(Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertRowRecords((RowDataView) dataView); - break; - case Column: - case NonAlignedColumn: - e = insertColumnRecords((ColumnDataView) dataView); - break; - } - if (e != null) { - return new TaskExecuteResult(null, - new PhysicalException("execute insert task in postgresql failure", e)); - } - return new TaskExecuteResult(null, null); - } - - private void createTimeSeriesIfNotExists(String table, String field, - Map tags, DataType dataType) { - try { - if (tags==null){ - tags=new HashMap<>(); - } - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = connection.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, - stringBuilder.toString())); - } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); - } - } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toPostgreSQL(dataType))); - } - } - } catch (SQLException e) { - logger.error("create timeseries error", e); - } - } - - private void useDatabase(String dbname) throws SQLException { - try { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("create database %s", dbname)); - } catch (SQLException e) { - logger.info("database exist!" ); - logger.info(dbname); - } - try { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), - dbname, username, password); - connection = DriverManager.getConnection(connUrl); - logger.info("change database success,the database is: ",dbname); - logger.info(dbname); - } catch (SQLException e) { - logger.info("change database error", e); - } - - } - - private Exception insertRowRecords(RowDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - for (int i = 0; i < data.getTimeSize(); i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - if (bitmapView.get(j)) { - String path = data.getPath(j); - DataType dataType = data.getDataType(j); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database=database_table.substring(0,database_table.lastIndexOf('.')); - String table=database_table.substring(database_table.lastIndexOf('.')+1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = data.getTags(i); - - //connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - - long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, time, columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); - } - - index++; - } - } - } - stmt.executeBatch(); - } catch (SQLException e) { - return e; - } - - return null; - } - - private Exception insertColumnRecords(ColumnDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - String columnsKeys=""; - String columnValues=""; - String table=""; - long time=0; - for (int i = 0; i < data.getPathNum(); i++) { - String path = data.getPath(i); - logger.info("insert path====="+path); - DataType dataType = data.getDataType(i); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database=database_table.substring(0,database_table.lastIndexOf('.')); - table=database_table.substring(database_table.lastIndexOf('.')+1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = data.getTags(i); - if (tags==null){ - tags=new HashMap<>(); - } - - // connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - time = data.getKey(j) / 1000; // 时间戳 - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - columnsKeys=columnsKeys+","+field; - columnValues=columnValues+","+value; - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues)); - columnsKeys=""; - columnValues=""; - } - index++; - } - } - } - String s=String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues); - stmt.execute(s); - } catch (SQLException e) { - return e; - } - - return null; - } - - private TaskExecuteResult executeDeleteTask(Delete delete) { - try { - for (int i = 0; i < delete.getPatterns().size(); i++) { - String path = delete.getPatterns().get(i); - String[] path_l=path.split("\\."); - if(path_l.length<3){ - if(path.equals("*.*")){} - else { - path = "postgres." + path; - } - } - TimeRange timeRange = delete.getTimeRanges().get(i); - String db_ta=path.substring(0,path.lastIndexOf('.')); - String database = db_ta.substring(0, db_ta.lastIndexOf('.')); - String table=db_ta.substring(db_ta.lastIndexOf(',')+1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - // 查询序列类型 - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet=null; - if(table.equals("*")){ - tableSet=databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } - else{ - tableSet=databaseMetaData.getTables(null,"%",table,new String[]{"table"}); - } - while(tableSet.next()) { - String tableName=tableSet.getString(3); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); - if (columnSet.next()) { - String statement = String - .format(DELETE_DATA, tableName, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = connection.createStatement(); - stmt.execute(statement); - } - } - } - return new TaskExecuteResult(null, null); - } catch (SQLException e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", - e)); - } - } - - @Override - public void release() throws PhysicalException { - try { - connection.close(); - } catch (SQLException e) { - throw new PhysicalException(e); - } - } + String database = database_table.substring(0, database_table.lastIndexOf('.')); + tableName = database_table.substring(database_table.lastIndexOf('.') + 1); + if (tableName.equals("*")) { + allTable = true; + } + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + databases.add(database); + field1 = path.substring(path.lastIndexOf('.') + 1); + field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + } + + for (int i = 0; i < databases.size(); i++) { + String database = (String) databases.get(i); + useDatabase(database); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = null; + if (allTable) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else { + tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); + } + while (tableSet.next()) { + String table = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); + if (field1.equals("*")) { + columnSet = databaseMetaData.getColumns(null, null, table, null); + } + while (columnSet.next()) { + String field = columnSet.getString("COLUMN_NAME"); + if (field.equals("time")) { + continue; + } + String statement = ""; + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + ResultSet rs; + try { + statement = String + .format("SELECT time,%s FROM %s where %s", field, table, filter1); + stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } catch (Exception e) { + statement = String + .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); + stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } + resultSets.add(rs); + } + } + } + } + RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.info("error: ", e); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", + e)); + } + } + + private TaskExecuteResult executeInsertTask(Insert insert) { + DataView dataView = insert.getData(); + Exception e = null; + switch (dataView.getRawDataType()) { + case Row: + case NonAlignedRow: + e = insertRowRecords((RowDataView) dataView); + break; + case Column: + case NonAlignedColumn: + e = insertColumnRecords((ColumnDataView) dataView); + break; + } + if (e != null) { + return new TaskExecuteResult(null, + new PhysicalException("execute insert task in postgresql failure", e)); + } + return new TaskExecuteResult(null, null); + } + + private void createTimeSeriesIfNotExists(String table, String field, + Map tags, DataType dataType) { + try { + if (tags == null) { + tags = new HashMap<>(); + } + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (!tableSet.next()) { + Statement stmt = connection.createStatement(); + StringBuilder stringBuilder = new StringBuilder(); + for (Entry tagsEntry : tags.entrySet()) { + stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + } + stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); + stmt.execute(String + .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, + stringBuilder.toString())); + } else { + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + } + } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, + DataTypeTransformer.toPostgreSQL(dataType))); + } + } + } catch (SQLException e) { + logger.error("create timeseries error", e); + } + } + + private void useDatabase(String dbname) throws SQLException { + try { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("create database %s", dbname)); + } catch (SQLException e) { + logger.info("database exist!"); + logger.info(dbname); + } + try { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), + dbname, username, password); + connection = DriverManager.getConnection(connUrl); + logger.info("change database success,the database is: ", dbname); + logger.info(dbname); + } catch (SQLException e) { + logger.info("change database error", e); + } + + } + + private Exception insertRowRecords(RowDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = connection.createStatement(); + for (int i = 0; i < data.getTimeSize(); i++) { + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getPathNum(); j++) { + if (bitmapView.get(j)) { + String path = data.getPath(j); + DataType dataType = data.getDataType(j); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + String table = database_table.substring(database_table.lastIndexOf('.') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = null; + try { + tags = data.getTags(i); + } catch (Exception e) { + logger.info("tagList[i] is null!"); + } + if (tags == null) { + tags = new HashMap<>(); + } + + //connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + + long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + String value; + if (data.getDataType(j) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + + StringBuilder columnsKeys = new StringBuilder(); + StringBuilder columnValues = new StringBuilder(); + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } + columnsKeys.append(field); + columnValues.append(value); + + stmt.addBatch(String + .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, + columnsKeys, time, columnValues)); + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.executeBatch(); + } + + index++; + } + } + } + stmt.executeBatch(); + } catch (SQLException e) { + return e; + } + + return null; + } + + private Exception insertColumnRecords(ColumnDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = connection.createStatement(); + String columnsKeys = ""; + String columnValues = ""; + String table = ""; + long time = 0; + for (int i = 0; i < data.getPathNum(); i++) { + String path = data.getPath(i); + logger.info("insert path=====" + path); + DataType dataType = data.getDataType(i); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + table = database_table.substring(database_table.lastIndexOf('.') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = null; + try { + tags = data.getTags(i); + } catch (Exception e) { + logger.info("tagList[i] is null!"); + } + if (tags == null) { + tags = new HashMap<>(); + } + + // connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getTimeSize(); j++) { + if (bitmapView.get(j)) { + time = data.getKey(j) / 1000; // 时间戳 + String value; + if (data.getDataType(i) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + columnsKeys = columnsKeys + "," + field; + columnValues = columnValues + "," + value; + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues)); + columnsKeys = ""; + columnValues = ""; + } + index++; + } + } + } + String s = String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues); + stmt.execute(s); + } catch (SQLException e) { + return e; + } + + return null; + } + + private TaskExecuteResult executeDeleteTask(Delete delete) { + try { + for (int i = 0; i < delete.getPatterns().size(); i++) { + String path = delete.getPatterns().get(i); + String[] path_l = path.split("\\."); + if (path_l.length < 3) { + if (path.equals("*.*")) { + } else { + path = "postgres." + path; + } + } + TimeRange timeRange = delete.getTimeRanges().get(i); + String db_ta = path.substring(0, path.lastIndexOf('.')); + String database = db_ta.substring(0, db_ta.lastIndexOf('.')); + String table = db_ta.substring(db_ta.lastIndexOf(',') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + // 查询序列类型 + useDatabase(database); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = null; + if (table.equals("*")) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else { + tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"table"}); + } + while (tableSet.next()) { + String tableName = tableSet.getString(3); + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); + if (columnSet.next()) { + String statement = String + .format(DELETE_DATA, tableName, + timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); + Statement stmt = connection.createStatement(); + stmt.execute(statement); + } + } + } + return new TaskExecuteResult(null, null); + } catch (SQLException e) { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", + e)); + } + } + + @Override + public void release() throws PhysicalException { + try { + connection.close(); + } catch (SQLException e) { + throw new PhysicalException(e); + } + } } \ No newline at end of file diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index e2e115071e..3b57845a60 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -6,112 +6,111 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - import cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgreSQLQueryRowStream implements RowStream { - - private final List resultSets; - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - - private final long[] currTimestamps; - - private final Object[] currValues; - - private final Header header; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; - public PostgreSQLQueryRowStream(List resultSets, List fields) { - this.resultSets = resultSets; - this.header = new Header(Field.KEY, fields); - this.currTimestamps = new long[resultSets.size()]; - this.currValues = new Object[resultSets.size()]; +public class PostgreSQLQueryRowStream implements RowStream { + private final List resultSets; + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); + + private final long[] currTimestamps; + + private final Object[] currValues; + + private final Header header; + + public PostgreSQLQueryRowStream(List resultSets, List fields) { + this.resultSets = resultSets; + this.header = new Header(Field.KEY, fields); + this.currTimestamps = new long[resultSets.size()]; + this.currValues = new Object[resultSets.size()]; // this.values=new ArrayList<>(); - // 默认填充一下timestamp列表 - try { - long j=1; - for (int i = 0; i < this.currTimestamps.length; i++) { - j=1; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - }catch (Exception e){ - this.currTimestamps[i] = j++; - } - this.currValues[i] = resultSet.getObject(2); - } - } - j++; - } catch (SQLException e) { - e.printStackTrace(); - // pass - } - - } - - @Override - public Header getHeader() { - return this.header; - } - - @Override - public void close() { - try { - for (ResultSet resultSet : resultSets) { - resultSet.close(); - } - } catch (SQLException e) { - // pass - } - } - - @Override - public boolean hasNext() throws PhysicalException { - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE && currTimestamp!=0) { - return true; - } - } - return false; - } - - @Override - public Row next() throws PhysicalException { - try { - long timestamp = Long.MAX_VALUE; - Object[] values = new Object[this.resultSets.size()]; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); - } - } - long j=1; - for (int i = 0; i < this.currTimestamps.length; i++) { - if (this.currTimestamps[i] == timestamp) { - values[i] = this.currValues[i]; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - }catch (Exception e){ - logger.info("have no timestamp,set default timestamp!"); - } - this.currValues[i] = resultSet.getObject(2); - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; - } - } - } - return new Row(header, timestamp, values); - } catch (SQLException e) { - throw new RowFetchException(e); - } - } + // 默认填充一下timestamp列表 + try { + long j = 1; + for (int i = 0; i < this.currTimestamps.length; i++) { + j = 1; + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + } catch (Exception e) { + this.currTimestamps[i] = j++; + } + this.currValues[i] = resultSet.getObject(2); + } + } + j++; + } catch (SQLException e) { + e.printStackTrace(); + // pass + } + + } + + @Override + public Header getHeader() { + return this.header; + } + + @Override + public void close() { + try { + for (ResultSet resultSet : resultSets) { + resultSet.close(); + } + } catch (SQLException e) { + // pass + } + } + + @Override + public boolean hasNext() throws PhysicalException { + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE && currTimestamp != 0) { + return true; + } + } + return false; + } + + @Override + public Row next() throws PhysicalException { + try { + long timestamp = Long.MAX_VALUE; + Object[] values = new Object[this.resultSets.size()]; + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE) { + timestamp = Math.min(timestamp, currTimestamp); + } + } + long j = 1; + for (int i = 0; i < this.currTimestamps.length; i++) { + if (this.currTimestamps[i] == timestamp) { + values[i] = this.currValues[i]; + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + } catch (Exception e) { + logger.info("have no timestamp,set default timestamp!"); + } + this.currValues[i] = resultSet.getObject(2); + } else { + // 值已经取完 + this.currTimestamps[i] = Long.MIN_VALUE; + this.currValues[i] = null; + } + } + } + return new Row(header, timestamp, values); + } catch (SQLException e) { + throw new RowFetchException(e); + } + } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java index a25cfe2d82..3aa9678d81 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java @@ -1,41 +1,38 @@ package cn.edu.tsinghua.iginx.postgresql.tools; -import static cn.edu.tsinghua.iginx.thrift.DataType.BINARY; -import static cn.edu.tsinghua.iginx.thrift.DataType.BOOLEAN; -import static cn.edu.tsinghua.iginx.thrift.DataType.DOUBLE; -import static cn.edu.tsinghua.iginx.thrift.DataType.LONG; - import cn.edu.tsinghua.iginx.thrift.DataType; -public class DataTypeTransformer { - - public static DataType fromPostgreSQL(String dataType) { - if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { - return LONG; - } else if (dataType.contains("bool")) { - return BOOLEAN; - } else if (dataType.contains("float")) { - return DOUBLE; - } else { - return BINARY; - } - } +import static cn.edu.tsinghua.iginx.thrift.DataType.*; - public static String toPostgreSQL(DataType dataType) { - switch (dataType){ - case BOOLEAN: - return "BOOLEAN"; - case INTEGER: - return "INTEGER"; - case LONG: - return "BIGINT"; - case FLOAT: - return "REAL"; - case DOUBLE: - return "DOUBLE PRECISION"; - case BINARY: - default: - return "TEXT"; - } - } +public class DataTypeTransformer { + + public static DataType fromPostgreSQL(String dataType) { + if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { + return LONG; + } else if (dataType.contains("bool")) { + return BOOLEAN; + } else if (dataType.contains("float")) { + return DOUBLE; + } else { + return BINARY; + } + } + + public static String toPostgreSQL(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return "BOOLEAN"; + case INTEGER: + return "INTEGER"; + case LONG: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE PRECISION"; + case BINARY: + default: + return "TEXT"; + } + } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index 01ef288677..ce098aecde 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -18,59 +18,53 @@ */ package cn.edu.tsinghua.iginx.postgresql.tools; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.NotFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.OrFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ValueFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; + import java.util.stream.Collectors; public class FilterTransformer { - - public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - public static String toString(Filter filter) { - if (filter == null) { - return ""; - } - switch (filter.getType()) { - case And: - return toString((AndFilter) filter); - case Or: - return toString((OrFilter) filter); - case Not: - return toString((NotFilter) filter); - case Value: - return toString((ValueFilter) filter); - case Key: - return toString((KeyFilter) filter); - default: - return ""; - } - } - - private static String toString(AndFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); - } - - private static String toString(NotFilter filter) { - return "not " + filter.toString(); - } - - private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; - } - - private static String toString(ValueFilter filter) { - return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); - } - - private static String toString(OrFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); - } - - - + + public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + + public static String toString(Filter filter) { + if (filter == null) { + return ""; + } + switch (filter.getType()) { + case And: + return toString((AndFilter) filter); + case Or: + return toString((OrFilter) filter); + case Not: + return toString((NotFilter) filter); + case Value: + return toString((ValueFilter) filter); + case Key: + return toString((KeyFilter) filter); + default: + return ""; + } + } + + private static String toString(AndFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); + } + + private static String toString(NotFilter filter) { + return "not " + filter.toString(); + } + + private static String toString(KeyFilter filter) { + return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; + } + + private static String toString(ValueFilter filter) { + return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); + } + + private static String toString(OrFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); + } + + } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index f9f6df9f5b..6f1ad526f9 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -6,49 +6,49 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; public class TagFilterUtils { - @SuppressWarnings("unused") - public static String transformToFilterStr(TagFilter filter) { - StringBuilder builder = new StringBuilder(); - transformToFilterStr(filter, builder); - return builder.toString(); - } - - private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { - switch (filter.getType()) { - case And: - AndTagFilter andFilter = (AndTagFilter) filter; - for (int i = 0; i < andFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(andFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" and "); - } - } - break; - case Or: - OrTagFilter orFilter = (OrTagFilter) filter; - for (int i = 0; i < orFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(orFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" or "); - } - } - break; - case Base: - BaseTagFilter baseFilter = (BaseTagFilter) filter; - builder.append(baseFilter.getTagKey()); - builder.append("="); - builder.append(baseFilter.getTagValue()); - break; - // TODO: case label - case BasePrecise: - break; - case Precise: - case WithoutTag: - break; - } - } + @SuppressWarnings("unused") + public static String transformToFilterStr(TagFilter filter) { + StringBuilder builder = new StringBuilder(); + transformToFilterStr(filter, builder); + return builder.toString(); + } + + private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { + switch (filter.getType()) { + case And: + AndTagFilter andFilter = (AndTagFilter) filter; + for (int i = 0; i < andFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(andFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" and "); + } + } + break; + case Or: + OrTagFilter orFilter = (OrTagFilter) filter; + for (int i = 0; i < orFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(orFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" or "); + } + } + break; + case Base: + BaseTagFilter baseFilter = (BaseTagFilter) filter; + builder.append(baseFilter.getTagKey()); + builder.append("="); + builder.append(baseFilter.getTagValue()); + break; + // TODO: case label + case BasePrecise: + break; + case Precise: + case WithoutTag: + break; + } + } } \ No newline at end of file From 564897d06c7abc9208ad58650cff40b4a294d272 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 1 Mar 2023 10:57:51 +0800 Subject: [PATCH 13/94] fix code style --- conf/config.properties | 66 ++---------------------------------------- 1 file changed, 3 insertions(+), 63 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 7e12032f40..0ae2fd9c76 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -3,16 +3,12 @@ #################### # iginx 绑定的 ip ip=0.0.0.0 - # iginx 绑定的端口 port=6888 - # iginx 本身的用户名 username=root - # iginx 本身的密码 password=root - # 时序数据库列表，使用','分隔不同实例 storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false @@ -20,142 +16,98 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456 #storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData - # 写入的副本个数 replicaNum=0 - # 底层数据库类名 -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage - # 内存任务执行线程池 memoryTaskThreadPoolSize=200 - # 每个存储节点对应的工作线程数 physicalTaskThreadPoolSizePerStorage=100 - # 每个存储节点任务最大堆积数 maxCachedPhysicalTaskPerStorage=500 - # 逻辑层优化策略 queryOptimizer=remove_not,filter_fragment - # 约束 constraintChecker=naive - # 物理层优化策略 physicalOptimizer=naive - # 分片策略 policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy #policyClassName=cn.edu.tsinghua.iginx.policy.simple.SimplePolicy - #统计信息收集类 # statisticsCollectorClassName=cn.edu.tsinghua.iginx.statistics.StatisticsCollector - # 统计信息打印间隔，单位毫秒 # statisticsLogInterval=5000 - # 重分片时，新分片的结束时间多加的间距，单位为秒 reshardFragmentTimeMargin=60 - # parquet是否为本地存储 isLocalParquetStorage=true - # thrift线程池最小线程数量 -minThriftWorkerThreadNum = 20 - +minThriftWorkerThreadNum=20 # thrift线程池最大线程数量 -maxThriftWrokerThreadNum = 2147483647; - +maxThriftWrokerThreadNum=2147483647; #################### ### Migration 相关配置 #################### - # 迁移时，每次迁移数据行数 migrationBatchSize=100 - # 按序列维度切分分片时，每次最多分成多少片 maxReshardFragmentsNum=3 - # 按序列维度切分分片时，时间序列负载高于平均值的最大差值倍数 maxTimeseriesLoadBalanceThreshold=2 - # 迁移策略类名 #migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.SimulationBasedMigrationPolicy migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.GreedyMigrationPolicy - #################### ### 元数据配置 #################### - # 目前支持 zookeeper, etcd # 文件是默认的存储后端，方便部署 metaStorage=zookeeper - # 如果使用 zookeeper 作为元数据存储后端，需要提供 zookeeperConnectionString=127.0.0.1:2181 - # 如果使用 etcd 作为元数据存储后端，需要提供，如果有多个 etcd 实例，以逗号分隔 #etcdEndpoints=http://localhost:2379 - # 是否开启元数据内存管理 enable_meta_cache_control=false - # 分片缓存最大内存限制，单位为 KB，默认 128 MB fragment_cache_threshold=131072 - ########################## ### 执行层配置 ########################## - enablePushDown=false - useStreamExecutor=false - ########################## ### 内存控制 ########################## - enable_memory_control=false - system_resource_metrics=default - heap_memory_threshold=0.9 - system_memory_threshold=0.9 - system_cpu_threshold=0.9 - #################### ### REST 服务配置 #################### - # rest 绑定的 ip restIp=0.0.0.0 - # rest 绑定的端口 restPort=6666 - # 是否启用 rest 服务 enableRestService=true - # 乱序数据 margin, 单位是秒 disorderMargin=10 - # rest 异步执行并发数 asyncRestThreadPool=100 - ########################## ### Python配置 ########################## # python脚本启动命令，建议使用"which python"查询出的绝对路径，如下所示 #pythonCMD=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 pythonCMD=python3 - # 是否初始化配置文件内指定的UDF/Transform needInitBasicUDFFunctions=false - ########################## ### Transform配置 ########################## @@ -165,32 +117,20 @@ batchSize=50 transformTaskThreadPoolSize=10 # Transform最大重试次数 transformMaxRetryTimes=3 - #################### ### MQTT 配置 #################### - enable_mqtt=false - mqtt_host=0.0.0.0 - mqtt_port=1883 - mqtt_handler_pool_size=1 - mqtt_payload_formatter=cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter - mqtt_max_message_size=1048576 - ########################## ### SimplePolicy 策略配置 ########################## - reAllocatePeriod=300000 - enableStorageGroupValueLimit=true - storageGroupValueLimit=200.0 - # 是否允许通过环境变量设置参数 enableEnvParameter=false From 63a83472b38897e204f5a08d174b81e6f5915e6d Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 1 Mar 2023 11:15:18 +0800 Subject: [PATCH 14/94] code style --- conf/config.properties | 108 +- .../iginx/postgresql/PostgreSQLStorage.java | 1134 ++++++++--------- .../entity/PostgreSQLQueryRowStream.java | 176 +-- .../postgresql/tools/DataTypeTransformer.java | 56 +- .../postgresql/tools/FilterTransformer.java | 84 +- .../postgresql/tools/TagFilterUtils.java | 86 +- 6 files changed, 821 insertions(+), 823 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 0ae2fd9c76..65b77975e7 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -1,86 +1,84 @@ #################### -### 基本配置 +### ???? #################### -# iginx 绑定的 ip +# iginx ??? ip ip=0.0.0.0 -# iginx 绑定的端口 +# iginx ????? port=6888 -# iginx 本身的用户名 +# iginx ?????? username=root -# iginx 本身的密码 +# iginx ????? password=root -# 时序数据库列表，使用','分隔不同实例 +# ??????????','?????? storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +#127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456 #storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData -# 写入的副本个数 +# ??????? replicaNum=0 -# 底层数据库类名 +# ??????? databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage -#,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage -# 内存任务执行线程池 +#,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage +#,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage +# ????????? memoryTaskThreadPoolSize=200 -# 每个存储节点对应的工作线程数 +# ?????????????? physicalTaskThreadPoolSizePerStorage=100 -# 每个存储节点任务最大堆积数 +# ????????????? maxCachedPhysicalTaskPerStorage=500 -# 逻辑层优化策略 +# ??????? queryOptimizer=remove_not,filter_fragment -# 约束 +# ?? constraintChecker=naive -# 物理层优化策略 +# ??????? physicalOptimizer=naive -# 分片策略 +# ???? policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy #policyClassName=cn.edu.tsinghua.iginx.policy.simple.SimplePolicy -#统计信息收集类 +#??????? # statisticsCollectorClassName=cn.edu.tsinghua.iginx.statistics.StatisticsCollector -# 统计信息打印间隔，单位毫秒 +# ????????????? # statisticsLogInterval=5000 -# 重分片时，新分片的结束时间多加的间距，单位为秒 +# ??????????????????????? reshardFragmentTimeMargin=60 -# parquet是否为本地存储 +# parquet??????? isLocalParquetStorage=true -# thrift线程池最小线程数量 -minThriftWorkerThreadNum=20 -# thrift线程池最大线程数量 -maxThriftWrokerThreadNum=2147483647; #################### -### Migration 相关配置 +### Migration ???? #################### -# 迁移时，每次迁移数据行数 +# ???????????? migrationBatchSize=100 -# 按序列维度切分分片时，每次最多分成多少片 +# ???????????????????? maxReshardFragmentsNum=3 -# 按序列维度切分分片时，时间序列负载高于平均值的最大差值倍数 +# ????????????????????????????? maxTimeseriesLoadBalanceThreshold=2 -# 迁移策略类名 +# ?????? #migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.SimulationBasedMigrationPolicy migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.GreedyMigrationPolicy #################### -### 元数据配置 +### ????? #################### -# 目前支持 zookeeper, etcd -# 文件是默认的存储后端，方便部署 +# ???? zookeeper, etcd +# ??????????????? metaStorage=zookeeper -# 如果使用 zookeeper 作为元数据存储后端，需要提供 +# ???? zookeeper ?????????????? zookeeperConnectionString=127.0.0.1:2181 -# 如果使用 etcd 作为元数据存储后端，需要提供，如果有多个 etcd 实例，以逗号分隔 +# ???? etcd ???????????????????? etcd ???????? #etcdEndpoints=http://localhost:2379 -# 是否开启元数据内存管理 +# ??????????? enable_meta_cache_control=false -# 分片缓存最大内存限制，单位为 KB，默认 128 MB +# ?????????????? KB??? 128 MB fragment_cache_threshold=131072 ########################## -### 执行层配置 +### ????? ########################## enablePushDown=false useStreamExecutor=false ########################## -### 内存控制 +### ???? ########################## enable_memory_control=false system_resource_metrics=default @@ -88,37 +86,37 @@ heap_memory_threshold=0.9 system_memory_threshold=0.9 system_cpu_threshold=0.9 #################### -### REST 服务配置 +### REST ???? #################### -# rest 绑定的 ip +# rest ??? ip restIp=0.0.0.0 -# rest 绑定的端口 +# rest ????? restPort=6666 -# 是否启用 rest 服务 +# ???? rest ?? enableRestService=true -# 乱序数据 margin, 单位是秒 +# ???? margin, ???? disorderMargin=10 -# rest 异步执行并发数 +# rest ??????? asyncRestThreadPool=100 ########################## -### Python配置 +### Python?? ########################## -# python脚本启动命令，建议使用"which python"查询出的绝对路径，如下所示 +# python???????????"which python"????????????? #pythonCMD=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 pythonCMD=python3 -# 是否初始化配置文件内指定的UDF/Transform +# ?????????????UDF/Transform needInitBasicUDFFunctions=false ########################## -### Transform配置 +### Transform?? ########################## -# 流式执行时，每批数据的大小 +# ????????????? batchSize=50 -# Transform任务执行线程池 +# Transform??????? transformTaskThreadPoolSize=10 -# Transform最大重试次数 +# Transform?????? transformMaxRetryTimes=3 #################### -### MQTT 配置 +### MQTT ?? #################### enable_mqtt=false mqtt_host=0.0.0.0 @@ -127,10 +125,10 @@ mqtt_handler_pool_size=1 mqtt_payload_formatter=cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter mqtt_max_message_size=1048576 ########################## -### SimplePolicy 策略配置 +### SimplePolicy ???? ########################## reAllocatePeriod=300000 enableStorageGroupValueLimit=true storageGroupValueLimit=200.0 -# 是否允许通过环境变量设置参数 -enableEnvParameter=false +# ?????????????? +enableEnvParameter=false \ No newline at end of file diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index aad4ec9cdd..24743343cb 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -53,612 +53,612 @@ import java.util.Map.Entry; public class PostgreSQLStorage implements IStorage { - - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - - private static final int BATCH_SIZE = 10000; - - private static final String STORAGE_ENGINE = "postgresql"; - - private static final String USERNAME = "username"; - - private static final String PASSWORD = "password"; - - private static final String DBNAME = "dbname"; - - private static final String DEFAULT_USERNAME = "postgres"; - - private static final String DEFAULT_PASSWORD = "postgres"; - - private static final String DEFAULT_DBNAME = "timeseries"; - - private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; - - private static final String FIRST_QUERY = "select first(%s, time) from %s"; - - private static final String LAST_QUERY = "select last(%s, time) from %s"; - - private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; - - private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - - private static final String IGINX_SEPARATOR = "."; - - private static final String POSTGRESQL_SEPARATOR = "$"; - - private static final String DATABASE_PREFIX = "unit"; - - private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - private final StorageEngineMeta meta; - - private Connection connection; - - public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { - this.meta = meta; - if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - connection = DriverManager.getConnection(connUrl); - } catch (SQLException e) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - } - - private boolean testConnection() { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - Class.forName("org.postgresql.Driver"); - DriverManager.getConnection(connUrl); - return true; - } catch (SQLException | ClassNotFoundException e) { - return false; - } - } - - @Override - public TaskExecuteResult execute(StoragePhysicalTask task) { - List operators = task.getOperators(); - if (operators.size() != 1) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); + + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); + + private static final int BATCH_SIZE = 10000; + + private static final String STORAGE_ENGINE = "postgresql"; + + private static final String USERNAME = "username"; + + private static final String PASSWORD = "password"; + + private static final String DBNAME = "dbname"; + + private static final String DEFAULT_USERNAME = "postgres"; + + private static final String DEFAULT_PASSWORD = "postgres"; + + private static final String DEFAULT_DBNAME = "timeseries"; + + private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; + + private static final String FIRST_QUERY = "select first(%s, time) from %s"; + + private static final String LAST_QUERY = "select last(%s, time) from %s"; + + private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; + + private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; + + private static final String IGINX_SEPARATOR = "."; + + private static final String POSTGRESQL_SEPARATOR = "$"; + + private static final String DATABASE_PREFIX = "unit"; + + private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + + private final StorageEngineMeta meta; + + private Connection connection; + + public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { + this.meta = meta; + if (!testConnection()) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), + username, password); + try { + connection = DriverManager.getConnection(connUrl); + } catch (SQLException e) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } } - FragmentMeta fragment = task.getTargetFragment(); - Operator op = operators.get(0); - String storageUnit = task.getStorageUnit(); - // 先切换数据库 - //connection.close(); - try { - useDatabase(storageUnit); - } catch (Exception e) { - logger.info("pass"); + + private boolean testConnection() { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), + username, password); + try { + Class.forName("org.postgresql.Driver"); + DriverManager.getConnection(connUrl); + return true; + } catch (SQLException | ClassNotFoundException e) { + return false; + } } - if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 - Project project = (Project) op; - Filter filter; - if (operators.size() == 2) { - filter = ((Select) operators.get(1)).getFilter(); - } else { - filter = new AndFilter(Arrays - .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), - new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); - } - return executeProjectTask(project, filter); - } else if (op.getType() == OperatorType.Insert) { - Insert insert = (Insert) op; - return executeInsertTask(insert); - } else if (op.getType() == OperatorType.Delete) { - Delete delete = (Delete) op; - return executeDeleteTask(delete); + @Override + public TaskExecuteResult execute(StoragePhysicalTask task) { + List operators = task.getOperators(); + if (operators.size() != 1) { + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); + } + FragmentMeta fragment = task.getTargetFragment(); + Operator op = operators.get(0); + String storageUnit = task.getStorageUnit(); + // 先切换数据库 + //connection.close(); + try { + useDatabase(storageUnit); + } catch (Exception e) { + logger.info("pass"); + } + + if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 + Project project = (Project) op; + Filter filter; + if (operators.size() == 2) { + filter = ((Select) operators.get(1)).getFilter(); + } else { + filter = new AndFilter(Arrays + .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), + new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); + } + return executeProjectTask(project, filter); + } else if (op.getType() == OperatorType.Insert) { + Insert insert = (Insert) op; + return executeInsertTask(insert); + } else if (op.getType() == OperatorType.Delete) { + Delete delete = (Delete) op; + return executeDeleteTask(delete); + } + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); } - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - - @Override - public List getTimeSeries() throws PhysicalException { - List timeseries = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取数据库名称 + + @Override + public List getTimeSeries() throws PhysicalException { + List timeseries = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取数据库名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); + //connection.close(); + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); // DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - if (tableName.startsWith("unit")) { - tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); - } - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) - timeseries.add(new Timeseries( - databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); - } - } + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + if (tableName.startsWith("unit")) { + tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); + } + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) + timeseries.add(new Timeseries( + databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName))); + } + } // } - } - } catch (SQLException e) { - throw new RuntimeException(e); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return timeseries; } - return timeseries; - } - - @Override - public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { - long minTime = Long.MAX_VALUE, maxTime = 0; - List paths = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取表名称 + + @Override + public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { + long minTime = Long.MAX_VALUE, maxTime = 0; + List paths = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取表名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); - // 获取first - String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); - Statement firstQueryStmt = connection.createStatement(); - String execIfNoTimestamp_first = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); - String execIfNoTimestamp_last = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); - try { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); - } - } catch (Exception e) { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 - } - } - // 获取last - String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - Statement lastQueryStmt = connection.createStatement(); - try { - ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } catch (Exception e) { //没有时间戳执行该部分 - ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } + //connection.close(); + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); + // 获取first + String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); + Statement firstQueryStmt = connection.createStatement(); + String execIfNoTimestamp_first = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); + String execIfNoTimestamp_last = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); + try { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); + } + } catch (Exception e) { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 + } + } + // 获取last + String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); + Statement lastQueryStmt = connection.createStatement(); + try { + ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } + } catch (Exception e) { //没有时间戳执行该部分 + ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } + } + } + } +// } } - } + } catch (SQLException e) { + throw new PhysicalException(e); } -// } - } - } catch (SQLException e) { - throw new PhysicalException(e); - } - paths.sort(String::compareTo); - - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), - new TimeInterval(minTime, maxTime + 1)); - } - - private TaskExecuteResult executeProjectTask(Project project, - Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 - String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); - String[] filter_list = filter1.split("\\s"); - long ft = Long.parseLong(filter_list[6].substring(0, filter_list[6].indexOf(")"))); - while (ft > 9223372036854L) { - ft = ft / 10; + paths.sort(String::compareTo); + + return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), + new TimeInterval(minTime, maxTime + 1)); } - filter1 = String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", - filter_list[1], Long.parseLong(filter_list[2]), filter.getType().toString(), filter_list[5], ft); - try { - List resultSets = new ArrayList<>(); - List fields = new ArrayList<>(); - for (String path : project.getPatterns()) { - ArrayList allDatabase = new ArrayList<>(); - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - allDatabase.add(databaseSet.getString(1)); - } - ArrayList databases = new ArrayList<>(); - String[] path_l = path.split("\\."); - if (path_l.length < 3) { - if (path.equals("*.*")) { - } else { - path = "postgres." + path; - } + + private TaskExecuteResult executeProjectTask(Project project, + Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 + String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); + String[] filter_list = filter1.split("\\s"); + long ft = Long.parseLong(filter_list[6].substring(0, filter_list[6].indexOf(")"))); + while (ft > 9223372036854L) { + ft = ft / 10; } - String database_table = path.substring(0, path.lastIndexOf('.')); - String tableName = ""; - String field1 = ""; - boolean allTable = false; - - if (path.equals("*.*")) { - stmt = connection.createStatement(); - databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - databases.add(databaseSet.getString(1)); - } - allTable = true; - } else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { - allTable = true; - } else { + filter1 = String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", + filter_list[1], Long.parseLong(filter_list[2]), filter.getType().toString(), filter_list[5], ft); + try { + List resultSets = new ArrayList<>(); + List fields = new ArrayList<>(); + for (String path : project.getPatterns()) { + ArrayList allDatabase = new ArrayList<>(); + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + allDatabase.add(databaseSet.getString(1)); + } + ArrayList databases = new ArrayList<>(); + String[] path_l = path.split("\\."); + if (path_l.length < 3) { + if (path.equals("*.*")) { + } else { + path = "postgres." + path; + } + } + String database_table = path.substring(0, path.lastIndexOf('.')); + String tableName = ""; + String field1 = ""; + boolean allTable = false; + + if (path.equals("*.*")) { + stmt = connection.createStatement(); + databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + databases.add(databaseSet.getString(1)); + } + allTable = true; + } else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { + allTable = true; + } else { // String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - tableName = database_table.substring(database_table.lastIndexOf('.') + 1); - if (tableName.equals("*")) { - allTable = true; - } - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - databases.add(database); - field1 = path.substring(path.lastIndexOf('.') + 1); - field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - } - - for (int i = 0; i < databases.size(); i++) { - String database = (String) databases.get(i); - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = null; - if (allTable) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); - } - while (tableSet.next()) { - String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); - if (field1.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, table, null); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + tableName = database_table.substring(database_table.lastIndexOf('.') + 1); + if (tableName.equals("*")) { + allTable = true; + } + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + databases.add(database); + field1 = path.substring(path.lastIndexOf('.') + 1); + field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + } + + for (int i = 0; i < databases.size(); i++) { + String database = (String) databases.get(i); + useDatabase(database); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = null; + if (allTable) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else { + tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); + } + while (tableSet.next()) { + String table = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); + if (field1.equals("*")) { + columnSet = databaseMetaData.getColumns(null, null, table, null); + } + while (columnSet.next()) { + String field = columnSet.getString("COLUMN_NAME"); + if (field.equals("time")) { + continue; + } + String statement = ""; + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + ResultSet rs; + try { + statement = String + .format("SELECT time,%s FROM %s where %s", field, table, filter1); + stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } catch (Exception e) { + statement = String + .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); + stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } + resultSets.add(rs); + } + } + } } - while (columnSet.next()) { - String field = columnSet.getString("COLUMN_NAME"); - if (field.equals("time")) { - continue; - } - String statement = ""; - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - ResultSet rs; - try { - statement = String - .format("SELECT time,%s FROM %s where %s", field, table, filter1); - stmt = connection.createStatement(); - rs = stmt.executeQuery(statement); - } catch (Exception e) { - statement = String - .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); - stmt = connection.createStatement(); - rs = stmt.executeQuery(statement); - } - resultSets.add(rs); - } - } + RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.info("error: ", e); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", + e)); } - } - RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - logger.info("error: ", e); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", - e)); - } - } - - private TaskExecuteResult executeInsertTask(Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertRowRecords((RowDataView) dataView); - break; - case Column: - case NonAlignedColumn: - e = insertColumnRecords((ColumnDataView) dataView); - break; - } - if (e != null) { - return new TaskExecuteResult(null, - new PhysicalException("execute insert task in postgresql failure", e)); } - return new TaskExecuteResult(null, null); - } - - private void createTimeSeriesIfNotExists(String table, String field, - Map tags, DataType dataType) { - try { - if (tags == null) { - tags = new HashMap<>(); - } - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = connection.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, - stringBuilder.toString())); - } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); - } + + private TaskExecuteResult executeInsertTask(Insert insert) { + DataView dataView = insert.getData(); + Exception e = null; + switch (dataView.getRawDataType()) { + case Row: + case NonAlignedRow: + e = insertRowRecords((RowDataView) dataView); + break; + case Column: + case NonAlignedColumn: + e = insertColumnRecords((ColumnDataView) dataView); + break; } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toPostgreSQL(dataType))); + if (e != null) { + return new TaskExecuteResult(null, + new PhysicalException("execute insert task in postgresql failure", e)); } - } - } catch (SQLException e) { - logger.error("create timeseries error", e); - } - } - - private void useDatabase(String dbname) throws SQLException { - try { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("create database %s", dbname)); - } catch (SQLException e) { - logger.info("database exist!"); - logger.info(dbname); - } - try { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), - dbname, username, password); - connection = DriverManager.getConnection(connUrl); - logger.info("change database success,the database is: ", dbname); - logger.info(dbname); - } catch (SQLException e) { - logger.info("change database error", e); + return new TaskExecuteResult(null, null); } - } - - private Exception insertRowRecords(RowDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - for (int i = 0; i < data.getTimeSize(); i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - if (bitmapView.get(j)) { - String path = data.getPath(j); - DataType dataType = data.getDataType(j); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - String table = database_table.substring(database_table.lastIndexOf('.') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = null; - try { - tags = data.getTags(i); - } catch (Exception e) { - logger.info("tagList[i] is null!"); - } + private void createTimeSeriesIfNotExists(String table, String field, + Map tags, DataType dataType) { + try { if (tags == null) { - tags = new HashMap<>(); + tags = new HashMap<>(); } - - //connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - - long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (!tableSet.next()) { + Statement stmt = connection.createStatement(); + StringBuilder stringBuilder = new StringBuilder(); + for (Entry tagsEntry : tags.entrySet()) { + stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + } + stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); + stmt.execute(String + .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, + stringBuilder.toString())); } else { - value = data.getValue(i, index).toString(); - } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, time, columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + } + } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, + DataTypeTransformer.toPostgreSQL(dataType))); + } } - - index++; - } + } catch (SQLException e) { + logger.error("create timeseries error", e); } - } - stmt.executeBatch(); - } catch (SQLException e) { - return e; } - return null; - } - - private Exception insertColumnRecords(ColumnDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - String columnsKeys = ""; - String columnValues = ""; - String table = ""; - long time = 0; - for (int i = 0; i < data.getPathNum(); i++) { - String path = data.getPath(i); - logger.info("insert path=====" + path); - DataType dataType = data.getDataType(i); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - table = database_table.substring(database_table.lastIndexOf('.') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = null; + private void useDatabase(String dbname) throws SQLException { try { - tags = data.getTags(i); - } catch (Exception e) { - logger.info("tagList[i] is null!"); + Statement stmt = connection.createStatement(); + stmt.execute(String.format("create database %s", dbname)); + } catch (SQLException e) { + logger.info("database exist!"); + logger.info(dbname); } - if (tags == null) { - tags = new HashMap<>(); + try { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), + dbname, username, password); + connection = DriverManager.getConnection(connUrl); + logger.info("change database success,the database is: ", dbname); + logger.info(dbname); + } catch (SQLException e) { + logger.info("change database error", e); } - // connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - time = data.getKey(j) / 1000; // 时间戳 - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - columnsKeys = columnsKeys + "," + field; - columnValues = columnValues + "," + value; - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues)); - columnsKeys = ""; - columnValues = ""; + } + + private Exception insertRowRecords(RowDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = connection.createStatement(); + for (int i = 0; i < data.getTimeSize(); i++) { + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getPathNum(); j++) { + if (bitmapView.get(j)) { + String path = data.getPath(j); + DataType dataType = data.getDataType(j); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + String table = database_table.substring(database_table.lastIndexOf('.') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = null; + try { + tags = data.getTags(i); + } catch (Exception e) { + logger.info("tagList[i] is null!"); + } + if (tags == null) { + tags = new HashMap<>(); + } + + //connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + + long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + String value; + if (data.getDataType(j) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + + StringBuilder columnsKeys = new StringBuilder(); + StringBuilder columnValues = new StringBuilder(); + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } + columnsKeys.append(field); + columnValues.append(value); + + stmt.addBatch(String + .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, + columnsKeys, time, columnValues)); + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.executeBatch(); + } + + index++; + } + } } - index++; - } + stmt.executeBatch(); + } catch (SQLException e) { + return e; } - } - String s = String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues); - stmt.execute(s); - } catch (SQLException e) { - return e; + + return null; } - return null; - } - - private TaskExecuteResult executeDeleteTask(Delete delete) { - try { - for (int i = 0; i < delete.getPatterns().size(); i++) { - String path = delete.getPatterns().get(i); - String[] path_l = path.split("\\."); - if (path_l.length < 3) { - if (path.equals("*.*")) { - } else { - path = "postgres." + path; - } - } - TimeRange timeRange = delete.getTimeRanges().get(i); - String db_ta = path.substring(0, path.lastIndexOf('.')); - String database = db_ta.substring(0, db_ta.lastIndexOf('.')); - String table = db_ta.substring(db_ta.lastIndexOf(',') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - // 查询序列类型 - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = null; - if (table.equals("*")) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"table"}); - } - while (tableSet.next()) { - String tableName = tableSet.getString(3); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); - if (columnSet.next()) { - String statement = String - .format(DELETE_DATA, tableName, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); + private Exception insertColumnRecords(ColumnDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { Statement stmt = connection.createStatement(); - stmt.execute(statement); - } + String columnsKeys = ""; + String columnValues = ""; + String table = ""; + long time = 0; + for (int i = 0; i < data.getPathNum(); i++) { + String path = data.getPath(i); + logger.info("insert path=====" + path); + DataType dataType = data.getDataType(i); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + table = database_table.substring(database_table.lastIndexOf('.') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = null; + try { + tags = data.getTags(i); + } catch (Exception e) { + logger.info("tagList[i] is null!"); + } + if (tags == null) { + tags = new HashMap<>(); + } + + // connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getTimeSize(); j++) { + if (bitmapView.get(j)) { + time = data.getKey(j) / 1000; // 时间戳 + String value; + if (data.getDataType(i) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + columnsKeys = columnsKeys + "," + field; + columnValues = columnValues + "," + value; + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues)); + columnsKeys = ""; + columnValues = ""; + } + index++; + } + } + } + String s = String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues); + stmt.execute(s); + } catch (SQLException e) { + return e; + } + + return null; + } + + private TaskExecuteResult executeDeleteTask(Delete delete) { + try { + for (int i = 0; i < delete.getPatterns().size(); i++) { + String path = delete.getPatterns().get(i); + String[] path_l = path.split("\\."); + if (path_l.length < 3) { + if (path.equals("*.*")) { + } else { + path = "postgres." + path; + } + } + TimeRange timeRange = delete.getTimeRanges().get(i); + String db_ta = path.substring(0, path.lastIndexOf('.')); + String database = db_ta.substring(0, db_ta.lastIndexOf('.')); + String table = db_ta.substring(db_ta.lastIndexOf(',') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + // 查询序列类型 + useDatabase(database); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = null; + if (table.equals("*")) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else { + tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"table"}); + } + while (tableSet.next()) { + String tableName = tableSet.getString(3); + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); + if (columnSet.next()) { + String statement = String + .format(DELETE_DATA, tableName, + timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); + Statement stmt = connection.createStatement(); + stmt.execute(statement); + } + } + } + return new TaskExecuteResult(null, null); + } catch (SQLException e) { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", + e)); } - } - return new TaskExecuteResult(null, null); - } catch (SQLException e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", - e)); } - } - - @Override - public void release() throws PhysicalException { - try { - connection.close(); - } catch (SQLException e) { - throw new PhysicalException(e); + + @Override + public void release() throws PhysicalException { + try { + connection.close(); + } catch (SQLException e) { + throw new PhysicalException(e); + } } - } } \ No newline at end of file diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 3b57845a60..9628473fde 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -15,102 +15,102 @@ import java.util.List; public class PostgreSQLQueryRowStream implements RowStream { - private final List resultSets; - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - - private final long[] currTimestamps; - - private final Object[] currValues; - - private final Header header; - - public PostgreSQLQueryRowStream(List resultSets, List fields) { - this.resultSets = resultSets; - this.header = new Header(Field.KEY, fields); - this.currTimestamps = new long[resultSets.size()]; - this.currValues = new Object[resultSets.size()]; + private final List resultSets; + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); + + private final long[] currTimestamps; + + private final Object[] currValues; + + private final Header header; + + public PostgreSQLQueryRowStream(List resultSets, List fields) { + this.resultSets = resultSets; + this.header = new Header(Field.KEY, fields); + this.currTimestamps = new long[resultSets.size()]; + this.currValues = new Object[resultSets.size()]; // this.values=new ArrayList<>(); - // 默认填充一下timestamp列表 - try { - long j = 1; - for (int i = 0; i < this.currTimestamps.length; i++) { - j = 1; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - } catch (Exception e) { - this.currTimestamps[i] = j++; - } - this.currValues[i] = resultSet.getObject(2); + // 默认填充一下timestamp列表 + try { + long j = 1; + for (int i = 0; i < this.currTimestamps.length; i++) { + j = 1; + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + } catch (Exception e) { + this.currTimestamps[i] = j++; + } + this.currValues[i] = resultSet.getObject(2); + } + } + j++; + } catch (SQLException e) { + e.printStackTrace(); + // pass } - } - j++; - } catch (SQLException e) { - e.printStackTrace(); - // pass + } - } - - @Override - public Header getHeader() { - return this.header; - } - - @Override - public void close() { - try { - for (ResultSet resultSet : resultSets) { - resultSet.close(); - } - } catch (SQLException e) { - // pass + @Override + public Header getHeader() { + return this.header; } - } - - @Override - public boolean hasNext() throws PhysicalException { - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE && currTimestamp != 0) { - return true; - } + + @Override + public void close() { + try { + for (ResultSet resultSet : resultSets) { + resultSet.close(); + } + } catch (SQLException e) { + // pass + } } - return false; - } - - @Override - public Row next() throws PhysicalException { - try { - long timestamp = Long.MAX_VALUE; - Object[] values = new Object[this.resultSets.size()]; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); + + @Override + public boolean hasNext() throws PhysicalException { + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE && currTimestamp != 0) { + return true; + } } - } - long j = 1; - for (int i = 0; i < this.currTimestamps.length; i++) { - if (this.currTimestamps[i] == timestamp) { - values[i] = this.currValues[i]; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - } catch (Exception e) { - logger.info("have no timestamp,set default timestamp!"); + return false; + } + + @Override + public Row next() throws PhysicalException { + try { + long timestamp = Long.MAX_VALUE; + Object[] values = new Object[this.resultSets.size()]; + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE) { + timestamp = Math.min(timestamp, currTimestamp); + } + } + long j = 1; + for (int i = 0; i < this.currTimestamps.length; i++) { + if (this.currTimestamps[i] == timestamp) { + values[i] = this.currValues[i]; + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + } catch (Exception e) { + logger.info("have no timestamp,set default timestamp!"); + } + this.currValues[i] = resultSet.getObject(2); + } else { + // 值已经取完 + this.currTimestamps[i] = Long.MIN_VALUE; + this.currValues[i] = null; + } + } } - this.currValues[i] = resultSet.getObject(2); - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; - } + return new Row(header, timestamp, values); + } catch (SQLException e) { + throw new RowFetchException(e); } - } - return new Row(header, timestamp, values); - } catch (SQLException e) { - throw new RowFetchException(e); } - } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java index 3aa9678d81..a301d5846e 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java @@ -5,34 +5,34 @@ import static cn.edu.tsinghua.iginx.thrift.DataType.*; public class DataTypeTransformer { - - public static DataType fromPostgreSQL(String dataType) { - if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { - return LONG; - } else if (dataType.contains("bool")) { - return BOOLEAN; - } else if (dataType.contains("float")) { - return DOUBLE; - } else { - return BINARY; + + public static DataType fromPostgreSQL(String dataType) { + if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { + return LONG; + } else if (dataType.contains("bool")) { + return BOOLEAN; + } else if (dataType.contains("float")) { + return DOUBLE; + } else { + return BINARY; + } } - } - - public static String toPostgreSQL(DataType dataType) { - switch (dataType) { - case BOOLEAN: - return "BOOLEAN"; - case INTEGER: - return "INTEGER"; - case LONG: - return "BIGINT"; - case FLOAT: - return "REAL"; - case DOUBLE: - return "DOUBLE PRECISION"; - case BINARY: - default: - return "TEXT"; + + public static String toPostgreSQL(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return "BOOLEAN"; + case INTEGER: + return "INTEGER"; + case LONG: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE PRECISION"; + case BINARY: + default: + return "TEXT"; + } } - } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index ce098aecde..291c788323 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -23,48 +23,48 @@ import java.util.stream.Collectors; public class FilterTransformer { - - public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - public static String toString(Filter filter) { - if (filter == null) { - return ""; + + public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + + public static String toString(Filter filter) { + if (filter == null) { + return ""; + } + switch (filter.getType()) { + case And: + return toString((AndFilter) filter); + case Or: + return toString((OrFilter) filter); + case Not: + return toString((NotFilter) filter); + case Value: + return toString((ValueFilter) filter); + case Key: + return toString((KeyFilter) filter); + default: + return ""; + } } - switch (filter.getType()) { - case And: - return toString((AndFilter) filter); - case Or: - return toString((OrFilter) filter); - case Not: - return toString((NotFilter) filter); - case Value: - return toString((ValueFilter) filter); - case Key: - return toString((KeyFilter) filter); - default: - return ""; + + private static String toString(AndFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); } - } - - private static String toString(AndFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); - } - - private static String toString(NotFilter filter) { - return "not " + filter.toString(); - } - - private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; - } - - private static String toString(ValueFilter filter) { - return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); - } - - private static String toString(OrFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); - } - - + + private static String toString(NotFilter filter) { + return "not " + filter.toString(); + } + + private static String toString(KeyFilter filter) { + return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; + } + + private static String toString(ValueFilter filter) { + return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); + } + + private static String toString(OrFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); + } + + } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index 6f1ad526f9..d0541497e5 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -6,49 +6,49 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; public class TagFilterUtils { - @SuppressWarnings("unused") - public static String transformToFilterStr(TagFilter filter) { - StringBuilder builder = new StringBuilder(); - transformToFilterStr(filter, builder); - return builder.toString(); - } - - private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { - switch (filter.getType()) { - case And: - AndTagFilter andFilter = (AndTagFilter) filter; - for (int i = 0; i < andFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(andFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" and "); - } - } - break; - case Or: - OrTagFilter orFilter = (OrTagFilter) filter; - for (int i = 0; i < orFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(orFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" or "); - } + @SuppressWarnings("unused") + public static String transformToFilterStr(TagFilter filter) { + StringBuilder builder = new StringBuilder(); + transformToFilterStr(filter, builder); + return builder.toString(); + } + + private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { + switch (filter.getType()) { + case And: + AndTagFilter andFilter = (AndTagFilter) filter; + for (int i = 0; i < andFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(andFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" and "); + } + } + break; + case Or: + OrTagFilter orFilter = (OrTagFilter) filter; + for (int i = 0; i < orFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(orFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" or "); + } + } + break; + case Base: + BaseTagFilter baseFilter = (BaseTagFilter) filter; + builder.append(baseFilter.getTagKey()); + builder.append("="); + builder.append(baseFilter.getTagValue()); + break; + // TODO: case label + case BasePrecise: + break; + case Precise: + case WithoutTag: + break; } - break; - case Base: - BaseTagFilter baseFilter = (BaseTagFilter) filter; - builder.append(baseFilter.getTagKey()); - builder.append("="); - builder.append(baseFilter.getTagValue()); - break; - // TODO: case label - case BasePrecise: - break; - case Precise: - case WithoutTag: - break; } - } } \ No newline at end of file From b7a0c62ffe1cf49b66ab64c71a6602ad9ed9da8d Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 1 Mar 2023 11:30:20 +0800 Subject: [PATCH 15/94] code style --- conf/config.properties | 60 +- .../iginx/postgresql/PostgreSQLStorage.java | 1202 ++++++++--------- .../entity/PostgreSQLQueryRowStream.java | 194 +-- .../postgresql/tools/DataTypeTransformer.java | 60 +- .../postgresql/tools/FilterTransformer.java | 88 +- .../postgresql/tools/TagFilterUtils.java | 90 +- 6 files changed, 876 insertions(+), 818 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 65b77975e7..437fbcb1a0 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -3,12 +3,16 @@ #################### # iginx ??? ip ip=0.0.0.0 + # iginx ????? port=6888 + # iginx ?????? username=root + # iginx ????? password=root + # ??????????','?????? storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #127.0.0.1#5432#postgresql#username=postgres#password=postgres @@ -17,95 +21,137 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456 #storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData + # ??????? replicaNum=0 + # ??????? -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage #,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage + # ????????? memoryTaskThreadPoolSize=200 + # ?????????????? physicalTaskThreadPoolSizePerStorage=100 + # ????????????? maxCachedPhysicalTaskPerStorage=500 + # ??????? queryOptimizer=remove_not,filter_fragment + # ?? constraintChecker=naive + # ??????? physicalOptimizer=naive + # ???? policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy #policyClassName=cn.edu.tsinghua.iginx.policy.simple.SimplePolicy + #??????? # statisticsCollectorClassName=cn.edu.tsinghua.iginx.statistics.StatisticsCollector + # ????????????? # statisticsLogInterval=5000 + # ??????????????????????? reshardFragmentTimeMargin=60 + # parquet??????? isLocalParquetStorage=true + #################### ### Migration ???? #################### + # ???????????? migrationBatchSize=100 + # ???????????????????? maxReshardFragmentsNum=3 + # ????????????????????????????? maxTimeseriesLoadBalanceThreshold=2 + # ?????? #migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.SimulationBasedMigrationPolicy migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.GreedyMigrationPolicy + #################### ### ????? #################### + # ???? zookeeper, etcd # ??????????????? metaStorage=zookeeper + # ???? zookeeper ?????????????? zookeeperConnectionString=127.0.0.1:2181 + # ???? etcd ???????????????????? etcd ???????? #etcdEndpoints=http://localhost:2379 + # ??????????? enable_meta_cache_control=false + # ?????????????? KB??? 128 MB fragment_cache_threshold=131072 + ########################## ### ????? ########################## + enablePushDown=false + useStreamExecutor=false + ########################## ### ???? ########################## + enable_memory_control=false + system_resource_metrics=default + heap_memory_threshold=0.9 + system_memory_threshold=0.9 + system_cpu_threshold=0.9 + #################### ### REST ???? #################### + # rest ??? ip restIp=0.0.0.0 + # rest ????? restPort=6666 + # ???? rest ?? enableRestService=true + # ???? margin, ???? disorderMargin=10 + # rest ??????? asyncRestThreadPool=100 + ########################## ### Python?? ########################## # python???????????"which python"????????????? #pythonCMD=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 pythonCMD=python3 + # ?????????????UDF/Transform needInitBasicUDFFunctions=false + ########################## ### Transform?? ########################## @@ -115,20 +161,32 @@ batchSize=50 transformTaskThreadPoolSize=10 # Transform?????? transformMaxRetryTimes=3 + #################### ### MQTT ?? #################### + enable_mqtt=false + mqtt_host=0.0.0.0 + mqtt_port=1883 + mqtt_handler_pool_size=1 + mqtt_payload_formatter=cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter + mqtt_max_message_size=1048576 + ########################## ### SimplePolicy ???? ########################## + reAllocatePeriod=300000 + enableStorageGroupValueLimit=true + storageGroupValueLimit=200.0 + # ?????????????? enableEnvParameter=false \ No newline at end of file diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 24743343cb..38c7e7bbb8 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -53,612 +53,612 @@ import java.util.Map.Entry; public class PostgreSQLStorage implements IStorage { - - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - - private static final int BATCH_SIZE = 10000; - - private static final String STORAGE_ENGINE = "postgresql"; - - private static final String USERNAME = "username"; - - private static final String PASSWORD = "password"; - - private static final String DBNAME = "dbname"; - - private static final String DEFAULT_USERNAME = "postgres"; - - private static final String DEFAULT_PASSWORD = "postgres"; - - private static final String DEFAULT_DBNAME = "timeseries"; - - private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; - - private static final String FIRST_QUERY = "select first(%s, time) from %s"; - - private static final String LAST_QUERY = "select last(%s, time) from %s"; - - private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; - - private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - - private static final String IGINX_SEPARATOR = "."; - - private static final String POSTGRESQL_SEPARATOR = "$"; - - private static final String DATABASE_PREFIX = "unit"; - - private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - private final StorageEngineMeta meta; - - private Connection connection; - - public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { - this.meta = meta; - if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - connection = DriverManager.getConnection(connUrl); - } catch (SQLException e) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - } - - private boolean testConnection() { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - Class.forName("org.postgresql.Driver"); - DriverManager.getConnection(connUrl); - return true; - } catch (SQLException | ClassNotFoundException e) { - return false; - } - } - - @Override - public TaskExecuteResult execute(StoragePhysicalTask task) { - List operators = task.getOperators(); - if (operators.size() != 1) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - FragmentMeta fragment = task.getTargetFragment(); - Operator op = operators.get(0); - String storageUnit = task.getStorageUnit(); - // 先切换数据库 - //connection.close(); - try { - useDatabase(storageUnit); - } catch (Exception e) { - logger.info("pass"); - } - - if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 - Project project = (Project) op; - Filter filter; - if (operators.size() == 2) { - filter = ((Select) operators.get(1)).getFilter(); - } else { - filter = new AndFilter(Arrays - .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), - new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); - } - return executeProjectTask(project, filter); - } else if (op.getType() == OperatorType.Insert) { - Insert insert = (Insert) op; - return executeInsertTask(insert); - } else if (op.getType() == OperatorType.Delete) { - Delete delete = (Delete) op; - return executeDeleteTask(delete); - } - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - - @Override - public List getTimeSeries() throws PhysicalException { - List timeseries = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取数据库名称 + + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); + + private static final int BATCH_SIZE = 10000; + + private static final String STORAGE_ENGINE = "postgresql"; + + private static final String USERNAME = "username"; + + private static final String PASSWORD = "password"; + + private static final String DBNAME = "dbname"; + + private static final String DEFAULT_USERNAME = "postgres"; + + private static final String DEFAULT_PASSWORD = "postgres"; + + private static final String DEFAULT_DBNAME = "timeseries"; + + private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; + + private static final String FIRST_QUERY = "select first(%s, time) from %s"; + + private static final String LAST_QUERY = "select last(%s, time) from %s"; + + private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; + + private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; + + private static final String IGINX_SEPARATOR = "."; + + private static final String POSTGRESQL_SEPARATOR = "$"; + + private static final String DATABASE_PREFIX = "unit"; + + private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + + private final StorageEngineMeta meta; + + private Connection connection; + + public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { + this.meta = meta; + if (!testConnection()) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), + username, password); + try { + connection = DriverManager.getConnection(connUrl); + } catch (SQLException e) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } + } + + private boolean testConnection() { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), + username, password); + try { + Class.forName("org.postgresql.Driver"); + DriverManager.getConnection(connUrl); + return true; + } catch (SQLException | ClassNotFoundException e) { + return false; + } + } + + @Override + public TaskExecuteResult execute(StoragePhysicalTask task) { + List operators = task.getOperators(); + if (operators.size() != 1) { + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); + } + FragmentMeta fragment = task.getTargetFragment(); + Operator op = operators.get(0); + String storageUnit = task.getStorageUnit(); + // 先切换数据库 + //connection.close(); + try { + useDatabase(storageUnit); + } catch (Exception e) { + logger.info("pass"); + } + + if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 + Project project = (Project) op; + Filter filter; + if (operators.size() == 2) { + filter = ((Select) operators.get(1)).getFilter(); + } else { + filter = new AndFilter(Arrays + .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), + new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); + } + return executeProjectTask(project, filter); + } else if (op.getType() == OperatorType.Insert) { + Insert insert = (Insert) op; + return executeInsertTask(insert); + } else if (op.getType() == OperatorType.Delete) { + Delete delete = (Delete) op; + return executeDeleteTask(delete); + } + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); + } + + @Override + public List getTimeSeries() throws PhysicalException { + List timeseries = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取数据库名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); + //connection.close(); + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); // DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - if (tableName.startsWith("unit")) { - tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); - } - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) - timeseries.add(new Timeseries( - databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); - } - } + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + if (tableName.startsWith("unit")) { + tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); + } + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) + timeseries.add(new Timeseries( + databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName))); + } + } // } - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - return timeseries; - } - - @Override - public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { - long minTime = Long.MAX_VALUE, maxTime = 0; - List paths = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取表名称 + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return timeseries; + } + + @Override + public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { + long minTime = Long.MAX_VALUE, maxTime = 0; + List paths = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + String databaseName = databaseSet.getString(1);//获取表名称 // if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); - // 获取first - String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); - Statement firstQueryStmt = connection.createStatement(); - String execIfNoTimestamp_first = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); - String execIfNoTimestamp_last = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); - try { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); - } - } catch (Exception e) { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 - } - } - // 获取last - String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - Statement lastQueryStmt = connection.createStatement(); - try { - ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } catch (Exception e) { //没有时间戳执行该部分 - ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } - } - } + //connection.close(); + useDatabase(databaseName); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); + // 获取first + String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); + Statement firstQueryStmt = connection.createStatement(); + String execIfNoTimestamp_first = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); + String execIfNoTimestamp_last = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); + try { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); + } + } catch (Exception e) { + ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); + if (firstQuerySet.next()) { + long currMinTime = firstQuerySet.getLong(1); + minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 + } + } + // 获取last + String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); + Statement lastQueryStmt = connection.createStatement(); + try { + ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } + } catch (Exception e) { //没有时间戳执行该部分 + ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); + if (lastQuerySet.next()) { + long currMaxTime = lastQuerySet.getLong(1); + maxTime = Math.max(currMaxTime, maxTime); + } + } + } + } // } - } - } catch (SQLException e) { - throw new PhysicalException(e); - } - paths.sort(String::compareTo); - - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), - new TimeInterval(minTime, maxTime + 1)); - } - - private TaskExecuteResult executeProjectTask(Project project, - Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 - String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); - String[] filter_list = filter1.split("\\s"); - long ft = Long.parseLong(filter_list[6].substring(0, filter_list[6].indexOf(")"))); - while (ft > 9223372036854L) { - ft = ft / 10; - } - filter1 = String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", - filter_list[1], Long.parseLong(filter_list[2]), filter.getType().toString(), filter_list[5], ft); - try { - List resultSets = new ArrayList<>(); - List fields = new ArrayList<>(); - for (String path : project.getPatterns()) { - ArrayList allDatabase = new ArrayList<>(); - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - allDatabase.add(databaseSet.getString(1)); - } - ArrayList databases = new ArrayList<>(); - String[] path_l = path.split("\\."); - if (path_l.length < 3) { - if (path.equals("*.*")) { - } else { - path = "postgres." + path; - } - } - String database_table = path.substring(0, path.lastIndexOf('.')); - String tableName = ""; - String field1 = ""; - boolean allTable = false; - - if (path.equals("*.*")) { - stmt = connection.createStatement(); - databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - databases.add(databaseSet.getString(1)); - } - allTable = true; - } else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { - allTable = true; - } else { + } + } catch (SQLException e) { + throw new PhysicalException(e); + } + paths.sort(String::compareTo); + + return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), + new TimeInterval(minTime, maxTime + 1)); + } + + private TaskExecuteResult executeProjectTask(Project project, + Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 + String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); + String[] filter_list = filter1.split("\\s"); + long ft = Long.parseLong(filter_list[6].substring(0, filter_list[6].indexOf(")"))); + while (ft > 9223372036854L) { + ft = ft / 10; + } + filter1 = String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", + filter_list[1], Long.parseLong(filter_list[2]), filter.getType().toString(), filter_list[5], ft); + try { + List resultSets = new ArrayList<>(); + List fields = new ArrayList<>(); + for (String path : project.getPatterns()) { + ArrayList allDatabase = new ArrayList<>(); + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + allDatabase.add(databaseSet.getString(1)); + } + ArrayList databases = new ArrayList<>(); + String[] path_l = path.split("\\."); + if (path_l.length < 3) { + if (path.equals("*.*")) { + } else { + path = "postgres." + path; + } + } + String database_table = path.substring(0, path.lastIndexOf('.')); + String tableName = ""; + String field1 = ""; + boolean allTable = false; + + if (path.equals("*.*")) { + stmt = connection.createStatement(); + databaseSet = stmt.executeQuery(QUERY_DATABASES); + while (databaseSet.next()) { + databases.add(databaseSet.getString(1)); + } + allTable = true; + } else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { + allTable = true; + } else { // String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - tableName = database_table.substring(database_table.lastIndexOf('.') + 1); - if (tableName.equals("*")) { - allTable = true; - } - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - databases.add(database); - field1 = path.substring(path.lastIndexOf('.') + 1); - field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - } - - for (int i = 0; i < databases.size(); i++) { - String database = (String) databases.get(i); - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = null; - if (allTable) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); - } - while (tableSet.next()) { - String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); - if (field1.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, table, null); - } - while (columnSet.next()) { - String field = columnSet.getString("COLUMN_NAME"); - if (field.equals("time")) { - continue; - } - String statement = ""; - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - ResultSet rs; - try { - statement = String - .format("SELECT time,%s FROM %s where %s", field, table, filter1); - stmt = connection.createStatement(); - rs = stmt.executeQuery(statement); - } catch (Exception e) { - statement = String - .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); - stmt = connection.createStatement(); - rs = stmt.executeQuery(statement); - } - resultSets.add(rs); - } - } - } - } - RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - logger.info("error: ", e); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", - e)); - } - } - - private TaskExecuteResult executeInsertTask(Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertRowRecords((RowDataView) dataView); - break; - case Column: - case NonAlignedColumn: - e = insertColumnRecords((ColumnDataView) dataView); - break; - } - if (e != null) { - return new TaskExecuteResult(null, - new PhysicalException("execute insert task in postgresql failure", e)); - } - return new TaskExecuteResult(null, null); - } - - private void createTimeSeriesIfNotExists(String table, String field, - Map tags, DataType dataType) { - try { - if (tags == null) { - tags = new HashMap<>(); - } - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = connection.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, - stringBuilder.toString())); - } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); - } - } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toPostgreSQL(dataType))); - } - } - } catch (SQLException e) { - logger.error("create timeseries error", e); - } - } - - private void useDatabase(String dbname) throws SQLException { - try { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("create database %s", dbname)); - } catch (SQLException e) { - logger.info("database exist!"); - logger.info(dbname); - } - try { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), - dbname, username, password); - connection = DriverManager.getConnection(connUrl); - logger.info("change database success,the database is: ", dbname); - logger.info(dbname); - } catch (SQLException e) { - logger.info("change database error", e); - } - - } - - private Exception insertRowRecords(RowDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - for (int i = 0; i < data.getTimeSize(); i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - if (bitmapView.get(j)) { - String path = data.getPath(j); - DataType dataType = data.getDataType(j); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - String table = database_table.substring(database_table.lastIndexOf('.') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = null; - try { - tags = data.getTags(i); - } catch (Exception e) { - logger.info("tagList[i] is null!"); - } - if (tags == null) { - tags = new HashMap<>(); - } - - //connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - - long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, time, columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); - } - - index++; - } - } - } - stmt.executeBatch(); - } catch (SQLException e) { - return e; - } - - return null; - } - - private Exception insertColumnRecords(ColumnDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - String columnsKeys = ""; - String columnValues = ""; - String table = ""; - long time = 0; - for (int i = 0; i < data.getPathNum(); i++) { - String path = data.getPath(i); - logger.info("insert path=====" + path); - DataType dataType = data.getDataType(i); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - table = database_table.substring(database_table.lastIndexOf('.') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = null; - try { - tags = data.getTags(i); - } catch (Exception e) { - logger.info("tagList[i] is null!"); - } - if (tags == null) { - tags = new HashMap<>(); - } - - // connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - time = data.getKey(j) / 1000; // 时间戳 - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - columnsKeys = columnsKeys + "," + field; - columnValues = columnValues + "," + value; - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues)); - columnsKeys = ""; - columnValues = ""; - } - index++; - } - } - } - String s = String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues); - stmt.execute(s); - } catch (SQLException e) { - return e; - } - - return null; - } - - private TaskExecuteResult executeDeleteTask(Delete delete) { - try { - for (int i = 0; i < delete.getPatterns().size(); i++) { - String path = delete.getPatterns().get(i); - String[] path_l = path.split("\\."); - if (path_l.length < 3) { - if (path.equals("*.*")) { - } else { - path = "postgres." + path; - } - } - TimeRange timeRange = delete.getTimeRanges().get(i); - String db_ta = path.substring(0, path.lastIndexOf('.')); - String database = db_ta.substring(0, db_ta.lastIndexOf('.')); - String table = db_ta.substring(db_ta.lastIndexOf(',') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - // 查询序列类型 - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = null; - if (table.equals("*")) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"table"}); - } - while (tableSet.next()) { - String tableName = tableSet.getString(3); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); - if (columnSet.next()) { - String statement = String - .format(DELETE_DATA, tableName, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = connection.createStatement(); - stmt.execute(statement); - } - } - } - return new TaskExecuteResult(null, null); - } catch (SQLException e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", - e)); - } - } - - @Override - public void release() throws PhysicalException { - try { - connection.close(); - } catch (SQLException e) { - throw new PhysicalException(e); - } - } + String database = database_table.substring(0, database_table.lastIndexOf('.')); + tableName = database_table.substring(database_table.lastIndexOf('.') + 1); + if (tableName.equals("*")) { + allTable = true; + } + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + databases.add(database); + field1 = path.substring(path.lastIndexOf('.') + 1); + field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + } + + for (int i = 0; i < databases.size(); i++) { + String database = (String) databases.get(i); + useDatabase(database); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = null; + if (allTable) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else { + tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); + } + while (tableSet.next()) { + String table = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); + if (field1.equals("*")) { + columnSet = databaseMetaData.getColumns(null, null, table, null); + } + while (columnSet.next()) { + String field = columnSet.getString("COLUMN_NAME"); + if (field.equals("time")) { + continue; + } + String statement = ""; + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + ResultSet rs; + try { + statement = String + .format("SELECT time,%s FROM %s where %s", field, table, filter1); + stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } catch (Exception e) { + statement = String + .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); + stmt = connection.createStatement(); + rs = stmt.executeQuery(statement); + } + resultSets.add(rs); + } + } + } + } + RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.info("error: ", e); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", + e)); + } + } + + private TaskExecuteResult executeInsertTask(Insert insert) { + DataView dataView = insert.getData(); + Exception e = null; + switch (dataView.getRawDataType()) { + case Row: + case NonAlignedRow: + e = insertRowRecords((RowDataView) dataView); + break; + case Column: + case NonAlignedColumn: + e = insertColumnRecords((ColumnDataView) dataView); + break; + } + if (e != null) { + return new TaskExecuteResult(null, + new PhysicalException("execute insert task in postgresql failure", e)); + } + return new TaskExecuteResult(null, null); + } + + private void createTimeSeriesIfNotExists(String table, String field, + Map tags, DataType dataType) { + try { + if (tags == null) { + tags = new HashMap<>(); + } + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (!tableSet.next()) { + Statement stmt = connection.createStatement(); + StringBuilder stringBuilder = new StringBuilder(); + for (Entry tagsEntry : tags.entrySet()) { + stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + } + stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); + stmt.execute(String + .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, + stringBuilder.toString())); + } else { + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + } + } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, + DataTypeTransformer.toPostgreSQL(dataType))); + } + } + } catch (SQLException e) { + logger.error("create timeseries error", e); + } + } + + private void useDatabase(String dbname) throws SQLException { + try { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("create database %s", dbname)); + } catch (SQLException e) { + logger.info("database exist!"); + logger.info(dbname); + } + try { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), + dbname, username, password); + connection = DriverManager.getConnection(connUrl); + logger.info("change database success,the database is: ", dbname); + logger.info(dbname); + } catch (SQLException e) { + logger.info("change database error", e); + } + + } + + private Exception insertRowRecords(RowDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = connection.createStatement(); + for (int i = 0; i < data.getTimeSize(); i++) { + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getPathNum(); j++) { + if (bitmapView.get(j)) { + String path = data.getPath(j); + DataType dataType = data.getDataType(j); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + String table = database_table.substring(database_table.lastIndexOf('.') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = null; + try { + tags = data.getTags(i); + } catch (Exception e) { + logger.info("tagList[i] is null!"); + } + if (tags == null) { + tags = new HashMap<>(); + } + + //connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + + long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + String value; + if (data.getDataType(j) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + + StringBuilder columnsKeys = new StringBuilder(); + StringBuilder columnValues = new StringBuilder(); + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } + columnsKeys.append(field); + columnValues.append(value); + + stmt.addBatch(String + .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, + columnsKeys, time, columnValues)); + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.executeBatch(); + } + + index++; + } + } + } + stmt.executeBatch(); + } catch (SQLException e) { + return e; + } + + return null; + } + + private Exception insertColumnRecords(ColumnDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = connection.createStatement(); + String columnsKeys = ""; + String columnValues = ""; + String table = ""; + long time = 0; + for (int i = 0; i < data.getPathNum(); i++) { + String path = data.getPath(i); + logger.info("insert path=====" + path); + DataType dataType = data.getDataType(i); + String database_table = path.substring(0, path.lastIndexOf('.')); + String database = database_table.substring(0, database_table.lastIndexOf('.')); + table = database_table.substring(database_table.lastIndexOf('.') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = null; + try { + tags = data.getTags(i); + } catch (Exception e) { + logger.info("tagList[i] is null!"); + } + if (tags == null) { + tags = new HashMap<>(); + } + + // connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getTimeSize(); j++) { + if (bitmapView.get(j)) { + time = data.getKey(j) / 1000; // 时间戳 + String value; + if (data.getDataType(i) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + columnsKeys = columnsKeys + "," + field; + columnValues = columnValues + "," + value; + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues)); + columnsKeys = ""; + columnValues = ""; + } + index++; + } + } + } + String s = String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues); + stmt.execute(s); + } catch (SQLException e) { + return e; + } + + return null; + } + + private TaskExecuteResult executeDeleteTask(Delete delete) { + try { + for (int i = 0; i < delete.getPatterns().size(); i++) { + String path = delete.getPatterns().get(i); + String[] path_l = path.split("\\."); + if (path_l.length < 3) { + if (path.equals("*.*")) { + } else { + path = "postgres." + path; + } + } + TimeRange timeRange = delete.getTimeRanges().get(i); + String db_ta = path.substring(0, path.lastIndexOf('.')); + String database = db_ta.substring(0, db_ta.lastIndexOf('.')); + String table = db_ta.substring(db_ta.lastIndexOf(',') + 1); + database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + // 查询序列类型 + useDatabase(database); + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = null; + if (table.equals("*")) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else { + tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"table"}); + } + while (tableSet.next()) { + String tableName = tableSet.getString(3); + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); + if (columnSet.next()) { + String statement = String + .format(DELETE_DATA, tableName, + timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); + Statement stmt = connection.createStatement(); + stmt.execute(statement); + } + } + } + return new TaskExecuteResult(null, null); + } catch (SQLException e) { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", + e)); + } + } + + @Override + public void release() throws PhysicalException { + try { + connection.close(); + } catch (SQLException e) { + throw new PhysicalException(e); + } + } } \ No newline at end of file diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 9628473fde..b111f4006c 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -15,102 +15,102 @@ import java.util.List; public class PostgreSQLQueryRowStream implements RowStream { - private final List resultSets; - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - - private final long[] currTimestamps; - - private final Object[] currValues; - - private final Header header; - - public PostgreSQLQueryRowStream(List resultSets, List fields) { - this.resultSets = resultSets; - this.header = new Header(Field.KEY, fields); - this.currTimestamps = new long[resultSets.size()]; - this.currValues = new Object[resultSets.size()]; + private final List resultSets; + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); + + private final long[] currTimestamps; + + private final Object[] currValues; + + private final Header header; + + public PostgreSQLQueryRowStream(List resultSets, List fields) { + this.resultSets = resultSets; + this.header = new Header(Field.KEY, fields); + this.currTimestamps = new long[resultSets.size()]; + this.currValues = new Object[resultSets.size()]; // this.values=new ArrayList<>(); - // 默认填充一下timestamp列表 - try { - long j = 1; - for (int i = 0; i < this.currTimestamps.length; i++) { - j = 1; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - } catch (Exception e) { - this.currTimestamps[i] = j++; - } - this.currValues[i] = resultSet.getObject(2); - } - } - j++; - } catch (SQLException e) { - e.printStackTrace(); - // pass - } - - } - - @Override - public Header getHeader() { - return this.header; - } - - @Override - public void close() { - try { - for (ResultSet resultSet : resultSets) { - resultSet.close(); - } - } catch (SQLException e) { - // pass - } - } - - @Override - public boolean hasNext() throws PhysicalException { - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE && currTimestamp != 0) { - return true; - } - } - return false; - } - - @Override - public Row next() throws PhysicalException { - try { - long timestamp = Long.MAX_VALUE; - Object[] values = new Object[this.resultSets.size()]; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); - } - } - long j = 1; - for (int i = 0; i < this.currTimestamps.length; i++) { - if (this.currTimestamps[i] == timestamp) { - values[i] = this.currValues[i]; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - } catch (Exception e) { - logger.info("have no timestamp,set default timestamp!"); - } - this.currValues[i] = resultSet.getObject(2); - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; - } - } - } - return new Row(header, timestamp, values); - } catch (SQLException e) { - throw new RowFetchException(e); - } - } + // 默认填充一下timestamp列表 + try { + long j = 1; + for (int i = 0; i < this.currTimestamps.length; i++) { + j = 1; + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + } catch (Exception e) { + this.currTimestamps[i] = j++; + } + this.currValues[i] = resultSet.getObject(2); + } + } + j++; + } catch (SQLException e) { + e.printStackTrace(); + // pass + } + + } + + @Override + public Header getHeader() { + return this.header; + } + + @Override + public void close() { + try { + for (ResultSet resultSet : resultSets) { + resultSet.close(); + } + } catch (SQLException e) { + // pass + } + } + + @Override + public boolean hasNext() throws PhysicalException { + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE && currTimestamp != 0) { + return true; + } + } + return false; + } + + @Override + public Row next() throws PhysicalException { + try { + long timestamp = Long.MAX_VALUE; + Object[] values = new Object[this.resultSets.size()]; + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE) { + timestamp = Math.min(timestamp, currTimestamp); + } + } + long j = 1; + for (int i = 0; i < this.currTimestamps.length; i++) { + if (this.currTimestamps[i] == timestamp) { + values[i] = this.currValues[i]; + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + try { + this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); + } catch (Exception e) { + logger.info("have no timestamp,set default timestamp!"); + } + this.currValues[i] = resultSet.getObject(2); + } else { + // 值已经取完 + this.currTimestamps[i] = Long.MIN_VALUE; + this.currValues[i] = null; + } + } + } + return new Row(header, timestamp, values); + } catch (SQLException e) { + throw new RowFetchException(e); + } + } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java index a301d5846e..ecd5fbed98 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java @@ -5,34 +5,34 @@ import static cn.edu.tsinghua.iginx.thrift.DataType.*; public class DataTypeTransformer { - - public static DataType fromPostgreSQL(String dataType) { - if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { - return LONG; - } else if (dataType.contains("bool")) { - return BOOLEAN; - } else if (dataType.contains("float")) { - return DOUBLE; - } else { - return BINARY; - } - } - - public static String toPostgreSQL(DataType dataType) { - switch (dataType) { - case BOOLEAN: - return "BOOLEAN"; - case INTEGER: - return "INTEGER"; - case LONG: - return "BIGINT"; - case FLOAT: - return "REAL"; - case DOUBLE: - return "DOUBLE PRECISION"; - case BINARY: - default: - return "TEXT"; - } - } + + public static DataType fromPostgreSQL(String dataType) { + if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { + return LONG; + } else if (dataType.contains("bool")) { + return BOOLEAN; + } else if (dataType.contains("float")) { + return DOUBLE; + } else { + return BINARY; + } + } + + public static String toPostgreSQL(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return "BOOLEAN"; + case INTEGER: + return "INTEGER"; + case LONG: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE PRECISION"; + case BINARY: + default: + return "TEXT"; + } + } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index 291c788323..77fcb511a6 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -23,48 +23,48 @@ import java.util.stream.Collectors; public class FilterTransformer { - - public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - public static String toString(Filter filter) { - if (filter == null) { - return ""; - } - switch (filter.getType()) { - case And: - return toString((AndFilter) filter); - case Or: - return toString((OrFilter) filter); - case Not: - return toString((NotFilter) filter); - case Value: - return toString((ValueFilter) filter); - case Key: - return toString((KeyFilter) filter); - default: - return ""; - } - } - - private static String toString(AndFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); - } - - private static String toString(NotFilter filter) { - return "not " + filter.toString(); - } - - private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; - } - - private static String toString(ValueFilter filter) { - return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); - } - - private static String toString(OrFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); - } - - + + public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + + public static String toString(Filter filter) { + if (filter == null) { + return ""; + } + switch (filter.getType()) { + case And: + return toString((AndFilter) filter); + case Or: + return toString((OrFilter) filter); + case Not: + return toString((NotFilter) filter); + case Value: + return toString((ValueFilter) filter); + case Key: + return toString((KeyFilter) filter); + default: + return ""; + } + } + + private static String toString(AndFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); + } + + private static String toString(NotFilter filter) { + return "not " + filter.toString(); + } + + private static String toString(KeyFilter filter) { + return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; + } + + private static String toString(ValueFilter filter) { + return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); + } + + private static String toString(OrFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); + } + + } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index d0541497e5..953f37aa21 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -6,49 +6,49 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; public class TagFilterUtils { - @SuppressWarnings("unused") - public static String transformToFilterStr(TagFilter filter) { - StringBuilder builder = new StringBuilder(); - transformToFilterStr(filter, builder); - return builder.toString(); - } - - private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { - switch (filter.getType()) { - case And: - AndTagFilter andFilter = (AndTagFilter) filter; - for (int i = 0; i < andFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(andFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" and "); - } - } - break; - case Or: - OrTagFilter orFilter = (OrTagFilter) filter; - for (int i = 0; i < orFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(orFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" or "); - } - } - break; - case Base: - BaseTagFilter baseFilter = (BaseTagFilter) filter; - builder.append(baseFilter.getTagKey()); - builder.append("="); - builder.append(baseFilter.getTagValue()); - break; - // TODO: case label - case BasePrecise: - break; - case Precise: - case WithoutTag: - break; - } - } + @SuppressWarnings("unused") + public static String transformToFilterStr(TagFilter filter) { + StringBuilder builder = new StringBuilder(); + transformToFilterStr(filter, builder); + return builder.toString(); + } + + private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { + switch (filter.getType()) { + case And: + AndTagFilter andFilter = (AndTagFilter) filter; + for (int i = 0; i < andFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(andFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" and "); + } + } + break; + case Or: + OrTagFilter orFilter = (OrTagFilter) filter; + for (int i = 0; i < orFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(orFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" or "); + } + } + break; + case Base: + BaseTagFilter baseFilter = (BaseTagFilter) filter; + builder.append(baseFilter.getTagKey()); + builder.append("="); + builder.append(baseFilter.getTagValue()); + break; + // TODO: case label + case BasePrecise: + break; + case Precise: + case WithoutTag: + break; + } + } } \ No newline at end of file From 6830194eacabbff48c1c6236ad4644b506870486 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 1 Mar 2023 11:38:39 +0800 Subject: [PATCH 16/94] code style --- conf/config.properties | 100 ++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 437fbcb1a0..dc17f89717 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -1,19 +1,19 @@ #################### -### ???? +### 基本配置 #################### -# iginx ??? ip +# iginx 绑定的 ip ip=0.0.0.0 -# iginx ????? +# iginx 绑定的端口 port=6888 -# iginx ?????? +# iginx 本身的用户名 username=root -# iginx ????? +# iginx 本身的密码 password=root -# ??????????','?????? +# 时序数据库列表，使用','分隔不同实例 storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false @@ -22,87 +22,87 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData -# ??????? +# 写入的副本个数 replicaNum=0 -# ??????? +# 底层数据库类名 databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage #,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage -# ????????? +# 内存任务执行线程池 memoryTaskThreadPoolSize=200 -# ?????????????? +# 每个存储节点对应的工作线程数 physicalTaskThreadPoolSizePerStorage=100 -# ????????????? +# 每个存储节点任务最大堆积数 maxCachedPhysicalTaskPerStorage=500 -# ??????? +# 逻辑层优化策略 queryOptimizer=remove_not,filter_fragment -# ?? +# 约束 constraintChecker=naive -# ??????? +# 物理层优化策略 physicalOptimizer=naive -# ???? +# 分片策略 policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy #policyClassName=cn.edu.tsinghua.iginx.policy.simple.SimplePolicy -#??????? +#统计信息收集类 # statisticsCollectorClassName=cn.edu.tsinghua.iginx.statistics.StatisticsCollector -# ????????????? +# 统计信息打印间隔，单位毫秒 # statisticsLogInterval=5000 -# ??????????????????????? +# 重分片时，新分片的结束时间多加的间距，单位为秒 reshardFragmentTimeMargin=60 -# parquet??????? +# parquet是否为本地存储 isLocalParquetStorage=true #################### -### Migration ???? +### Migration 相关配置 #################### -# ???????????? +# 迁移时，每次迁移数据行数 migrationBatchSize=100 -# ???????????????????? +# 按序列维度切分分片时，每次最多分成多少片 maxReshardFragmentsNum=3 -# ????????????????????????????? +# 按序列维度切分分片时，时间序列负载高于平均值的最大差值倍数 maxTimeseriesLoadBalanceThreshold=2 -# ?????? +# 迁移策略类名 #migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.SimulationBasedMigrationPolicy migrationPolicyClassName=cn.edu.tsinghua.iginx.migration.GreedyMigrationPolicy #################### -### ????? +### 元数据配置 #################### -# ???? zookeeper, etcd -# ??????????????? +# 目前支持 zookeeper, etcd +# 文件是默认的存储后端，方便部署 metaStorage=zookeeper -# ???? zookeeper ?????????????? +# 如果使用 zookeeper 作为元数据存储后端，需要提供 zookeeperConnectionString=127.0.0.1:2181 -# ???? etcd ???????????????????? etcd ???????? +# 如果使用 etcd 作为元数据存储后端，需要提供，如果有多个 etcd 实例，以逗号分隔 #etcdEndpoints=http://localhost:2379 -# ??????????? +# 是否开启元数据内存管理 enable_meta_cache_control=false -# ?????????????? KB??? 128 MB +# 分片缓存最大内存限制，单位为 KB，默认 128 MB fragment_cache_threshold=131072 ########################## -### ????? +### 执行层配置 ########################## enablePushDown=false @@ -110,7 +110,7 @@ enablePushDown=false useStreamExecutor=false ########################## -### ???? +### 内存控制 ########################## enable_memory_control=false @@ -124,46 +124,46 @@ system_memory_threshold=0.9 system_cpu_threshold=0.9 #################### -### REST ???? +### REST 服务配置 #################### -# rest ??? ip +# rest 绑定的 ip restIp=0.0.0.0 -# rest ????? +# rest 绑定的端口 restPort=6666 -# ???? rest ?? +# 是否启用 rest 服务 enableRestService=true -# ???? margin, ???? +# 乱序数据 margin, 单位是秒 disorderMargin=10 -# rest ??????? +# rest 异步执行并发数 asyncRestThreadPool=100 ########################## -### Python?? +### Python配置 ########################## -# python???????????"which python"????????????? +# python脚本启动命令，建议使用"which python"查询出的绝对路径，如下所示 #pythonCMD=/Library/Frameworks/Python.framework/Versions/3.7/bin/python3 pythonCMD=python3 -# ?????????????UDF/Transform +# 是否初始化配置文件内指定的UDF/Transform needInitBasicUDFFunctions=false ########################## -### Transform?? +### Transform配置 ########################## -# ????????????? +# 流式执行时，每批数据的大小 batchSize=50 -# Transform??????? +# Transform任务执行线程池 transformTaskThreadPoolSize=10 -# Transform?????? +# Transform最大重试次数 transformMaxRetryTimes=3 #################### -### MQTT ?? +### MQTT 配置 #################### enable_mqtt=false @@ -179,7 +179,7 @@ mqtt_payload_formatter=cn.edu.tsinghua.iginx.mqtt.JsonPayloadFormatter mqtt_max_message_size=1048576 ########################## -### SimplePolicy ???? +### SimplePolicy 策略配置 ########################## reAllocatePeriod=300000 @@ -188,5 +188,5 @@ enableStorageGroupValueLimit=true storageGroupValueLimit=200.0 -# ?????????????? -enableEnvParameter=false \ No newline at end of file +# 是否允许通过环境变量设置参数 +enableEnvParameter=false From bcda1c3616c685a5923ee1cbc54f1ee9df61f92e Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 1 Mar 2023 11:39:28 +0800 Subject: [PATCH 17/94] code style --- conf/config.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.properties b/conf/config.properties index dc17f89717..2b1068075a 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -26,7 +26,7 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool replicaNum=0 # 底层数据库类名 -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage #,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage From 8a13997c097598ed8ceebb9466d356e474546410 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Wed, 1 Mar 2023 16:39:07 +0800 Subject: [PATCH 18/94] Update config.properties --- conf/config.properties | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 2b1068075a..61eb55dab6 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -15,20 +15,18 @@ password=root # 时序数据库列表，使用','分隔不同实例 storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false -#127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 -#storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456 -#storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 +#storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres +#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 replicaNum=0 # 底层数据库类名 -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage -#,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage # 内存任务执行线程池 memoryTaskThreadPoolSize=200 From 1ecd48060ecf5c5a678c69e29c2b36ae0fffe139 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Thu, 2 Mar 2023 09:38:52 +0800 Subject: [PATCH 19/94] fix tags null and empty value --- .../iginx/postgresql/PostgreSQLStorage.java | 253 +++++++++--------- 1 file changed, 124 insertions(+), 129 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 38c7e7bbb8..cfc81035bc 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -53,47 +53,47 @@ import java.util.Map.Entry; public class PostgreSQLStorage implements IStorage { - + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - + private static final int BATCH_SIZE = 10000; - + private static final String STORAGE_ENGINE = "postgresql"; - + private static final String USERNAME = "username"; - + private static final String PASSWORD = "password"; - + private static final String DBNAME = "dbname"; - + private static final String DEFAULT_USERNAME = "postgres"; - + private static final String DEFAULT_PASSWORD = "postgres"; - + private static final String DEFAULT_DBNAME = "timeseries"; - + private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; - + private static final String FIRST_QUERY = "select first(%s, time) from %s"; - + private static final String LAST_QUERY = "select last(%s, time) from %s"; - + private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; - + private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - + private static final String IGINX_SEPARATOR = "."; - + private static final String POSTGRESQL_SEPARATOR = "$"; - + private static final String DATABASE_PREFIX = "unit"; - + private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - + private final StorageEngineMeta meta; - + private Connection connection; - + public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { this.meta = meta; if (!testConnection()) { @@ -111,7 +111,7 @@ public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationExc throw new StorageInitializationException("cannot connect to " + meta.toString()); } } - + private boolean testConnection() { Map extraParams = meta.getExtraParams(); String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); @@ -127,7 +127,7 @@ private boolean testConnection() { return false; } } - + @Override public TaskExecuteResult execute(StoragePhysicalTask task) { List operators = task.getOperators(); @@ -145,7 +145,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { } catch (Exception e) { logger.info("pass"); } - + if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 Project project = (Project) op; Filter filter; @@ -167,7 +167,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { return new TaskExecuteResult( new NonExecutablePhysicalTaskException("unsupported physical task")); } - + @Override public List getTimeSeries() throws PhysicalException { List timeseries = new ArrayList<>(); @@ -207,7 +207,7 @@ public List getTimeSeries() throws PhysicalException { } return timeseries; } - + @Override public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { long minTime = Long.MAX_VALUE, maxTime = 0; @@ -271,11 +271,11 @@ public Pair getBoundaryOfStorage(String prefix) t throw new PhysicalException(e); } paths.sort(String::compareTo); - + return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), new TimeInterval(minTime, maxTime + 1)); } - + private TaskExecuteResult executeProjectTask(Project project, Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); @@ -308,7 +308,7 @@ private TaskExecuteResult executeProjectTask(Project project, String tableName = ""; String field1 = ""; boolean allTable = false; - + if (path.equals("*.*")) { stmt = connection.createStatement(); databaseSet = stmt.executeQuery(QUERY_DATABASES); @@ -330,7 +330,7 @@ private TaskExecuteResult executeProjectTask(Project project, field1 = path.substring(path.lastIndexOf('.') + 1); field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); } - + for (int i = 0; i < databases.size(); i++) { String database = (String) databases.get(i); useDatabase(database); @@ -383,7 +383,7 @@ private TaskExecuteResult executeProjectTask(Project project, e)); } } - + private TaskExecuteResult executeInsertTask(Insert insert) { DataView dataView = insert.getData(); Exception e = null; @@ -403,45 +403,44 @@ private TaskExecuteResult executeInsertTask(Insert insert) { } return new TaskExecuteResult(null, null); } - + private void createTimeSeriesIfNotExists(String table, String field, Map tags, DataType dataType) { try { - if (tags == null) { - tags = new HashMap<>(); - } - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = connection.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, - stringBuilder.toString())); - } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); + if (tags != null && !tags.isEmpty()) { + DatabaseMetaData databaseMetaData = connection.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (!tableSet.next()) { + Statement stmt = connection.createStatement(); + StringBuilder stringBuilder = new StringBuilder(); + for (Entry tagsEntry : tags.entrySet()) { + stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + } + stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); + stmt.execute(String + .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, + stringBuilder.toString())); + } else { + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); + if (!columnSet.next()) { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + } + } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); if (!columnSet.next()) { Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, + DataTypeTransformer.toPostgreSQL(dataType))); } } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toPostgreSQL(dataType))); - } } } catch (SQLException e) { logger.error("create timeseries error", e); } } - + private void useDatabase(String dbname) throws SQLException { try { Statement stmt = connection.createStatement(); @@ -463,9 +462,9 @@ private void useDatabase(String dbname) throws SQLException { } catch (SQLException e) { logger.info("change database error", e); } - + } - + private Exception insertRowRecords(RowDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { @@ -489,41 +488,39 @@ private Exception insertRowRecords(RowDataView data) { } catch (Exception e) { logger.info("tagList[i] is null!"); } - if (tags == null) { - tags = new HashMap<>(); - } - - //connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - - long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, time, columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); + if (tags != null && !tags.isEmpty()) { + //connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + + long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + String value; + if (data.getDataType(j) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + + StringBuilder columnsKeys = new StringBuilder(); + StringBuilder columnValues = new StringBuilder(); + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } + columnsKeys.append(field); + columnValues.append(value); + + stmt.addBatch(String + .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, + columnsKeys, time, columnValues)); + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.executeBatch(); + } + + index++; } - - index++; } } } @@ -531,10 +528,10 @@ private Exception insertRowRecords(RowDataView data) { } catch (SQLException e) { return e; } - + return null; } - + private Exception insertColumnRecords(ColumnDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { @@ -559,37 +556,35 @@ private Exception insertColumnRecords(ColumnDataView data) { } catch (Exception e) { logger.info("tagList[i] is null!"); } - if (tags == null) { - tags = new HashMap<>(); - } - - // connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - time = data.getKey(j) / 1000; // 时间戳 - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - columnsKeys = columnsKeys + "," + field; - columnValues = columnValues + "," + value; - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues)); - columnsKeys = ""; - columnValues = ""; + if (tags != null && !tags.isEmpty()) { + // connection.close(); + useDatabase(database); + stmt = connection.createStatement(); + createTimeSeriesIfNotExists(table, field, tags, dataType); + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getTimeSize(); j++) { + if (bitmapView.get(j)) { + time = data.getKey(j) / 1000; // 时间戳 + String value; + if (data.getDataType(i) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + + "'"; + } else { + value = data.getValue(i, index).toString(); + } + columnsKeys = columnsKeys + "," + field; + columnValues = columnValues + "," + value; + if (index > 0 && (index + 1) % batchSize == 0) { + stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, + columnsKeys, + time, + columnValues)); + columnsKeys = ""; + columnValues = ""; + } + index++; } - index++; } } } @@ -601,10 +596,10 @@ private Exception insertColumnRecords(ColumnDataView data) { } catch (SQLException e) { return e; } - + return null; } - + private TaskExecuteResult executeDeleteTask(Delete delete) { try { for (int i = 0; i < delete.getPatterns().size(); i++) { @@ -652,7 +647,7 @@ private TaskExecuteResult executeDeleteTask(Delete delete) { e)); } } - + @Override public void release() throws PhysicalException { try { From 1ed47841636fd808551c564cf1b84c3def3ee86f Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 3 Mar 2023 11:03:04 +0800 Subject: [PATCH 20/94] Fix insert --- .../engine/shared/data/write/DataView.java | 4 + .../iginx/postgresql/PostgreSQLStorage.java | 300 +++++++++--------- .../session/PostgreSQLSessionExample.java | 48 ++- 3 files changed, 197 insertions(+), 155 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java index 152b8b8d91..f6fad2ebbb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java @@ -116,4 +116,8 @@ public Map getTags(int index) { return tagsList.get(startPathIndex + index); } + public boolean hasTagsList() { + return data.getTagsList() != null && data.getTagsList().size() != 0; + } + } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index cfc81035bc..643453d2a9 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -44,6 +44,7 @@ import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; +import org.postgresql.ds.PGConnectionPoolDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +52,7 @@ import java.sql.*; import java.util.*; import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; public class PostgreSQLStorage implements IStorage { @@ -92,6 +94,8 @@ public class PostgreSQLStorage implements IStorage { private final StorageEngineMeta meta; + private final Map connectionPoolMap = new ConcurrentHashMap<>(); + private Connection connection; public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { @@ -128,6 +132,21 @@ private boolean testConnection() { } } + private Connection getConnection(String dbname, String url) { + try { + if (connectionPoolMap.containsKey(dbname)) { + return connectionPoolMap.get(dbname).getConnection(); + } + PGConnectionPoolDataSource connectionPool = new PGConnectionPoolDataSource(); + connectionPool.setUrl(url); + connectionPoolMap.put(dbname, connectionPool); + return connectionPool.getConnection(); + } catch (SQLException e) { + logger.error("cannot get connection for database: {}", dbname); + return null; + } + } + @Override public TaskExecuteResult execute(StoragePhysicalTask task) { List operators = task.getOperators(); @@ -140,8 +159,9 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { String storageUnit = task.getStorageUnit(); // 先切换数据库 //connection.close(); + Connection conn = null; try { - useDatabase(storageUnit); + conn = useDatabase(storageUnit); } catch (Exception e) { logger.info("pass"); } @@ -156,13 +176,13 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); } - return executeProjectTask(project, filter); + return executeProjectTask(conn, project, filter); } else if (op.getType() == OperatorType.Insert) { Insert insert = (Insert) op; - return executeInsertTask(insert); + return executeInsertTask(conn, insert); } else if (op.getType() == OperatorType.Delete) { Delete delete = (Delete) op; - return executeDeleteTask(delete); + return executeDeleteTask(conn, delete); } return new TaskExecuteResult( new NonExecutablePhysicalTaskException("unsupported physical task")); @@ -276,7 +296,7 @@ public Pair getBoundaryOfStorage(String prefix) t new TimeInterval(minTime, maxTime + 1)); } - private TaskExecuteResult executeProjectTask(Project project, + private TaskExecuteResult executeProjectTask(Connection conn, Project project, Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); String[] filter_list = filter1.split("\\s"); @@ -291,7 +311,7 @@ private TaskExecuteResult executeProjectTask(Project project, List fields = new ArrayList<>(); for (String path : project.getPatterns()) { ArrayList allDatabase = new ArrayList<>(); - Statement stmt = connection.createStatement(); + Statement stmt = conn.createStatement(); ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); while (databaseSet.next()) { allDatabase.add(databaseSet.getString(1)); @@ -310,7 +330,7 @@ private TaskExecuteResult executeProjectTask(Project project, boolean allTable = false; if (path.equals("*.*")) { - stmt = connection.createStatement(); + stmt = conn.createStatement(); databaseSet = stmt.executeQuery(QUERY_DATABASES); while (databaseSet.next()) { databases.add(databaseSet.getString(1)); @@ -334,7 +354,7 @@ private TaskExecuteResult executeProjectTask(Project project, for (int i = 0; i < databases.size(); i++) { String database = (String) databases.get(i); useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); + DatabaseMetaData databaseMetaData = conn.getMetaData(); ResultSet tableSet = null; if (allTable) { tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); @@ -361,12 +381,12 @@ private TaskExecuteResult executeProjectTask(Project project, try { statement = String .format("SELECT time,%s FROM %s where %s", field, table, filter1); - stmt = connection.createStatement(); + stmt = conn.createStatement(); rs = stmt.executeQuery(statement); } catch (Exception e) { statement = String .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); - stmt = connection.createStatement(); + stmt = conn.createStatement(); rs = stmt.executeQuery(statement); } resultSets.add(rs); @@ -384,17 +404,17 @@ private TaskExecuteResult executeProjectTask(Project project, } } - private TaskExecuteResult executeInsertTask(Insert insert) { + private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { DataView dataView = insert.getData(); Exception e = null; switch (dataView.getRawDataType()) { case Row: case NonAlignedRow: - e = insertRowRecords((RowDataView) dataView); + e = insertRowRecords(conn, (RowDataView) dataView); break; case Column: case NonAlignedColumn: - e = insertColumnRecords((ColumnDataView) dataView); + e = insertColumnRecords(conn, (ColumnDataView) dataView); break; } if (e != null) { @@ -404,71 +424,65 @@ private TaskExecuteResult executeInsertTask(Insert insert) { return new TaskExecuteResult(null, null); } - private void createTimeSeriesIfNotExists(String table, String field, - Map tags, DataType dataType) { + private void createTimeSeriesIfNotExists(Connection conn, String table, String field, Map tags, DataType dataType) { try { - if (tags != null && !tags.isEmpty()) { - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = connection.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, - stringBuilder.toString())); - } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); - } - } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (!tableSet.next()) { + Statement stmt = conn.createStatement(); + StringBuilder stringBuilder = new StringBuilder(); + for (Entry tagsEntry : tags.entrySet()) { + stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + } + stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); + stmt.execute(String + .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, + stringBuilder.toString())); + } else { + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toPostgreSQL(dataType))); + Statement stmt = conn.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); } } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (!columnSet.next()) { + Statement stmt = conn.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, + DataTypeTransformer.toPostgreSQL(dataType))); + } } } catch (SQLException e) { logger.error("create timeseries error", e); } } - private void useDatabase(String dbname) throws SQLException { + private Connection useDatabase(String dbname) throws SQLException { try { + // TODO 为什么不能直接判断数据库是否存在，而是通过创建数据库来捕获异常？ Statement stmt = connection.createStatement(); stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { - logger.info("database exist!"); - logger.info(dbname); - } - try { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), - dbname, username, password); - connection = DriverManager.getConnection(connUrl); - logger.info("change database success,the database is: ", dbname); - logger.info(dbname); - } catch (SQLException e) { - logger.info("change database error", e); + logger.info("database {} exist!", dbname); } + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), + dbname, username, password); + Connection conn = getConnection(dbname, connUrl); + logger.info("change database success, the current database is: {}", dbname); + return conn; } - private Exception insertRowRecords(RowDataView data) { + private Exception insertRowRecords(Connection conn, RowDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { - Statement stmt = connection.createStatement(); + Statement stmt = conn.createStatement(); + int cnt = 0; for (int i = 0; i < data.getTimeSize(); i++) { BitmapView bitmapView = data.getBitmapView(i); int index = 0; @@ -476,123 +490,105 @@ private Exception insertRowRecords(RowDataView data) { if (bitmapView.get(j)) { String path = data.getPath(j); DataType dataType = data.getDataType(j); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - String table = database_table.substring(database_table.lastIndexOf('.') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = null; - try { + String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = new HashMap<>(); + if (data.hasTagsList()) { tags = data.getTags(i); - } catch (Exception e) { - logger.info("tagList[i] is null!"); } - if (tags != null && !tags.isEmpty()) { - //connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - - long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } + // TODO connection为什么从来没close过？ + // TODO 为什么这么频繁地切换database？ + // TODO database ?= storage unit + createTimeSeriesIfNotExists(conn, table, field, tags, dataType); + + // TODO timescaledb？ + long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + String value; + if (data.getDataType(j) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + } else { + value = data.getValue(i, index).toString(); + } - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, time, columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); - } + StringBuilder columnsKeys = new StringBuilder(); + StringBuilder columnValues = new StringBuilder(); + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } + columnsKeys.append(field); + columnValues.append(value); - index++; + stmt.addBatch(String + .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, + columnsKeys, time, columnValues)); + + index++; + cnt++; + if (cnt % batchSize == 0) { + stmt.executeBatch(); } } } } stmt.executeBatch(); + conn.close(); } catch (SQLException e) { + logger.error(e.getMessage()); return e; } return null; } - private Exception insertColumnRecords(ColumnDataView data) { + private Exception insertColumnRecords(Connection conn, ColumnDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { - Statement stmt = connection.createStatement(); - String columnsKeys = ""; - String columnValues = ""; - String table = ""; - long time = 0; + Statement stmt = conn.createStatement(); + int cnt = 0; for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); - logger.info("insert path=====" + path); DataType dataType = data.getDataType(i); - String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - table = database_table.substring(database_table.lastIndexOf('.') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = null; - try { + String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Map tags = new HashMap<>(); + if (data.hasTagsList()) { tags = data.getTags(i); - } catch (Exception e) { - logger.info("tagList[i] is null!"); } - if (tags != null && !tags.isEmpty()) { - // connection.close(); - useDatabase(database); - stmt = connection.createStatement(); - createTimeSeriesIfNotExists(table, field, tags, dataType); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - time = data.getKey(j) / 1000; // 时间戳 - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - columnsKeys = columnsKeys + "," + field; - columnValues = columnValues + "," + value; - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.execute(String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues)); - columnsKeys = ""; - columnValues = ""; - } - index++; + + createTimeSeriesIfNotExists(conn, table, field, tags, dataType); + + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getTimeSize(); j++) { + if (bitmapView.get(j)) { + long time = data.getKey(j) / 1000; // 时间戳 + String value; + if (data.getDataType(i) == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + } else { + value = data.getValue(i, index).toString(); + } + + StringBuilder columnsKeys = new StringBuilder(); + StringBuilder columnValues = new StringBuilder(); + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } + columnsKeys.append(field); + columnValues.append(value); + + cnt++; + index++; + stmt.addBatch(String.format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, columnsKeys, time, columnValues)); + if (cnt % batchSize == 0) { + stmt.executeBatch(); } } } + stmt.executeBatch(); } - String s = String.format("INSERT INTO %s (time %s) values (to_timestamp(%d) %s)", table, - columnsKeys, - time, - columnValues); - stmt.execute(s); } catch (SQLException e) { return e; } @@ -600,7 +596,7 @@ private Exception insertColumnRecords(ColumnDataView data) { return null; } - private TaskExecuteResult executeDeleteTask(Delete delete) { + private TaskExecuteResult executeDeleteTask(Connection conn, Delete delete) { try { for (int i = 0; i < delete.getPatterns().size(); i++) { String path = delete.getPatterns().get(i); @@ -620,8 +616,8 @@ private TaskExecuteResult executeDeleteTask(Delete delete) { String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); // 查询序列类型 - useDatabase(database); - DatabaseMetaData databaseMetaData = connection.getMetaData(); +// useDatabase(database); + DatabaseMetaData databaseMetaData = conn.getMetaData(); ResultSet tableSet = null; if (table.equals("*")) { tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); @@ -635,7 +631,7 @@ private TaskExecuteResult executeDeleteTask(Delete delete) { String statement = String .format(DELETE_DATA, tableName, timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = connection.createStatement(); + Statement stmt = conn.createStatement(); stmt.execute(statement); } } diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java index 5b285dee97..afcb767599 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java @@ -21,6 +21,7 @@ import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.thrift.TimePrecision; import org.apache.commons.lang3.RandomStringUtils; import java.util.ArrayList; @@ -47,14 +48,55 @@ public static void main(String[] args) throws SessionException, ExecutionExcepti session = new Session("127.0.0.1", 6888, "root", "root"); // 打开 Session session.openSession(); -// 行式插入对齐数据 + + // 列式插入对齐数据 + insertColumnRecords(); + // 行式插入对齐数据 insertRowRecords(); - queryData(); - deleteDataInColumns(); +// queryData(); +// deleteDataInColumns(); + // 关闭 Session session.closeSession(); } + private static void insertColumnRecords() throws SessionException, ExecutionException { + List paths = new ArrayList<>(); + paths.add(S1); + paths.add(S2); + paths.add(S3); + paths.add(S4); + + int size = (int) (COLUMN_END_TIMESTAMP - COLUMN_START_TIMESTAMP + 1); + long[] timestamps = new long[size]; + for (long i = 0; i < size; i++) { + timestamps[(int) i] = i + COLUMN_START_TIMESTAMP; + } + + Object[] valuesList = new Object[4]; + for (long i = 0; i < 4; i++) { + Object[] values = new Object[size]; + for (long j = 0; j < size; j++) { + if (i < 2) { + values[(int) j] = i + j; + } else { + values[(int) j] = RandomStringUtils.randomAlphanumeric(10).getBytes(); + } + } + valuesList[(int) i] = values; + } + + List dataTypeList = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + dataTypeList.add(DataType.LONG); + } + for (int i = 0; i < 2; i++) { + dataTypeList.add(DataType.BINARY); + } + + session.insertColumnRecords(paths, timestamps, valuesList, dataTypeList, null, TimePrecision.NS); + } + private static void insertRowRecords() throws SessionException, ExecutionException { List paths = new ArrayList<>(); paths.add(S1); From d2cf51aab43f91791611a33c59f809a49cdf880c Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 8 Mar 2023 14:58:39 +0800 Subject: [PATCH 21/94] fix --- dataSources/dmdb/test.java | 319 +++++++++++ dataSources/postgresql/pom.xml | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 497 +++++++++--------- .../entity/PostgreSQLQueryRowStream.java | 53 +- .../postgresql/tools/FilterTransformer.java | 20 +- .../session/PostgreSQLSessionExample.java | 5 +- tools-viewPartitions/fragment.png | Bin 0 -> 11986 bytes 7 files changed, 622 insertions(+), 274 deletions(-) create mode 100644 dataSources/dmdb/test.java create mode 100644 tools-viewPartitions/fragment.png diff --git a/dataSources/dmdb/test.java b/dataSources/dmdb/test.java new file mode 100644 index 0000000000..1b9675a14f --- /dev/null +++ b/dataSources/dmdb/test.java @@ -0,0 +1,319 @@ +/*该例程实现插入数据，修改数据，删除数据，数据查询等基本操作。*/ + +import java.awt.Color; +import java.awt.Font; +import java.awt.Graphics2D; +import java.awt.font.FontRenderContext; +import java.awt.geom.Rectangle2D; +import java.awt.image.BufferedImage; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.Date; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import javax.imageio.ImageIO; + +public class test { + // 定义DM JDBC驱动串 + String jdbcString = "dm.jdbc.driver.DmDriver"; + // 定义DM URL连接串 + String urlString = "jdbc:dm://localhost:5236"; + // 定义连接用户名 + String userName = "SYSDBA"; + // 定义连接用户口令 + String password = "SYSDBA001"; + // 定义连接对象 + Connection conn = null; + + /* 加载JDBC驱动程序 + * @throws SQLException 异常 */ + public void loadJdbcDriver() throws SQLException { + try { + System.out.println("Loading JDBC Driver..."); + // 加载JDBC驱动程序 + Class.forName(jdbcString); + } catch (ClassNotFoundException e) { + throw new SQLException("Load JDBC Driver Error : " + e.getMessage()); + } catch (Exception ex) { + throw new SQLException("Load JDBC Driver Error : " + + ex.getMessage()); + } + } + + /* 连接DM数据库 + * @throws SQLException 异常 */ + public void connect() throws SQLException { + try { + System.out.println("Connecting to DM Server..."); + // 连接DM数据库 + conn = DriverManager.getConnection(urlString, userName, password); + } catch (SQLException e) { + throw new SQLException("Connect to DM Server Error : " + + e.getMessage()); + } + } + + /* 关闭连接 + * @throws SQLException 异常 */ + public void disConnect() throws SQLException { + try { + // 关闭连接 + conn.close(); + } catch (SQLException e) { + throw new SQLException("close connection error : " + e.getMessage()); + } + } + + /* 往产品信息表插入数据 + * @throws SQLException 异常 */ + public void insertTable() throws SQLException { + // 插入数据语句 + String sql = "INSERT INTO production.product(name,author,publisher,publishtime," + + "product_subcategoryid,productno,satetystocklevel,originalprice,nowprice,discount," + + "description,photo,type,papertotal,wordtotal,sellstarttime,sellendtime) " + + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);"; + // 创建语句对象 + PreparedStatement pstmt = conn.prepareStatement(sql); + // 为参数赋值 + pstmt.setString(1, "三国演义"); + pstmt.setString(2, "罗贯中"); + pstmt.setString(3, "中华书局"); + pstmt.setDate(4, Date.valueOf("2005-04-01")); + pstmt.setInt(5, 4); + pstmt.setString(6, "9787101046121"); + pstmt.setInt(7, 10); + pstmt.setBigDecimal(8, new BigDecimal(19.0000)); + pstmt.setBigDecimal(9, new BigDecimal(15.2000)); + pstmt.setBigDecimal(10, new BigDecimal(8.0)); + pstmt.setString(11, "《三国演义》是中国第一部长篇章回体小说，中国小说由短篇发展至长篇的原因与说书有关。"); + // 设置大字段参数 + try { + // 创建一个图片用于插入大字段 + String filePath = "c:\\三国演义.jpg"; + CreateImage(filePath); + File file = new File(filePath); + InputStream in = new BufferedInputStream(new FileInputStream(file)); + pstmt.setBinaryStream(12, in, (int) file.length()); + } catch (FileNotFoundException e) { + System.out.println(e.getMessage()); + // 如果没有图片设置为NULL + pstmt.setNull(12, java.sql.Types.BINARY); + } catch (IOException e) { + System.out.println(e.getMessage()); + } + pstmt.setString(13, "25"); + pstmt.setInt(14, 943); + pstmt.setInt(15, 93000); + pstmt.setDate(16, Date.valueOf("2006-03-20")); + pstmt.setDate(17, Date.valueOf("1900-01-01")); + // 执行语句 + pstmt.executeUpdate(); + // 关闭语句 + pstmt.close(); + } + + /* 查询产品信息表 + * @throws SQLException 异常 */ + public void queryProduct() throws SQLException { + // 查询语句 + String sql = "SELECT productid,name,author,description,photo FROM production.product WHERE productid=11"; + // 创建语句对象 + Statement stmt = conn.createStatement(); + // 执行查询 + ResultSet rs = stmt.executeQuery(sql); + // 显示结果集 + displayResultSet(rs); + // 关闭结果集 + rs.close(); + // 关闭语句 + stmt.close(); + } + + /* 修改产品信息表数据 + * @throws SQLException 异常 */ + public void updateTable() throws SQLException { + // 更新数据语句 + String sql = "UPDATE production.product SET name = ?" + + "WHERE productid = 11;"; + // 创建语句对象 + PreparedStatement pstmt = conn.prepareStatement(sql); + // 为参数赋值 + pstmt.setString(1, "三国演义（上）"); + // 执行语句 + pstmt.executeUpdate(); + // 关闭语句 + pstmt.close(); + } + + /* 删除产品信息表数据 + * @throws SQLException 异常 */ + public void deleteTable() throws SQLException { + // 删除数据语句 + String sql = "DELETE FROM production.product WHERE productid = 11;"; + // 创建语句对象 + Statement stmt = conn.createStatement(); + // 执行语句 + stmt.executeUpdate(sql); + // 关闭语句 + stmt.close(); + } + + /* 查询产品信息表 + * @throws SQLException 异常 */ + public void queryTable() throws SQLException { + // 查询语句 + String sql = "SELECT productid,name,author,publisher FROM production.product"; + // 创建语句对象 + Statement stmt = conn.createStatement(); + // 执行查询 + ResultSet rs = stmt.executeQuery(sql); + // 显示结果集 + displayResultSet(rs); + // 关闭结果集 + rs.close(); + // 关闭语句 + stmt.close(); + } + + /* 调用存储过程修改产品信息表数据 + * @throws SQLException 异常 */ + public void updateProduct() throws SQLException { + // 更新数据语句 + String sql = "{ CALL production.updateProduct(?,?) }"; + // 创建语句对象 + CallableStatement cstmt = conn.prepareCall(sql); + // 为参数赋值 + cstmt.setInt(1, 1); + cstmt.setString(2, "红楼梦（上）"); + // 执行语句 + cstmt.execute(); + // 关闭语句 + cstmt.close(); + } + + /* 显示结果集 + * @param rs 结果集对象 + * @throws SQLException 异常 */ + private void displayResultSet(ResultSet rs) throws SQLException { + // 取得结果集元数据 + ResultSetMetaData rsmd = rs.getMetaData(); + // 取得结果集所包含的列数 + int numCols = rsmd.getColumnCount(); + // 显示列标头 + for (int i = 1; i <= numCols; i++) { + if (i > 1) { + System.out.print(","); + } + System.out.print(rsmd.getColumnLabel(i)); + } + System.out.println(""); + // 显示结果集中所有数据 + while (rs.next()) { + for (int i = 1; i <= numCols; i++) { + if (i > 1) { + System.out.print(","); + } + // 处理大字段 + if ("IMAGE".equals(rsmd.getColumnTypeName(i))) { + byte[] data = rs.getBytes(i); + if (data != null && data.length > 0) { + FileOutputStream fos; + try { + fos = new FileOutputStream("c:\\三国演义1.jpg"); + fos.write(data); + fos.close(); + } catch (FileNotFoundException e) { + System.out.println(e.getMessage()); + } catch (IOException e) { + System.out.println(e.getMessage()); + } + } + System.out.print("字段内容已写入文件c:\\三国演义1.jpg，长度" + data.length); + } else { + // 普通字段 + System.out.print(rs.getString(i)); + } + } + System.out.println(""); + } + } + + /* 创建一个图片用于插入大字段 + * @throws IOException 异常 */ + private void CreateImage(String path) throws IOException { + int width = 100; + int height = 100; + String s = "三国演义"; + File file = new File(path); + Font font = new Font("Serif", Font.BOLD, 10); + BufferedImage bi = new BufferedImage(width, height, + BufferedImage.TYPE_INT_RGB); + Graphics2D g2 = (Graphics2D) bi.getGraphics(); + g2.setBackground(Color.WHITE); + g2.clearRect(0, 0, width, height); + g2.setPaint(Color.RED); + FontRenderContext context = g2.getFontRenderContext(); + Rectangle2D bounds = font.getStringBounds(s, context); + double x = (width - bounds.getWidth()) / 2; + double y = (height - bounds.getHeight()) / 2; + double ascent = -bounds.getY(); + double baseY = y + ascent; + g2.drawString(s, (int) x, (int) baseY); + ImageIO.write(bi, "jpg", file); + } + + //类主方法 @param args 参数 + public static void main(String args[]) { + try { + // 定义类对象 + BasicApp basicApp = new BasicApp(); + // 加载驱动程序 + basicApp.loadJdbcDriver(); + // 连接DM数据库 + basicApp.connect(); + // 插入数据 + System.out.println("--- 插入产品信息 ---"); + basicApp.insertTable(); + // 查询含有大字段的产品信息 + System.out.println("--- 显示插入结果 ---"); + basicApp.queryProduct(); + // 在修改前查询产品信息表 + System.out.println("--- 在修改前查询产品信息 ---"); + basicApp.queryTable(); + // 修改产品信息表 + System.out.println("--- 修改产品信息 ---"); + basicApp.updateTable(); + // 在修改后查询产品信息表 + System.out.println("--- 在修改后查询产品信息 ---"); + basicApp.queryTable(); + // 删除产品信息表 + System.out.println("--- 删除产品信息 ---"); + basicApp.deleteTable(); + // 在删除后查询产品信息表 + System.out.println("--- 在删除后查询产品信息 ---"); + basicApp.queryTable(); + // 调用存储过程修改产品信息表 + System.out.println("--- 调用存储过程修改产品信息 ---"); + basicApp.updateProduct(); + // 在存储过程更新后查询产品信息表 + System.out.println("--- 调用存储过程后查询产品信息 ---"); + basicApp.queryTable(); + // 关闭连接 + basicApp.disConnect(); + } catch (SQLException e) { + System.out.println(e.getMessage()); + } + } +} diff --git a/dataSources/postgresql/pom.xml b/dataSources/postgresql/pom.xml index 78da27ffc1..7ee0ca6f0e 100644 --- a/dataSources/postgresql/pom.xml +++ b/dataSources/postgresql/pom.xml @@ -29,7 +29,7 @@ org.postgresql postgresql - 42.4.3 + LATEST diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 643453d2a9..9e460cbc7d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -42,6 +42,7 @@ import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.postgresql.entity.PostgreSQLQueryRowStream; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; +import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; import org.postgresql.ds.PGConnectionPoolDataSource; @@ -98,6 +99,8 @@ public class PostgreSQLStorage implements IStorage { private Connection connection; + private String D_URL = ""; + public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { this.meta = meta; if (!testConnection()) { @@ -132,7 +135,23 @@ private boolean testConnection() { } } + private String getUrl(String dbname) { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), dbname, + username, password); + return connUrl; + } + private Connection getConnection(String dbname, String url) { + try { + Statement stmt = connection.createStatement(); + stmt.execute(String.format("create database %s", dbname)); + } catch (SQLException e) { + //database exists + } try { if (connectionPoolMap.containsKey(dbname)) { return connectionPoolMap.get(dbname).getConnection(); @@ -142,7 +161,7 @@ private Connection getConnection(String dbname, String url) { connectionPoolMap.put(dbname, connectionPool); return connectionPool.getConnection(); } catch (SQLException e) { - logger.error("cannot get connection for database: {}", dbname); + logger.error("cannot get connection for database: {}", e); return null; } } @@ -157,16 +176,10 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { FragmentMeta fragment = task.getTargetFragment(); Operator op = operators.get(0); String storageUnit = task.getStorageUnit(); - // 先切换数据库 - //connection.close(); - Connection conn = null; - try { - conn = useDatabase(storageUnit); - } catch (Exception e) { - logger.info("pass"); - } + boolean isDummyStorageUnit = task.isDummyStorageUnit(); + Connection conn = getConnection(storageUnit, getUrl(storageUnit)); - if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 + if (op.getType() == OperatorType.Project) { Project project = (Project) op; Filter filter; if (operators.size() == 2) { @@ -176,7 +189,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); } - return executeProjectTask(conn, project, filter); + return isDummyStorageUnit ? executeHistoryProjectTask(project, filter) : executeProjectTask(conn, project, filter); } else if (op.getType() == OperatorType.Insert) { Insert insert = (Insert) op; return executeInsertTask(conn, insert); @@ -185,42 +198,52 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { return executeDeleteTask(conn, delete); } return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); + new NonExecutablePhysicalTaskException("unsupported physical task in postgresql ")); } @Override - public List getTimeSeries() throws PhysicalException { + public List getTimeSeries() { List timeseries = new ArrayList<>(); + Map extraParams = meta.getExtraParams(); try { Statement stmt = connection.createStatement(); ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取数据库名称 -// if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - -// DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - if (tableName.startsWith("unit")) { - tableName = tableName.substring(tableName.indexOf(POSTGRESQL_SEPARATOR) + 1); + try { + String databaseName = databaseSet.getString(1);//获取数据库名称 + if (extraParams.get("has_data").equals("true")) { + } else { + if (databaseName.startsWith("unit")) { + } else { + continue; + } } - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - //if((tableName+"."+columnName).startsWith(meta.getDataPrefix())) - timeseries.add(new Timeseries( - databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); + Connection conn1 = getConnection(databaseName, getUrl(databaseName)); + DatabaseMetaData databaseMetaData = conn1.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString(3);//获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + if (databaseName.startsWith(DATABASE_PREFIX)) { + timeseries.add(new Timeseries( + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName))); + } else { + timeseries.add(new Timeseries( + databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName))); + } + } } + } catch (Exception e) { + logger.info("error:", e); } -// } } } catch (SQLException e) { throw new RuntimeException(e); @@ -228,6 +251,11 @@ public List getTimeSeries() throws PhysicalException { return timeseries; } + + private long toHash(String s) { + return (long) Integer.valueOf(s); + } + @Override public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { long minTime = Long.MAX_VALUE, maxTime = 0; @@ -236,174 +264,201 @@ public Pair getBoundaryOfStorage(String prefix) t Statement stmt = connection.createStatement(); ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取表名称 -// if (databaseName.startsWith(DATABASE_PREFIX)) { - //connection.close(); - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); + String databaseName = databaseSet.getString(1);//获取database名称 + Connection conn2 = getConnection(databaseName, getUrl(databaseName)); + DatabaseMetaData databaseMetaData = conn2.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); while (tableSet.next()) { String tableName = tableSet.getString(3);//获取表名称 ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + String fields = ""; while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); - // 获取first - String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); - Statement firstQueryStmt = connection.createStatement(); - String execIfNoTimestamp_first = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); - String execIfNoTimestamp_last = String.format("select first(%s,time) from (select to_timestamp(row_number() over()) as time,%s from %s)", columnName, columnName, tableName); - try { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); - } - } catch (Exception e) { - ResultSet firstQuerySet = firstQueryStmt.executeQuery(execIfNoTimestamp_first); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); //没有时间列执行该语句 - } - } - // 获取last - String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - Statement lastQueryStmt = connection.createStatement(); - try { - ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } catch (Exception e) { //没有时间戳执行该部分 - ResultSet lastQuerySet = lastQueryStmt.executeQuery(execIfNoTimestamp_last); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } + fields = fields + columnName + ","; //c1,c2,c3, + } + fields = fields.substring(0, fields.lastIndexOf(",")); //c1,c2,c3 + + // 获取first + String firstQueryStatement = String.format("select concat(%s) from %s", fields, tableName); + Statement firstQueryStmt = conn2.createStatement(); + ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); + while (firstQuerySet.next()) { + String s = firstQuerySet.getString(0); + long logic_time = toHash(s); + minTime = Math.min(logic_time, minTime); + maxTime = Math.max(logic_time, maxTime); } } -// } } } catch (SQLException e) { - throw new PhysicalException(e); + throw new RuntimeException(e); } paths.sort(String::compareTo); - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), new TimeInterval(minTime, maxTime + 1)); } - private TaskExecuteResult executeProjectTask(Connection conn, Project project, - Filter filter) { // 未来可能要用 tsInterval 对查询出来的数据进行过滤 - String filter1 = filter.toString().replace("key", "time").replace("&&", filter.getType().toString()); - String[] filter_list = filter1.split("\\s"); - long ft = Long.parseLong(filter_list[6].substring(0, filter_list[6].indexOf(")"))); - while (ft > 9223372036854L) { - ft = ft / 10; - } - filter1 = String.format("(time %s to_timestamp(%d) %s time %s to_timestamp(%d))", - filter_list[1], Long.parseLong(filter_list[2]), filter.getType().toString(), filter_list[5], ft); + + private TaskExecuteResult executeProjectTask(Connection conn, Project project, Filter filter) { try { List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); for (String path : project.getPatterns()) { - ArrayList allDatabase = new ArrayList<>(); Statement stmt = conn.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - allDatabase.add(databaseSet.getString(1)); - } - ArrayList databases = new ArrayList<>(); - String[] path_l = path.split("\\."); - if (path_l.length < 3) { - if (path.equals("*.*")) { - } else { - path = "postgres." + path; - } + String tableName = path.substring(0, path.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String columnName = path.substring(path.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = null; + ResultSet columnSet = null; + if (path.equals("*.*")) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else if (columnName.equals("*")) { + columnSet = databaseMetaData.getColumns(null, null, tableName, null); + } else { } - String database_table = path.substring(0, path.lastIndexOf('.')); - String tableName = ""; - String field1 = ""; - boolean allTable = false; - if (path.equals("*.*")) { - stmt = conn.createStatement(); - databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - databases.add(databaseSet.getString(1)); + if (tableSet == null && columnSet == null) { + ResultSet rs = stmt.executeQuery(String.format("select time,%s from %s where %s", columnName, tableName, FilterTransformer.toString(filter))); + resultSets.add(rs); + ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, columnName); + String typeName = "INT"; + if (columnSet_.next()) { + typeName = columnSet_.getString("TYPE_NAME");//列字段类型 + } + fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + } else if (tableSet == null && columnSet != null) { + while (columnSet.next()) { + String field = columnSet.getString("COLUMN_NAME"); + if (!field.equals("time")) { + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + ResultSet rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, tableName, FilterTransformer.toString(filter))); + resultSets.add(rs); + fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + } } - allTable = true; - } else if (database_table.substring(database_table.lastIndexOf(".")).equals("*")) { - allTable = true; } else { -// String database_table = path.substring(0, path.lastIndexOf('.')); - String database = database_table.substring(0, database_table.lastIndexOf('.')); - tableName = database_table.substring(database_table.lastIndexOf('.') + 1); - if (tableName.equals("*")) { - allTable = true; + while (tableSet.next()) { + String table = tableSet.getString(3);//获取表名称 + ResultSet columnSet1 = databaseMetaData.getColumns(null, null, table, null); + while (columnSet1.next()) { + String field = columnSet1.getString("COLUMN_NAME"); + if (!field.equals("time")) { + String typeName = columnSet1.getString("TYPE_NAME");//列字段类型 + ResultSet rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, table, FilterTransformer.toString(filter))); + resultSets.add(rs); + fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + } + } } - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - databases.add(database); - field1 = path.substring(path.lastIndexOf('.') + 1); - field1 = field1.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); } + } + RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields, false); + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.info("error: ", e); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); + } + } - for (int i = 0; i < databases.size(); i++) { - String database = (String) databases.get(i); - useDatabase(database); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - if (allTable) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); + + private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { + try { + List resultSets = new ArrayList<>(); + List fields = new ArrayList<>(); + for (String path : project.getPatterns()) { + String database_table = path.substring(0, path.lastIndexOf(".")); + String dataBaseName = database_table.substring(0, database_table.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String tableName = database_table.substring(database_table.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String columnName = path.substring(path.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + Connection conn = getConnection(dataBaseName, getUrl(dataBaseName)); + Statement stmt = conn.createStatement(); + + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = null; + ResultSet columnSet = null; + if (path.equals("*.*")) { + tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + } else if (columnName.equals("*")) { + columnSet = databaseMetaData.getColumns(null, null, tableName, null); + } + + if (tableSet == null && columnSet == null) { + ResultSet columnSet2 = databaseMetaData.getColumns(null, null, tableName, null); + String hv = ""; + while (columnSet2.next()) { + String columnName2 = columnSet2.getString("COLUMN_NAME");//获取列名称 + hv = hv + columnName2 + ","; //c1,c2,c3, + } + hv = hv.substring(0, hv.lastIndexOf(",")); + ResultSet rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName)); + resultSets.add(rs); + ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, columnName); + String typeName = "INT"; + if (columnSet_.next()) { + typeName = columnSet_.getString("TYPE_NAME");//列字段类型 + } + fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); + } else if (tableSet == null && columnSet != null) { + while (columnSet.next()) { + String hv = ""; + String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); + while (columnSet_.next()) { + String columnName2 = columnSet_.getString("COLUMN_NAME");//获取列名称 + hv = hv + columnName2 + ","; //c1,c2,c3, + } + hv = hv.substring(0, hv.lastIndexOf(",")); + + String field = columnSet.getString("COLUMN_NAME"); + ResultSet rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, tableName)); + resultSets.add(rs); + fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + , DataTypeTransformer.fromPostgreSQL(typeName))); } + } else { while (tableSet.next()) { String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, null, table, field1); - if (field1.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, table, null); + ResultSet columnSet1 = databaseMetaData.getColumns(null, null, table, null); + String hv = ""; + while (columnSet1.next()) { + String columnName1 = columnSet1.getString("COLUMN_NAME");//获取列名称 + hv = hv + columnName1 + ","; //c1,c2,c3, } - while (columnSet.next()) { - String field = columnSet.getString("COLUMN_NAME"); - if (field.equals("time")) { - continue; - } - String statement = ""; - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields.add(new Field(database.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + hv = hv.substring(0, hv.lastIndexOf(",")); + while (columnSet1.next()) { + String field = columnSet1.getString("COLUMN_NAME"); + String typeName = columnSet1.getString("TYPE_NAME");//列字段类型 + ResultSet rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, table)); + resultSets.add(rs); + fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); - ResultSet rs; - try { - statement = String - .format("SELECT time,%s FROM %s where %s", field, table, filter1); - stmt = conn.createStatement(); - rs = stmt.executeQuery(statement); - } catch (Exception e) { - statement = String - .format("SELECT time,%s FROM (select to_timestamp(row_number() over()) as time,%s from %s) as a where %s", field, field, table, filter1); - stmt = conn.createStatement(); - rs = stmt.executeQuery(statement); - } - resultSets.add(rs); } } } } - RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields); + RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields, true); return new TaskExecuteResult(rowStream); } catch (SQLException e) { logger.info("error: ", e); return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", - e)); + new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); } } + private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { DataView dataView = insert.getData(); Exception e = null; @@ -426,24 +481,29 @@ private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { private void createTimeSeriesIfNotExists(Connection conn, String table, String field, Map tags, DataType dataType) { try { + DatabaseMetaData databaseMetaData = conn.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); if (!tableSet.next()) { Statement stmt = conn.createStatement(); StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + if (tags != null && !tags.isEmpty()) { + for (Entry tagsEntry : tags.entrySet()) { + stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + } } stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, + .format("CREATE TABLE %s (time INTEGER NOT NULL,%s NULL)", table, stringBuilder.toString())); } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = conn.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + if (tags != null && tags.isEmpty()) { + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); + if (!columnSet.next()) { + Statement stmt = conn.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + } } } ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); @@ -458,26 +518,6 @@ private void createTimeSeriesIfNotExists(Connection conn, String table, String f } } - private Connection useDatabase(String dbname) throws SQLException { - try { - // TODO 为什么不能直接判断数据库是否存在，而是通过创建数据库来捕获异常？ - Statement stmt = connection.createStatement(); - stmt.execute(String.format("create database %s", dbname)); - } catch (SQLException e) { - logger.info("database {} exist!", dbname); - } - - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), - dbname, username, password); - Connection conn = getConnection(dbname, connUrl); - logger.info("change database success, the current database is: {}", dbname); - return conn; - } - private Exception insertRowRecords(Connection conn, RowDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { @@ -496,13 +536,9 @@ private Exception insertRowRecords(Connection conn, RowDataView data) { if (data.hasTagsList()) { tags = data.getTags(i); } - // TODO connection为什么从来没close过？ - // TODO 为什么这么频繁地切换database？ - // TODO database ?= storage unit createTimeSeriesIfNotExists(conn, table, field, tags, dataType); - // TODO timescaledb？ - long time = data.getKey(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 + long time = data.getKey(i); String value; if (data.getDataType(j) == DataType.BINARY) { value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; @@ -512,15 +548,17 @@ private Exception insertRowRecords(Connection conn, RowDataView data) { StringBuilder columnsKeys = new StringBuilder(); StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); + if (tags != null && !tags.isEmpty()) { + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } } columnsKeys.append(field); columnValues.append(value); stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, + .format("INSERT INTO %s (time, %s) values (%d, %s)", table, columnsKeys, time, columnValues)); index++; @@ -528,6 +566,7 @@ private Exception insertRowRecords(Connection conn, RowDataView data) { if (cnt % batchSize == 0) { stmt.executeBatch(); } + } } } @@ -546,23 +585,27 @@ private Exception insertColumnRecords(Connection conn, ColumnDataView data) { try { Statement stmt = conn.createStatement(); int cnt = 0; + String fields = ""; + String values = ""; + String table = ""; + long time = 0; for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); DataType dataType = data.getDataType(i); - String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); Map tags = new HashMap<>(); if (data.hasTagsList()) { tags = data.getTags(i); } - + fields = fields + "," + field; // ,id1,id2 createTimeSeriesIfNotExists(conn, table, field, tags, dataType); BitmapView bitmapView = data.getBitmapView(i); int index = 0; for (int j = 0; j < data.getTimeSize(); j++) { if (bitmapView.get(j)) { - long time = data.getKey(j) / 1000; // 时间戳 + time = data.getKey(j); String value; if (data.getDataType(i) == DataType.BINARY) { value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; @@ -572,24 +615,29 @@ private Exception insertColumnRecords(Connection conn, ColumnDataView data) { StringBuilder columnsKeys = new StringBuilder(); StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); + if (tags != null && !tags.isEmpty()) { + for (Entry tagEntry : tags.entrySet()) { + columnsKeys.append(tagEntry.getValue()).append(" "); + columnValues.append(tagEntry.getValue()).append(" "); + } } columnsKeys.append(field); columnValues.append(value); + values = values + "," + value; //,123,456 cnt++; index++; - stmt.addBatch(String.format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, columnsKeys, time, columnValues)); - if (cnt % batchSize == 0) { - stmt.executeBatch(); - } +// stmt.addBatch(String.format("INSERT INTO %s (time, %s) values (%d, %s)", table, columnsKeys, time, columnValues)); +// if (cnt % batchSize == 0) { +// stmt.executeBatch(); +// } } +// stmt.executeBatch(); } - stmt.executeBatch(); } + stmt.execute(String.format("INSERT INTO %s (time %s) values (%d %s)", table, fields, time, values)); } catch (SQLException e) { + logger.info("error", e); return e; } @@ -600,47 +648,26 @@ private TaskExecuteResult executeDeleteTask(Connection conn, Delete delete) { try { for (int i = 0; i < delete.getPatterns().size(); i++) { String path = delete.getPatterns().get(i); - String[] path_l = path.split("\\."); - if (path_l.length < 3) { - if (path.equals("*.*")) { - } else { - path = "postgres." + path; - } - } TimeRange timeRange = delete.getTimeRanges().get(i); - String db_ta = path.substring(0, path.lastIndexOf('.')); - String database = db_ta.substring(0, db_ta.lastIndexOf('.')); - String table = db_ta.substring(db_ta.lastIndexOf(',') + 1); - database = database.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String table = path.substring(0, path.lastIndexOf('.')); table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String field = path.substring(path.lastIndexOf('.') + 1); field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); // 查询序列类型 -// useDatabase(database); DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - if (table.equals("*")) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"table"}); - } - while (tableSet.next()) { - String tableName = tableSet.getString(3); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, field); - if (columnSet.next()) { - String statement = String - .format(DELETE_DATA, tableName, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = conn.createStatement(); - stmt.execute(statement); - } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (columnSet.next()) { + String statement = String.format("delete from %s where (time>%d and time<%d)", table, + timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); + Statement stmt = conn.createStatement(); + stmt.execute(statement); } } return new TaskExecuteResult(null, null); } catch (SQLException e) { + logger.info("error:", e); return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", - e)); + new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", e)); } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index b111f4006c..065f99f903 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -17,47 +17,44 @@ public class PostgreSQLQueryRowStream implements RowStream { private final List resultSets; private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - + private final long[] currTimestamps; - + private final Object[] currValues; - + private final Header header; - - public PostgreSQLQueryRowStream(List resultSets, List fields) { + private boolean isdummy; + + public PostgreSQLQueryRowStream(List resultSets, List fields, boolean isdummy) throws SQLException { this.resultSets = resultSets; + this.isdummy = isdummy; this.header = new Header(Field.KEY, fields); this.currTimestamps = new long[resultSets.size()]; this.currValues = new Object[resultSets.size()]; -// this.values=new ArrayList<>(); - // 默认填充一下timestamp列表 try { - long j = 1; for (int i = 0; i < this.currTimestamps.length; i++) { - j = 1; ResultSet resultSet = this.resultSets.get(i); if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - } catch (Exception e) { - this.currTimestamps[i] = j++; + if (isdummy) { + this.currTimestamps[i] = toHash(resultSet.getString(1)); + } else { + this.currTimestamps[i] = resultSet.getLong(1); } this.currValues[i] = resultSet.getObject(2); } } - j++; } catch (SQLException e) { e.printStackTrace(); // pass } - + } - + @Override public Header getHeader() { return this.header; } - + @Override public void close() { try { @@ -68,17 +65,21 @@ public void close() { // pass } } - + @Override - public boolean hasNext() throws PhysicalException { + public boolean hasNext() { for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE && currTimestamp != 0) { + if (currTimestamp != Long.MIN_VALUE) { return true; } } return false; } - + + private long toHash(String s) { + return Math.abs((long) Integer.valueOf(s)); + } + @Override public Row next() throws PhysicalException { try { @@ -89,16 +90,15 @@ public Row next() throws PhysicalException { timestamp = Math.min(timestamp, currTimestamp); } } - long j = 1; for (int i = 0; i < this.currTimestamps.length; i++) { if (this.currTimestamps[i] == timestamp) { values[i] = this.currValues[i]; ResultSet resultSet = this.resultSets.get(i); if (resultSet.next()) { - try { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - } catch (Exception e) { - logger.info("have no timestamp,set default timestamp!"); + if (isdummy) { + this.currTimestamps[i] = toHash(resultSet.getString(1)); + } else { + this.currTimestamps[i] = resultSet.getLong(1); } this.currValues[i] = resultSet.getObject(2); } else { @@ -110,6 +110,7 @@ public Row next() throws PhysicalException { } return new Row(header, timestamp, values); } catch (SQLException e) { + logger.info("error:", e); throw new RowFetchException(e); } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index 77fcb511a6..011a0762ac 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -23,9 +23,9 @@ import java.util.stream.Collectors; public class FilterTransformer { - + public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - + public static String toString(Filter filter) { if (filter == null) { return ""; @@ -45,26 +45,26 @@ public static String toString(Filter filter) { return ""; } } - + private static String toString(AndFilter filter) { return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); } - + private static String toString(NotFilter filter) { return "not " + filter.toString(); } - + private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; + return "time " + Op.op2Str(filter.getOp()) + Math.min(filter.getValue(), MAX_TIMESTAMP); } - + private static String toString(ValueFilter filter) { return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); } - + private static String toString(OrFilter filter) { return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); } - - + + } diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java index afcb767599..5fd4d58833 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java @@ -47,14 +47,15 @@ public class PostgreSQLSessionExample { public static void main(String[] args) throws SessionException, ExecutionException { session = new Session("127.0.0.1", 6888, "root", "root"); // 打开 Session + session.openSession(); // 列式插入对齐数据 insertColumnRecords(); // 行式插入对齐数据 insertRowRecords(); -// queryData(); -// deleteDataInColumns(); + queryData(); + deleteDataInColumns(); // 关闭 Session session.closeSession(); diff --git a/tools-viewPartitions/fragment.png b/tools-viewPartitions/fragment.png new file mode 100644 index 0000000000000000000000000000000000000000..d81949bc2794324cf6725288c172ab1790a31c83 GIT binary patch literal 11986 zcmeHtXH-*Lw{`&O5LBc$3xY_OCLKWqL^=ve2LTZXksc|b1&)Fsy*CBvMNny>1qFc+ zlwLz3kVp@`1^Bk-z2A4sxbOLM@3`X|cbxpl$k;o3uQ}J6YtHAH&w6QOpv}y{%>V*{ zn00k-n}9%+79bGSMLJ603Y0P?2?UbJ(7k=jEFgPz1`ZwB$@z6k)TO(0H{5v2DOo_Y z&unbs^A`&Ea{GG>0t>FYUPDK%?5ORQa^dc6>)U+>Z56&BW%^dcP5p8dO6Emma7Lab@VdAA^L4-5iRowzEOqRu-v?zykO_F7X|aboBSf~v4?S@Z%*klecU8gj2=&E`?Ui}%~$+(=qG zC8ST)SP|r2B3ZRNdVy19b$QS31G9C4S{ zhzOkP)5znNmWRHV?mUypV)25~Dnsz8&tu(+e3hq!r6~6K+L||pEx(znQ1jy2O;ROX zV66I%oOPkpu7gAA@)?v6#%eA1 zDJ8Ri?Xp+79H&^(yW95#bWzct*WyeR%KK*p@Tzj9xK(!(F#sYNm`8-=r;I);lVN#a zI&;qrpMY#mG0R2BBq;IRyd*Y&EeYKn$}E^$s=w@lL6&03kIZ6crzomanh|VfAM-}p zM@?-Fk14=tmlH;$k^YW!oD=`Eca@n-s4Fo4fqlV338pt`cNrgOy6x^_FFU^CGl%1V z2ud;g>|77^OsHJi+?R?f(xLDjZs}Cci17IMz&HhPt9?S1TEr1&b&kLVZpPW)r1StW zAv|i>g`7N*45CK?8P|LHcyUoOK^)Ee1pF0V*LTdEz280DE&C>Q@Apmzq?nrn?-#YS z#iu+gQ~&wysDY)V_{7P^2K{qpCAWVC^*ipcZ+_(oSfuC(Hto<$i5n$(D=^!(U)1S* zI8Ngo=!Q^2Kth<3l*c`N*B#RzKP~N_2ro&5Zv+4qmPu%mwGiPa%&z zT33sI?yzZ%2On=M`?716v@XzYfqg=x)Q6k#$?N#@>hj#$2`kE z(6;)Pi=9kXeO`;WN>9(Awt=F?^{zJUl9MGZPOu~X(*79;;?bSExnhd%x8E(v32fq_ zQSnYIS}NQ0dDFd>b+6ppbq!Ma>)s&f&*yC}I0 zl|$cS;Ma;-{xodUDJM5? zDLSxf;c-49m4^Gh^y>Ah$LOCP6XGhU1R8D#arb#_;D>p3g8n$)@+0_Fb4ttc8@U@A z;a|>1&Clp3im7SL>T|mqEDGiXsr=5 z4y?F}iSB}HPQ51Sl@akvy`2P^m!0A}lU5ze2!Y8hjIp*}a;YbB$!YedrFX`WcS!3W zcQj=BwdyP_vg#1xAFbZZpknSp?6HwWBc`V+18wL&md%U*S_()Id_rz^IZ8TG$-Jxc zh7QV@lX=}GzAZz>`NGOKY)HK@D`6t=eawS`jab{RTEU)oynOw?_UbJ6oiXfj4#ROP z?5%-i;^-Wy^@0{_5B$F`7Yj3$Z_7wJ@ffoo`-3Hzm&`RZgWtg;P^yY>nrjiV$(^^- z8F=id#li%wPag}*xI1=4!uTUnT0#s8-0eEKM+!CM>fEU2`6avC zM2aGKflLGLZZ1Q*ccz;eJ?!hV!sp^{NrKBIJRN^b1Gk08q${MtxY*3li%5hP=Z(lq zu+GhxaC(%B%&dQ`>F2skDE{sn$SQx%OE8f?Kdd&Z6*L~AE; zU!uYCChp1J!?zVI>SX9X3Gdh)7yPRrTJ2yy$wViDZr!>y=u8wVjzFJG%ib@XZ>Nnn9!~N_M z^A(&jNEQ{cf2)T+tdj&%F%5b^YbPQWEfm1YuEG7j!%%7V*rOwK*4gEk*Hv3x`ppnz zwPig7@avs+JZ9USh{>sLJ9**Zu_`xaWol~by;x)m!FLP5kuP?ZCMPF-lvEwQ{IMQH zRMv2GoI5!_f|DWylGD%L4B97Av8a&HG6$CF_liEdPt>}NRarOt^=HTsBUseNn|znQ zykzZ;7hnlGIUo+G9l66`2g9YN-En-)keQ||NRulG*KxGFI#uT~p!ls!^7Xx?sQP^R zwJO#V0o6ULz|~J0dU|?VTAd$VhpL*E%D%>Mcuwsuu=6W=6&4iWcc(lB6d=+0vJ zwYJlc{P*t(J;HfsLJo)W{N@|Y4+cvLA=jd{jKGedr{P>u=kKEQ6+1KSUhM8C=iX1> z5S)JSgsc<*-bHPJK5F{z7A#e*vGbR%tTl#QO-V}=Q1U53B7@NB(i*0wL$O>kcjI^! zz2?7VK@R)a<>ln`^VC$mgL;!qEYlV2yOIpmU}M(Sp7_@a*mPaO_y@=JhB>eud4HqX zfBmFOp*2BAFf%JFsb{{%z7zN%d_VpB_;>@bI=V?h!N<$C>Y~NAndXs+O?XQwju3DR z=xJ2JMD#Q7s88+G6H^>JPc-eQt!T_SxNKPlI{Qc$K_gJHBO^<5f5!MPyOo~7Zfnzx z>Zb<-9O949$SNqX@EtC&kB*)=FpfCu~LdYtY}!N?W<~SS{4@* zJ-VU5#>SSxv#_wRy1F`5@7cD6QI_#UD+cXD2@BIArtej@&@!yT1}gn^tCzH{l?MJ3 z3?LC<4>iQR3v512w0}dH561TJyNfnhj*KLby5a@e08gLXfNlT!@#CF#+v!P~qpfk=7R+@A^b zO|V!SYwO@nW`&B!pUMgf3MwkZatzXd*iZ?8E)AFR$h%trmLy))OYC`aW8y2jbCkz) zgS0>DLHWa$-AOlAq5I<>U0G{_4mN9RYrFXa-4}XO5Hj-_vaW;g;?=P<&{sSIiQeiURB?IvpngRfdRy zx8Kme;RKy~n1_@C@!NBqTlz$Maif&2+kXNH?T+TVrll980-z`UK_}z`Mh*#HX@}d_ zQ+ic^{q&wA{^rDg zcL)Bbnj-&2n!LN)(`xlcVYR&0Z)QAB-L4sM#u*h@e3*EFtV(XPR`og0AG$~yd}Q;! zFbvaXRPF7==N|P#iZS#*vR&f(oH}(og#|;O9_B!1UjX{H+(qDwq1KfZ^}gb(uKBG`)%}QaJ8AbmurehxR5><+LwTTvze-GzcpCNt3aieIDy+Wv7c_Gh z_m5)tl}P5bi8!S`J>t!dEv&wvRpR~g&zd8Q*ZucJ$H{Y8OO-eDfTWqA7~gIYzTSXr#F%Bq`0vI{BRi;^tJJXmSYG}Qh=uPr?*687Jl7MHa6-W zXg7C3=FN7>JX^tE@jF3^afWIQxu7$96cY(N$^(i*M{nH-uitBd0+=4|b~N7Owm84& z11HyH-@c5G-e+=u86I!nYc=p6^5SJ>+|9!L_giKWdKmiO$#L(a;YB2hZ@4`%AO8Qi zRa*I73r5|Nh;uo#X8!z&f`JJ0n-9|H(IugZ#f>Jq1p#+Tu=t%X8bmPI$L5@0mS<|h zU$syEc@O=6ox1S1L;nN9QN-a3imz9JFD0!t91w)pqi!qZtS!swBo4a(VCSzys}WmX zWxo}Lwgde&W@fROi4Fid?{4ehK|6^xr_9`%I~+YvHuu_2gV$bL2^;1H;lgO&8|Ix1 z>PhZ=UGjNWaUDf>-(^@-1Qy>|2Pm}w0@ilk=^ojqRjSHK21E^^T@b8l}C z87j!jug$idW{W+_$vyy68mjJGfa2Q2iuBSXY+I3~rLF+#21e8=9vXVDpQQrx!X^Ny zRMIo_lso&g%xs6*^-lm$1V{~lINUJViUL8Tj?qd>VE`G1kOOdA1M2YI375XUzG8zM zCBKz%-{sL6pCSDSS{7`*=WMQO;L_3(=BsY51duKzK7RT%6TCOWVH>mopnP`m$I17F zSa@%eCf%x?_=E8X4&bdn>P5|5U7pj90bq0?1?cg8cJCoG=?k9Miel0gN`h%K&~EoO zeuWa0$45tjsXziK*DzNF)|)Jn7rZ-x&X7ryum*6KF^;@7kf$2B>rp@BcY3@>B_{`{ zlm~*Op}doAUC8wpiK+Md{ zKGxUotvdK{V}fXx|NH>cBLzPpbJzqFG&LOpWG5pUprn8qzim*ROQKsv``1)dR5(V( zDNaoRq+<4De=Z)r*F5{%z~9UQZm;Upn>M}X}EA@gD%A05xDI5;{=!^EfL0{~xu zAm_-snh?6mt6Q7A7J6H4N}EQ9_@)7>ELx2;v9><`p)D}uKO3wO9DLj-*@g_*_&L+; zKai{1OaeAm&K;MHwhn;DsUH4%mnH9^w?jg)e*z<{4UrU;mvsp!5Tk)5Zs@dE0(_lmr_ zU7SnG%9a93EB|G@<+hI9-CY8K08G1$rK{wP$faUz$kD1N{AlWBqySjis4;@dI-5`S zx9ar&f1j_?Hk!oP`n0&E+WZGXuxAg}mKAY|&I80hiMA(zAW_#c&XGGmpP?QY@4QB+ zfrP*$b(0K|))o_DA5=_s#9^)T3d5X@Y$fyEL`>8P-CCT321g@S$5bKElVkLk>Dk3OL}4O1s4s6R9^cA!9XGv@ug zpFIBSwC~g!{qq*ni|p=b8FI4r=UTb-;gc|UL;O+h!i0=fVPSP*2si4rJ2KS`{*6UB z4*4yndUJ{1fc%HR>J#F#hg7B+N~s^Z6;Fgb*PQnjtU~D1RQl=`l3n;I{c5*-ZK{iQ zxvNSG2qqFU9dTo@V--wR51O%2YG@KZH+#b{`SPcXep{$T7P+apE^c7zi$p< zm{8t#Cw06-eX~36H709Gn%}Z_P_LmdOPj&4GW=C*OLodI%g#{RUCD((efMWYpXR=e zSgD4eW6juC%u`2Q8Ol(5?aH!6A_j#-Uh((d8{3}*CP9rj;JoWG?Wev+CH_(NyVPq} z^XDTD?4FKO{JScsEB;Vz)g5Np3u?YR9>6nStyChoVl0vcAKldD@u=Q>mKG7yKf7^N z`{tlV=jyat+;hpz%ine8OjSlU25x~N|5lx^N5z$fe2FhBZB0_n!)H{wi)WXO`#0q5 z-pV=U+|~<$DfqiUI4m_55x%9+gd+bM$9G$pg)3^@T{UU20}k?A1be% zs;`k8O(x~-7o!ota#(jc0a8{6TiZfCaQq-Kpk5}*VuHeR)|%WMAeyPINYu8xa~6cf%%{DP-rPf7(p9;K%Z>ac?;rw4 z*o7ISuGa!HwmoY4V6RU7&UODFR{f!j+zbx}(@@XN{&TUvIF0muMJq?lh}g=)+CEP+ zKHr^TDwa(gd^2yBcmy+LI|8fsBsEH^fBx|~XO(5chyeLO?E|;J$_}G8Paf}xdtLSR z`ks{!`B%n<75|rA*Vj6xW_oZIAt9y;$Jftlv-{jRz~aYOQ#hs)8N%87NWrS5_~xL8+VXnG_3iTI}sNqB_92|CZM zbafvw+CM$LG5E_KoMxKaXnpIB@+u-}zLzIk;WP_*U`~Fpw$J`p;&+?NK|QX#bnOox z9*?Up{4Q}gL(Gg*0x7H~86H}7IhlL=b|=_wlut{rTWaiaJakC7>Q@ul$(!3Z{rH)( zjmg??^PB&O!TzjVC*`bFSecwTR*2*xdWQ({02?o)S>}~7-+IP$52kIwQ<9 zIpJk{^?{sJ=8t6yf8}~A&rJf|-B<^+@Z0e=*F>`mzu-^@wm1Zmf@dPV@mOL#&hASf zGVKWa!#K9=-5MXZF^nfvezL4&vJZ>Sy0wYypUU77*!VF^Qb3q4Sbr$eJ!L%I93%U# zH9t-iRH%bdFe=4ekjUTOP=S^ATRwToC+PA!?zpQCWZ;-HD`$j5rDGKr06*M{Mi}n( zju>h+WR7@}x)4?k(W>x)St>2Arug8PN65&llAAvPflP<2aC+<*GQE*iqkI2FvbMs9 z#k0{jv>~{t2Nloq3u0?!7rlCgZ)_X|!G)DYODhTpxA@PD)~RiEWpCYF&pRL@zg{pf zaagE_>fyj6&9#O!5;MbxDs~gM?vmk8Vn{K<6jpjUl-OL;C@wxyz2>EUyH{$O@h;QA6Rc?9n+Vg4ri9lp#%q2mdwO1Z zS3N($Vk+J@je`eazx}>y^CD7Pjksmm$ky;O%K%n5H!N71ep_i~hlk_H>ecYMIIDJQ zC&l$&id`Sj55Ap?0|BdvHFvV<5euSVO&=g5#L!j)Y)6}>y*Y)E(+5N;;q|v?Z~m6- zB#vL$F^WGhdtmh8Pth)|5k|FJcj`i3;UNFKrmbv)RMiqOiQvkv4?W+##ZS5U(9%7j zs6HRa10R1p)OA|RyVK+?Xr4B8Nu)O9#T_~(XK43&s*(>GDz;1YF}4=TAV1iU`bjs@ zci8t#MKS$~SK@guVqSaXDNfz%^g7+u*{;EYs7&;0hU)~W;_;dry_kp8Vbg`aV)Ly~C|4T3%ft>yZ>M=0g8!nSdn} zEaV^BjndOdxrq*VsQTtYH)ene2SOuy6j|B%B|i)g89AiE+tUOVl$nK`S>D^-PywB? zh9V~FESYA-Zce~Mk8l$*Hb8|-St?+97$*;stw^Q+>+alvkwDz>qlygm)^KL;0&zG&im@IW> ztxI2R)e4RsEto$Ut!O3uxq3DzpUpzE8y*?IPfN0b&2%50dzTVPXq*RH?gpR){Y#Xcs(?+ z{Z%db+J+%3Uyk@~M2b=fMI_PPk9H+V1%b73kbQ0|FtO9Ufq9KKpPJi_+lo5Io@XUY z8wW0lj;hLSVqS}iv8NFWe`22Wv7`JB4M$enoL-y;)iihxcL&6cFU^L7Q^}Zl%+{5X zC}lj?_+%1nlJSIYTdzSmFz_FG&P7<++|>&bwcESCD_;1S9tX+T z3C2cdfh<`3JE%$2ynSlcrt;ZCcD%!#JD}kXwSaoNL@QW?f=@17y-+FMGkkGk;ys9@ z&zT^T-!wH_q*OwSyR~n)A5u?*)o%60d(`mp_hLQz4L5&qs#YJLIxHWMv8u;VdD@k`T z8&{yucXy4(sh-#lO@ZS({R_^6wYW3AZQfM$haB*bTuS;6az^_VC7b!8Hdj^|o~}Rp zZSNKV&U#X|wa?ygG4O~9e%Us?#dzdksqrHll*=l|d)_>!*&=64JMuGTV|g3eb>Q+d$J3> zL*_csBS{(W=}l7|9ay(sxGx8KXU{hcZqJgUynE|1(W!myT6SS1P;;(v)<_2)S#wgX z`-ZIlDhU^@Y>`5i6JIC?&RyBhf}oS4;9+fRFb_b6&b?Ij#v14XEOiPFgbtt_io zT+(84`Zy*i80)H!`9Vr~i&8pliW3h-uZh^gneop8s3am)NqfN2zTg_p)b^N~Gv`UO zq`7R{DE)A4dsg0F{zi(2x2TeDarfF^4ifOwygJ+-0jd~il_sj}6T^R(4V1Wbf8Mf^ z1%=+ljAJ>=ZsmusU`!AFe31<7?NZ|uZQ+@w{afoK?)#zoAhGBh0YXQ2CnBWSc1S|5*T&JL7l#+`;0kWM>-L9t=d6VaiUYS?VTp6%1EiF(y0 znq5pV;0Po-*y?qhYslXksCJxSOt_-4d%WFvqxuPZYbZTB8Zoil=wKUiuQ4h4jq+)B zc#J0Q8q->rUU)k@aI%mwZaq4$LlXBCBUT_SSL{m``0G=9o!bw8ZX81wwFm;oszb(CLqZ z>hSML{$I@6vwy~?=Cm*jD!R(e+|1oOd*`8L!?(fNQwduiJ}Z(HBC7q(&-f~IYn7<3 v;^78cfuL^7fNhq}-)9Z~@`&ZpDFu(QEcdrO5nte}I!O19!R;zdyYT-5feqm- literal 0 HcmV?d00001 From 6398c1a7bd2f5821915d9010dea8f330e6f27e7a Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 8 Mar 2023 18:48:38 +0800 Subject: [PATCH 22/94] fix --- conf/config.properties | 4 ++-- .../tsinghua/iginx/postgresql/PostgreSQLStorage.java | 2 +- .../postgresql/entity/PostgreSQLQueryRowStream.java | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 61eb55dab6..ca4989c5bb 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,11 +14,11 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres +storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=true #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 9e460cbc7d..0d84e353b1 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -211,7 +211,7 @@ public List getTimeSeries() { while (databaseSet.next()) { try { String databaseName = databaseSet.getString(1);//获取数据库名称 - if (extraParams.get("has_data").equals("true")) { + if (extraParams.get("has_data") != null && extraParams.get("has_data").equals("true")) { } else { if (databaseName.startsWith("unit")) { } else { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 065f99f903..5acb195c02 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -77,7 +77,16 @@ public boolean hasNext() { } private long toHash(String s) { - return Math.abs((long) Integer.valueOf(s)); + char c[] = s.toCharArray(); + long hv = 0; + long base = 131; + for (int i = 0; i < c.length; i++) { + hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 + } + if (hv < 0) { + return -1 * hv; + } + return hv; } @Override From 091087c1ea60b96c3621bc68b0b42528b19b6c66 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Wed, 8 Mar 2023 23:36:15 +0800 Subject: [PATCH 23/94] fix --- .../iginx/postgresql/PostgreSQLStorage.java | 20 ++++++++++++++++-- tools-viewPartitions/fragment.png | Bin 11986 -> 6457 bytes 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 0d84e353b1..a3ad4f6297 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -151,6 +151,7 @@ private Connection getConnection(String dbname, String url) { stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { //database exists + logger.info("database exists!", e); } try { if (connectionPoolMap.containsKey(dbname)) { @@ -253,7 +254,16 @@ public List getTimeSeries() { private long toHash(String s) { - return (long) Integer.valueOf(s); + char c[] = s.toCharArray(); + long hv = 0; + long base = 131; + for (int i = 0; i < c.length; i++) { + hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 + } + if (hv < 0) { + return -1 * hv; + } + return hv; } @Override @@ -266,6 +276,9 @@ public Pair getBoundaryOfStorage(String prefix) t while (databaseSet.next()) { String databaseName = databaseSet.getString(1);//获取database名称 Connection conn2 = getConnection(databaseName, getUrl(databaseName)); + if (conn2 == null) { + continue; + } DatabaseMetaData databaseMetaData = conn2.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); while (tableSet.next()) { @@ -285,7 +298,7 @@ public Pair getBoundaryOfStorage(String prefix) t Statement firstQueryStmt = conn2.createStatement(); ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); while (firstQuerySet.next()) { - String s = firstQuerySet.getString(0); + String s = firstQuerySet.getString(1); long logic_time = toHash(s); minTime = Math.min(logic_time, minTime); maxTime = Math.max(logic_time, maxTime); @@ -380,6 +393,9 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt String tableName = database_table.substring(database_table.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = path.substring(path.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); Connection conn = getConnection(dataBaseName, getUrl(dataBaseName)); + if (conn == null) { + continue; + } Statement stmt = conn.createStatement(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/tools-viewPartitions/fragment.png b/tools-viewPartitions/fragment.png index d81949bc2794324cf6725288c172ab1790a31c83..6c278386e941f5cdb31659ee9158dabaeb68a165 100644 GIT binary patch literal 6457 zcmeHMcTiK?x~C&bIf{sMk0_vkM37D>Dgsg!6+%hEp-Jx@5~N-aMGVqQKm$kq&Z++j|&&^EqxK0b6 zW?*38GSI*Kkb!|w8+fNqF##4ub3z0Ig8+lUU7bh4x$DFT@5j4&)UBX%m*Dz;q%u37>=Vy~(%k_6x+^mrIZ$E#jt$&KB?TbX(V8SrU9Ob9+@i7OAhyjFrzO{ zQpC;KAKg(PbogK7sW4;aWu)>MsV1BaybUIFUJe!u)L}J>ye+jcC-v~;!aQH2$i}ld z5dV|gOxzvPrMM%FT$O~gqA}mRg?b~k+BAZNlW(iLul>F8#Z0zTMydF&0mY8fEwsrjhKnlyHePRz@hN-3#l4R7#MrtB4ZRhco}cKGrwVd%PmmOkI zM-czN_D^2*7X?IUaEx~u0)?4sukIiEwBd32k+@UHll*^84G^>D{<%$0*nkfI+cV0u zMb(v6ODRK)ToK^-$YfD)lc%i~Cg#{ei+(5B=!+ds_~|+jt0!9lHSUf%qN*qk5s}s@>UU z9&7=Vg!b>}@BAm>klg4B#4>g3N7&xl!OpO$o10syQI_v^pE7emKtL7l@H)tEvG=BZ z7lU+lyt2}BxWw4e(NTHY-`v0ee>KDAW7z&i zTwL4-`!7)B@orsRU5=7R)AH+t#lD>zE38XR^3 z%_?YjZ7s$W&K=Gv*6vPa*=w^>In^3{W$oGLYJMv?TR11VO=g_a*jtu%E?I}UH!Ae) zGgj4T2SnW+=I~Pui{a87@dd&&Q9*NT!Dsi{MF!d8;jF?@RUK7xC++?!Gt-y&hsIX4 ztc;C~$83Y9(`Bs%g@iJ0*f#ks^_yxQIP~YLiXmxRdSVf`04#r52J*pfW0m0G%~>*H zAL}5GaURJ3oWM_>#x@ieJ9Z@s{A#aGZdPy`EeBvxR#dbNTd!-SEy%qPQ6C)~1OfKL z!^7#)eTFtwo)_%x&ZV|OdP$+%W+NKlzk6&12Uq}A0`vEyYuyo2w=N5`b3q@}k}HD`ZmUc3u~29sQ;N+z>7Vbog;jm$zrHjtMFet410EnP z{mf&kUe>m0=X-+O#@d=j;F_h5j?UtxEDD8UWo32l`CS(m7ehls7ni-Ox$=(PZwyi+ z0hc8PjM=uuaPSHW3RYqkv&$nSO>!+CKi>VsYx?NP6KLp4@lKu!BaeC=h{dXqD-LaC z&UrNFR(%n{DLGe`C}LeeCHPL1#Xc*&FbP*^W_JX&F6p?=-fR>(J?G7XEX~8AQ$(8? zn&g$FzK&n=I~I$#>(s=?+==6K@$m3)a}&~})figZ*fb+g=v5KxZVCKy^sN`jaLPQs zH2;v~+*({*+(-qQJlUJLDy*_uvKVmM4Sw(eejsRGn=blUN$lFz;s>9rnh zIA?h8Z>$-r06hR>l?{e6MWXMmP1Mw|8F> zb5*tg=>hyUqAyz!y}d$3eF|X}W#YMWjnJWB#v%mgns3+h7ITjuyToB@#-c4=9;LX+ zBoiO5@6^m$ViX^1#vW_NlxiaOM!=~f$Bg5>9lLP%c#y0{7(G97IgV3+vf0U#k)A#} zI;y$dtDvi^E2QLZS>t33h0Xwc^(RH_dysI)bz@LNiBwx#TcClqz5S1OkKd1$L$*84 z$jQKP8d_RfSwbh)C1`lv0@#Ui0yz@9ye| zdf=Lh|1wf;ZnDk|9kJJ&0SY2XqQfbL@uE#MN{<(2A)Ox66umKXjd) zmjG(Xk2Tc)dP>A306JWLiO~rQ3j=yfKUtej@JsN zkSxmf_AuJXJdPW@evVu4RRo8*7v%lBclGWQDE%wlJw1Af%Zulnl2aA+5A6^L#G&{t zkoRnxZbJXwxnEzKzluG&MOp&5yZmB%2+~u8q(8NVe_3A$^; zObeVtBEXDgq#l4$5ny%$bgEb3zsz$+V8Q@Z_*<#Ei&6bwBS&CY&OL{n9FAq3Q7b~w zGAx@S{r&wP0pg>qJT^LNg4T?nQMW28DijqJf$)O`vHKoooF%`IDE{x`2ZmdUiWRU{ zTtNYRu{X=UD{-PJ;3uKXoSjRMFh1T0=riDq+dim>b_;kO&-(iMKGAPrbFOo#KM(7> z(0%u^&*p5q1vKR9?(`BGN&obm*I9V3on1^A2!=cwrj>vZwP~r(-V*fKCVSi&9>)4at z-Q7*-=t>d-WN<=(`!NK93Ef>4Q4bVW^Ovey!~?3hTo`|&l}54l#4LQ~hvjM^=@@Vp z?uVA-Bbb`+yuN{f!J(ae*4o;dFbJ+KpGB;58=Gkf>Arcx8onCD6)fpV~l>^{U{NH+F=@&?X-DYrZzw2-HE7l5rD0v=nlMW2& z=cqSG#(C4%pV)p0fJh!$&_<4lt@*n!Z`84#78licCil}DEJnS>iCIYL;}(l#`iJu5 zCW{chOnmRnMAiPS={3&+I~jYqCCYBOT*0MX*@EOIWB2PtYZ9K<3+GQ)zAX;q&AIXY zbx5yJS2cnBH{`sSIs0lsQTD1+wp&?V|ISw8fXKXbcyZ?Ue5HOtErXAl-?`NuVi{VO z_Nyqa0*(f#>3q6Svk#vIxVO^)<(HYt=ly$ZSz)qYI8T`7!myQVDQLmTvtPuOftjA) zTJ?I>T+_JXfSt~$(mO}q%nK;TMyqekau=kHrj;9wObxR%*`mY#W;AW`F+-Xvgh<5= zLhf)mc^}FbA9Er7q#N>Q@2<|7QaJ#`Uu;uN4_rp^7akA>NSZX*?r_7Ph4*Lfz}lf~ zsGqoUGW;$ppYg2-G#Q!d)o<5dq%nEdT*zbJd%%L)ZP~@`xXg4RJlad6jk)b?vW)YA z@FKB|>%C=NM$6H}NdykbA7pGl>6^YJ+LX5!*66I#xg;NSRhsvTFS|==gt!oPy0LU? z7+d?TK6jxe3!J|tG|uqo!7eWXMRzR;kyMuiF@6>I0!8y)`RJ4>>MGFlr<7F(DxiGo zb0x7bG&ByoEa-h$MMiJcgf&Xm5r+a|t;>$!~{MRxg*7^?~*9_$rzniGN@R12(nr=N>oFW8Hy1hLRB^iR( zT$r4-K0qZ@lYLrj+IM0_J5qZ61bSz&7;H*MM({N>O>8q-A{#oFhWw7EN1y0>)u&Yq zY!Ml`Y0@W?>-fMypOokfZ@fUM$ZajIiglFMVl?~ zTbjClNqJID7pZSUYtB@@L963s0!9VcrWX&+>aM_Utk>=j&4J>IFT{baY;zfs^A}g7 zF_e3rMpYqk&hlX!e_!5258s+DBWsx{ChgG;hu=3ZzDTm`4IL%hnYu7JIdDIvM7bD->R))Q(u6K$S>FW{e~?k&8`KvV@Q`LeG#{vd9dYu(EdYR2c3Es!Wv zXA`MjboFiMj?}$0b2>LUJgMb}{)WHl(vX3(Yc!cMO>mBh7G{wjsC_lW zt=0sdcw!}&8M@NI*xLO*XF?;<4q(+1P1VfEM?48&S)8KKcy*j&$b7^ny?r0pVJh@$ z+IvV#v`D-l-K|Q-$TpWcySFTcYEvAxCvdGoiSR# zsKc+7E)Hkqt0PKsFb6LA(9Ud@=LF}kJ05RpU5t*D=^lLV1)o~BF~Mcs&-as&e9Wu0s&n{H?ep#Z4$h0+dk$f)g^ClC+3tPQ?-eE{a@{uq+xYN&N2sGv z(s80Z`-<)&^-Ss+Kjo)}^#NDZL&e@)oj^^Ns6mcxC>SDFHp*BUC9Rg4r-#z(*VwH| z)tB&<9PBwP_gd($FE!Xha18OQxp~Us(`S~DG9l&r9!)L6KD)yfaQJW0L+HGrA@an- z6EMg^^8g}d{{HP`M>@-M-|k^DB5blna+-T6p?6DCehWs;=sme|uc{$q=)RPJET_Zl zCAI~Vf{a}V>2>_XqRGd7!@>2OZ5_^TwI`MBoT9z6`M4vQfs&aM z7>I(xZ=9IFc|#{h9qC^>zIpSXFSP%nSC9Yg;{D&w^XNn~{Q3o!ramh0ON7Bd*W_;H IZFtmw0J{7X$N&HU literal 11986 zcmeHtXH-*Lw{`&O5LBc$3xY_OCLKWqL^=ve2LTZXksc|b1&)Fsy*CBvMNny>1qFc+ zlwLz3kVp@`1^Bk-z2A4sxbOLM@3`X|cbxpl$k;o3uQ}J6YtHAH&w6QOpv}y{%>V*{ zn00k-n}9%+79bGSMLJ603Y0P?2?UbJ(7k=jEFgPz1`ZwB$@z6k)TO(0H{5v2DOo_Y z&unbs^A`&Ea{GG>0t>FYUPDK%?5ORQa^dc6>)U+>Z56&BW%^dcP5p8dO6Emma7Lab@VdAA^L4-5iRowzEOqRu-v?zykO_F7X|aboBSf~v4?S@Z%*klecU8gj2=&E`?Ui}%~$+(=qG zC8ST)SP|r2B3ZRNdVy19b$QS31G9C4S{ zhzOkP)5znNmWRHV?mUypV)25~Dnsz8&tu(+e3hq!r6~6K+L||pEx(znQ1jy2O;ROX zV66I%oOPkpu7gAA@)?v6#%eA1 zDJ8Ri?Xp+79H&^(yW95#bWzct*WyeR%KK*p@Tzj9xK(!(F#sYNm`8-=r;I);lVN#a zI&;qrpMY#mG0R2BBq;IRyd*Y&EeYKn$}E^$s=w@lL6&03kIZ6crzomanh|VfAM-}p zM@?-Fk14=tmlH;$k^YW!oD=`Eca@n-s4Fo4fqlV338pt`cNrgOy6x^_FFU^CGl%1V z2ud;g>|77^OsHJi+?R?f(xLDjZs}Cci17IMz&HhPt9?S1TEr1&b&kLVZpPW)r1StW zAv|i>g`7N*45CK?8P|LHcyUoOK^)Ee1pF0V*LTdEz280DE&C>Q@Apmzq?nrn?-#YS z#iu+gQ~&wysDY)V_{7P^2K{qpCAWVC^*ipcZ+_(oSfuC(Hto<$i5n$(D=^!(U)1S* zI8Ngo=!Q^2Kth<3l*c`N*B#RzKP~N_2ro&5Zv+4qmPu%mwGiPa%&z zT33sI?yzZ%2On=M`?716v@XzYfqg=x)Q6k#$?N#@>hj#$2`kE z(6;)Pi=9kXeO`;WN>9(Awt=F?^{zJUl9MGZPOu~X(*79;;?bSExnhd%x8E(v32fq_ zQSnYIS}NQ0dDFd>b+6ppbq!Ma>)s&f&*yC}I0 zl|$cS;Ma;-{xodUDJM5? zDLSxf;c-49m4^Gh^y>Ah$LOCP6XGhU1R8D#arb#_;D>p3g8n$)@+0_Fb4ttc8@U@A z;a|>1&Clp3im7SL>T|mqEDGiXsr=5 z4y?F}iSB}HPQ51Sl@akvy`2P^m!0A}lU5ze2!Y8hjIp*}a;YbB$!YedrFX`WcS!3W zcQj=BwdyP_vg#1xAFbZZpknSp?6HwWBc`V+18wL&md%U*S_()Id_rz^IZ8TG$-Jxc zh7QV@lX=}GzAZz>`NGOKY)HK@D`6t=eawS`jab{RTEU)oynOw?_UbJ6oiXfj4#ROP z?5%-i;^-Wy^@0{_5B$F`7Yj3$Z_7wJ@ffoo`-3Hzm&`RZgWtg;P^yY>nrjiV$(^^- z8F=id#li%wPag}*xI1=4!uTUnT0#s8-0eEKM+!CM>fEU2`6avC zM2aGKflLGLZZ1Q*ccz;eJ?!hV!sp^{NrKBIJRN^b1Gk08q${MtxY*3li%5hP=Z(lq zu+GhxaC(%B%&dQ`>F2skDE{sn$SQx%OE8f?Kdd&Z6*L~AE; zU!uYCChp1J!?zVI>SX9X3Gdh)7yPRrTJ2yy$wViDZr!>y=u8wVjzFJG%ib@XZ>Nnn9!~N_M z^A(&jNEQ{cf2)T+tdj&%F%5b^YbPQWEfm1YuEG7j!%%7V*rOwK*4gEk*Hv3x`ppnz zwPig7@avs+JZ9USh{>sLJ9**Zu_`xaWol~by;x)m!FLP5kuP?ZCMPF-lvEwQ{IMQH zRMv2GoI5!_f|DWylGD%L4B97Av8a&HG6$CF_liEdPt>}NRarOt^=HTsBUseNn|znQ zykzZ;7hnlGIUo+G9l66`2g9YN-En-)keQ||NRulG*KxGFI#uT~p!ls!^7Xx?sQP^R zwJO#V0o6ULz|~J0dU|?VTAd$VhpL*E%D%>Mcuwsuu=6W=6&4iWcc(lB6d=+0vJ zwYJlc{P*t(J;HfsLJo)W{N@|Y4+cvLA=jd{jKGedr{P>u=kKEQ6+1KSUhM8C=iX1> z5S)JSgsc<*-bHPJK5F{z7A#e*vGbR%tTl#QO-V}=Q1U53B7@NB(i*0wL$O>kcjI^! zz2?7VK@R)a<>ln`^VC$mgL;!qEYlV2yOIpmU}M(Sp7_@a*mPaO_y@=JhB>eud4HqX zfBmFOp*2BAFf%JFsb{{%z7zN%d_VpB_;>@bI=V?h!N<$C>Y~NAndXs+O?XQwju3DR z=xJ2JMD#Q7s88+G6H^>JPc-eQt!T_SxNKPlI{Qc$K_gJHBO^<5f5!MPyOo~7Zfnzx z>Zb<-9O949$SNqX@EtC&kB*)=FpfCu~LdYtY}!N?W<~SS{4@* zJ-VU5#>SSxv#_wRy1F`5@7cD6QI_#UD+cXD2@BIArtej@&@!yT1}gn^tCzH{l?MJ3 z3?LC<4>iQR3v512w0}dH561TJyNfnhj*KLby5a@e08gLXfNlT!@#CF#+v!P~qpfk=7R+@A^b zO|V!SYwO@nW`&B!pUMgf3MwkZatzXd*iZ?8E)AFR$h%trmLy))OYC`aW8y2jbCkz) zgS0>DLHWa$-AOlAq5I<>U0G{_4mN9RYrFXa-4}XO5Hj-_vaW;g;?=P<&{sSIiQeiURB?IvpngRfdRy zx8Kme;RKy~n1_@C@!NBqTlz$Maif&2+kXNH?T+TVrll980-z`UK_}z`Mh*#HX@}d_ zQ+ic^{q&wA{^rDg zcL)Bbnj-&2n!LN)(`xlcVYR&0Z)QAB-L4sM#u*h@e3*EFtV(XPR`og0AG$~yd}Q;! zFbvaXRPF7==N|P#iZS#*vR&f(oH}(og#|;O9_B!1UjX{H+(qDwq1KfZ^}gb(uKBG`)%}QaJ8AbmurehxR5><+LwTTvze-GzcpCNt3aieIDy+Wv7c_Gh z_m5)tl}P5bi8!S`J>t!dEv&wvRpR~g&zd8Q*ZucJ$H{Y8OO-eDfTWqA7~gIYzTSXr#F%Bq`0vI{BRi;^tJJXmSYG}Qh=uPr?*687Jl7MHa6-W zXg7C3=FN7>JX^tE@jF3^afWIQxu7$96cY(N$^(i*M{nH-uitBd0+=4|b~N7Owm84& z11HyH-@c5G-e+=u86I!nYc=p6^5SJ>+|9!L_giKWdKmiO$#L(a;YB2hZ@4`%AO8Qi zRa*I73r5|Nh;uo#X8!z&f`JJ0n-9|H(IugZ#f>Jq1p#+Tu=t%X8bmPI$L5@0mS<|h zU$syEc@O=6ox1S1L;nN9QN-a3imz9JFD0!t91w)pqi!qZtS!swBo4a(VCSzys}WmX zWxo}Lwgde&W@fROi4Fid?{4ehK|6^xr_9`%I~+YvHuu_2gV$bL2^;1H;lgO&8|Ix1 z>PhZ=UGjNWaUDf>-(^@-1Qy>|2Pm}w0@ilk=^ojqRjSHK21E^^T@b8l}C z87j!jug$idW{W+_$vyy68mjJGfa2Q2iuBSXY+I3~rLF+#21e8=9vXVDpQQrx!X^Ny zRMIo_lso&g%xs6*^-lm$1V{~lINUJViUL8Tj?qd>VE`G1kOOdA1M2YI375XUzG8zM zCBKz%-{sL6pCSDSS{7`*=WMQO;L_3(=BsY51duKzK7RT%6TCOWVH>mopnP`m$I17F zSa@%eCf%x?_=E8X4&bdn>P5|5U7pj90bq0?1?cg8cJCoG=?k9Miel0gN`h%K&~EoO zeuWa0$45tjsXziK*DzNF)|)Jn7rZ-x&X7ryum*6KF^;@7kf$2B>rp@BcY3@>B_{`{ zlm~*Op}doAUC8wpiK+Md{ zKGxUotvdK{V}fXx|NH>cBLzPpbJzqFG&LOpWG5pUprn8qzim*ROQKsv``1)dR5(V( zDNaoRq+<4De=Z)r*F5{%z~9UQZm;Upn>M}X}EA@gD%A05xDI5;{=!^EfL0{~xu zAm_-snh?6mt6Q7A7J6H4N}EQ9_@)7>ELx2;v9><`p)D}uKO3wO9DLj-*@g_*_&L+; zKai{1OaeAm&K;MHwhn;DsUH4%mnH9^w?jg)e*z<{4UrU;mvsp!5Tk)5Zs@dE0(_lmr_ zU7SnG%9a93EB|G@<+hI9-CY8K08G1$rK{wP$faUz$kD1N{AlWBqySjis4;@dI-5`S zx9ar&f1j_?Hk!oP`n0&E+WZGXuxAg}mKAY|&I80hiMA(zAW_#c&XGGmpP?QY@4QB+ zfrP*$b(0K|))o_DA5=_s#9^)T3d5X@Y$fyEL`>8P-CCT321g@S$5bKElVkLk>Dk3OL}4O1s4s6R9^cA!9XGv@ug zpFIBSwC~g!{qq*ni|p=b8FI4r=UTb-;gc|UL;O+h!i0=fVPSP*2si4rJ2KS`{*6UB z4*4yndUJ{1fc%HR>J#F#hg7B+N~s^Z6;Fgb*PQnjtU~D1RQl=`l3n;I{c5*-ZK{iQ zxvNSG2qqFU9dTo@V--wR51O%2YG@KZH+#b{`SPcXep{$T7P+apE^c7zi$p< zm{8t#Cw06-eX~36H709Gn%}Z_P_LmdOPj&4GW=C*OLodI%g#{RUCD((efMWYpXR=e zSgD4eW6juC%u`2Q8Ol(5?aH!6A_j#-Uh((d8{3}*CP9rj;JoWG?Wev+CH_(NyVPq} z^XDTD?4FKO{JScsEB;Vz)g5Np3u?YR9>6nStyChoVl0vcAKldD@u=Q>mKG7yKf7^N z`{tlV=jyat+;hpz%ine8OjSlU25x~N|5lx^N5z$fe2FhBZB0_n!)H{wi)WXO`#0q5 z-pV=U+|~<$DfqiUI4m_55x%9+gd+bM$9G$pg)3^@T{UU20}k?A1be% zs;`k8O(x~-7o!ota#(jc0a8{6TiZfCaQq-Kpk5}*VuHeR)|%WMAeyPINYu8xa~6cf%%{DP-rPf7(p9;K%Z>ac?;rw4 z*o7ISuGa!HwmoY4V6RU7&UODFR{f!j+zbx}(@@XN{&TUvIF0muMJq?lh}g=)+CEP+ zKHr^TDwa(gd^2yBcmy+LI|8fsBsEH^fBx|~XO(5chyeLO?E|;J$_}G8Paf}xdtLSR z`ks{!`B%n<75|rA*Vj6xW_oZIAt9y;$Jftlv-{jRz~aYOQ#hs)8N%87NWrS5_~xL8+VXnG_3iTI}sNqB_92|CZM zbafvw+CM$LG5E_KoMxKaXnpIB@+u-}zLzIk;WP_*U`~Fpw$J`p;&+?NK|QX#bnOox z9*?Up{4Q}gL(Gg*0x7H~86H}7IhlL=b|=_wlut{rTWaiaJakC7>Q@ul$(!3Z{rH)( zjmg??^PB&O!TzjVC*`bFSecwTR*2*xdWQ({02?o)S>}~7-+IP$52kIwQ<9 zIpJk{^?{sJ=8t6yf8}~A&rJf|-B<^+@Z0e=*F>`mzu-^@wm1Zmf@dPV@mOL#&hASf zGVKWa!#K9=-5MXZF^nfvezL4&vJZ>Sy0wYypUU77*!VF^Qb3q4Sbr$eJ!L%I93%U# zH9t-iRH%bdFe=4ekjUTOP=S^ATRwToC+PA!?zpQCWZ;-HD`$j5rDGKr06*M{Mi}n( zju>h+WR7@}x)4?k(W>x)St>2Arug8PN65&llAAvPflP<2aC+<*GQE*iqkI2FvbMs9 z#k0{jv>~{t2Nloq3u0?!7rlCgZ)_X|!G)DYODhTpxA@PD)~RiEWpCYF&pRL@zg{pf zaagE_>fyj6&9#O!5;MbxDs~gM?vmk8Vn{K<6jpjUl-OL;C@wxyz2>EUyH{$O@h;QA6Rc?9n+Vg4ri9lp#%q2mdwO1Z zS3N($Vk+J@je`eazx}>y^CD7Pjksmm$ky;O%K%n5H!N71ep_i~hlk_H>ecYMIIDJQ zC&l$&id`Sj55Ap?0|BdvHFvV<5euSVO&=g5#L!j)Y)6}>y*Y)E(+5N;;q|v?Z~m6- zB#vL$F^WGhdtmh8Pth)|5k|FJcj`i3;UNFKrmbv)RMiqOiQvkv4?W+##ZS5U(9%7j zs6HRa10R1p)OA|RyVK+?Xr4B8Nu)O9#T_~(XK43&s*(>GDz;1YF}4=TAV1iU`bjs@ zci8t#MKS$~SK@guVqSaXDNfz%^g7+u*{;EYs7&;0hU)~W;_;dry_kp8Vbg`aV)Ly~C|4T3%ft>yZ>M=0g8!nSdn} zEaV^BjndOdxrq*VsQTtYH)ene2SOuy6j|B%B|i)g89AiE+tUOVl$nK`S>D^-PywB? zh9V~FESYA-Zce~Mk8l$*Hb8|-St?+97$*;stw^Q+>+alvkwDz>qlygm)^KL;0&zG&im@IW> ztxI2R)e4RsEto$Ut!O3uxq3DzpUpzE8y*?IPfN0b&2%50dzTVPXq*RH?gpR){Y#Xcs(?+ z{Z%db+J+%3Uyk@~M2b=fMI_PPk9H+V1%b73kbQ0|FtO9Ufq9KKpPJi_+lo5Io@XUY z8wW0lj;hLSVqS}iv8NFWe`22Wv7`JB4M$enoL-y;)iihxcL&6cFU^L7Q^}Zl%+{5X zC}lj?_+%1nlJSIYTdzSmFz_FG&P7<++|>&bwcESCD_;1S9tX+T z3C2cdfh<`3JE%$2ynSlcrt;ZCcD%!#JD}kXwSaoNL@QW?f=@17y-+FMGkkGk;ys9@ z&zT^T-!wH_q*OwSyR~n)A5u?*)o%60d(`mp_hLQz4L5&qs#YJLIxHWMv8u;VdD@k`T z8&{yucXy4(sh-#lO@ZS({R_^6wYW3AZQfM$haB*bTuS;6az^_VC7b!8Hdj^|o~}Rp zZSNKV&U#X|wa?ygG4O~9e%Us?#dzdksqrHll*=l|d)_>!*&=64JMuGTV|g3eb>Q+d$J3> zL*_csBS{(W=}l7|9ay(sxGx8KXU{hcZqJgUynE|1(W!myT6SS1P;;(v)<_2)S#wgX z`-ZIlDhU^@Y>`5i6JIC?&RyBhf}oS4;9+fRFb_b6&b?Ij#v14XEOiPFgbtt_io zT+(84`Zy*i80)H!`9Vr~i&8pliW3h-uZh^gneop8s3am)NqfN2zTg_p)b^N~Gv`UO zq`7R{DE)A4dsg0F{zi(2x2TeDarfF^4ifOwygJ+-0jd~il_sj}6T^R(4V1Wbf8Mf^ z1%=+ljAJ>=ZsmusU`!AFe31<7?NZ|uZQ+@w{afoK?)#zoAhGBh0YXQ2CnBWSc1S|5*T&JL7l#+`;0kWM>-L9t=d6VaiUYS?VTp6%1EiF(y0 znq5pV;0Po-*y?qhYslXksCJxSOt_-4d%WFvqxuPZYbZTB8Zoil=wKUiuQ4h4jq+)B zc#J0Q8q->rUU)k@aI%mwZaq4$LlXBCBUT_SSL{m``0G=9o!bw8ZX81wwFm;oszb(CLqZ z>hSML{$I@6vwy~?=Cm*jD!R(e+|1oOd*`8L!?(fNQwduiJ}Z(HBC7q(&-f~IYn7<3 v;^78cfuL^7fNhq}-)9Z~@`&ZpDFu(QEcdrO5nte}I!O19!R;zdyYT-5feqm- From 1a706aea6ee4f0ee6798c21f2e7da6bb41bc8f2a Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 9 Mar 2023 16:33:22 +0800 Subject: [PATCH 24/94] Optimize insert --- .../engine/shared/data/write/DataView.java | 17 +- .../iginx/postgresql/PostgreSQLStorage.java | 330 ++++++++++++------ 2 files changed, 242 insertions(+), 105 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java index f6fad2ebbb..47921426f5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java @@ -83,11 +83,19 @@ public int getPathIndex(String path) { return data.getPaths().contains(path) ? data.getPaths().indexOf(path) - startPathIndex : -1; } + public List getPaths() { + return data.getPaths().subList(startPathIndex, endPathIndex); + } + public String getPath(int index) { checkPathIndexRange(index); return data.getPaths().get(startPathIndex + index); } + public List getDataTypeList() { + return data.getDataTypeList().subList(startPathIndex, endPathIndex); + } + public DataType getDataType(int index) { checkTypeIndexRange(index); return data.getDataTypeList().get(startPathIndex + index); @@ -107,6 +115,13 @@ public Long getKey(int index) { public abstract BitmapView getBitmapView(int index); + public List> getTagsList() { + if (data.getTagsList() != null && !data.getTagsList().isEmpty()) { + return data.getTagsList().subList(startPathIndex, endPathIndex); + } + return data.getTagsList(); + } + public Map getTags(int index) { checkPathIndexRange(index); List> tagsList = data.getTagsList(); @@ -117,7 +132,7 @@ public Map getTags(int index) { } public boolean hasTagsList() { - return data.getTagsList() != null && data.getTagsList().size() != 0; + return getTagsList() != null && getTagsList().size() != 0; } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 9e460cbc7d..400c1e3939 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -54,6 +54,7 @@ import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class PostgreSQLStorage implements IStorage { @@ -465,11 +466,11 @@ private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { switch (dataView.getRawDataType()) { case Row: case NonAlignedRow: - e = insertRowRecords(conn, (RowDataView) dataView); + e = insertNonAlignedRowRecords(conn, (RowDataView) dataView); break; case Column: case NonAlignedColumn: - e = insertColumnRecords(conn, (ColumnDataView) dataView); + e = insertNonAlignedColumnRecords(conn, (ColumnDataView) dataView); break; } if (e != null) { @@ -479,98 +480,153 @@ private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { return new TaskExecuteResult(null, null); } - private void createTimeSeriesIfNotExists(Connection conn, String table, String field, Map tags, DataType dataType) { - try { + private void createOrAlterTables(Connection conn, List paths, List> tagsList, List dataTypeList) { + for (int i = 0; i < paths.size(); i++) { + String path = paths.get(i); + Map tags = new HashMap<>(); + if (tagsList != null && !tagsList.isEmpty()) { + tags = tagsList.get(i); + } + DataType dataType = dataTypeList.get(i); + String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = conn.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - if (tags != null && !tags.isEmpty()) { - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); + try { + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (!tableSet.next()) { + Statement stmt = conn.createStatement(); + StringBuilder stringBuilder = new StringBuilder(); + if (tags != null && !tags.isEmpty()) { + for (String tag : tags.keySet()) { + // 列名=序列名$tagKey + stringBuilder.append(field).append(POSTGRESQL_SEPARATOR).append(tag).append(" TEXT, "); + } } - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time INTEGER NOT NULL,%s NULL)", table, - stringBuilder.toString())); - } else { - if (tags != null && tags.isEmpty()) { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = conn.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); + stringBuilder.append(field).append(" ").append(DataTypeTransformer.toPostgreSQL(dataType)); + stmt.execute(String.format("CREATE TABLE %s (time BIGINT NOT NULL, %s, PRIMARY KEY(time))", table, stringBuilder)); + } else { + if (tags != null && !tags.isEmpty()) { + for (String tag : tags.keySet()) { + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field + POSTGRESQL_SEPARATOR + tag); + if (!columnSet.next()) { + Statement stmt = conn.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT", table, field + POSTGRESQL_SEPARATOR + tag)); + } } } + ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (!columnSet.next()) { + Statement stmt = conn.createStatement(); + stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, + DataTypeTransformer.toPostgreSQL(dataType))); + } } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (!columnSet.next()) { - Statement stmt = conn.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toPostgreSQL(dataType))); - } + } catch (SQLException e) { + logger.error("create or alter table {} field {} error: {}", table, field, e.getMessage()); } - } catch (SQLException e) { - logger.error("create timeseries error", e); } } - private Exception insertRowRecords(Connection conn, RowDataView data) { + private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); + + // 创建表 + createOrAlterTables(conn, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + + // 插入数据 + Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> int cnt = 0; - for (int i = 0; i < data.getTimeSize(); i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - if (bitmapView.get(j)) { + boolean firstRound = true; + while (cnt < data.getTimeSize()) { + int size = Math.min(data.getTimeSize() - cnt, batchSize); + Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 + for (int i = cnt; i < cnt + size; i++) { + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getPathNum(); j++) { String path = data.getPath(j); DataType dataType = data.getDataType(j); String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { tags = data.getTags(i); } - createTimeSeriesIfNotExists(conn, table, field, tags, dataType); - long time = data.getKey(i); - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; - } else { - value = data.getValue(i, index).toString(); + StringBuilder columnKeys = new StringBuilder(); + List columnValues = new ArrayList<>(); + if (tableToColumnEntries.containsKey(table)) { + columnKeys = new StringBuilder(tableToColumnEntries.get(table).k); + columnValues = tableToColumnEntries.get(table).v; } - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - if (tags != null && !tags.isEmpty()) { - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); + String value = "null"; + if (bitmapView.get(j)) { + if (dataType == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + } else { + value = data.getValue(i, index).toString(); + } + index++; + if (tableHasData.containsKey(table)) { + tableHasData.get(table)[i - cnt] = true; + } else { + boolean[] hasData = new boolean[size]; + hasData[i - cnt] = true; + tableHasData.put(table, hasData); } } - columnsKeys.append(field); - columnValues.append(value); - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (%d, %s)", table, - columnsKeys, time, columnValues)); + if (firstRound) { + if (tags != null && !tags.isEmpty()) { + for (Entry tagEntry : tags.entrySet()) { + columnKeys.append(field).append(POSTGRESQL_SEPARATOR).append(tagEntry.getKey()).append(", "); + columnValues = columnValues.stream().map(x -> x + tagEntry.getValue() + ", ").collect(Collectors.toList()); + } + } + columnKeys.append(field).append(", "); + } - index++; - cnt++; - if (cnt % batchSize == 0) { - stmt.executeBatch(); + if (i - cnt < columnValues.size()) { + columnValues.set(i - cnt, columnValues.get(i - cnt) + value + ", "); + } else { + columnValues.add(data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 } + tableToColumnEntries.put(table, new Pair<>(columnKeys.toString(), columnValues)); + } + + firstRound = false; + } + + for (Map.Entry entry : tableHasData.entrySet()) { + String table = entry.getKey(); + boolean[] hasData = entry.getValue(); + String columnKeys = tableToColumnEntries.get(table).k; + List columnValues = tableToColumnEntries.get(table).v; + boolean needToInsert = false; + for (int i = hasData.length - 1; i >= 0; i--) { + if (!hasData[i]) { + columnValues.remove(i); + } else { + needToInsert = true; + } + } + if (needToInsert) { + tableToColumnEntries.put(table, new Pair<>(columnKeys, columnValues)); } } + + executeBatch(stmt, tableToColumnEntries); + for (Pair> columnEntries : tableToColumnEntries.values()) { + columnEntries.v.clear(); + } + + cnt += size; } - stmt.executeBatch(); conn.close(); } catch (SQLException e) { logger.error(e.getMessage()); @@ -580,62 +636,110 @@ private Exception insertRowRecords(Connection conn, RowDataView data) { return null; } - private Exception insertColumnRecords(Connection conn, ColumnDataView data) { + private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); + + // 创建表 + createOrAlterTables(conn, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + + // 插入数据 + Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> + Map pathIndexToBitmapIndex = new HashMap<>(); int cnt = 0; - String fields = ""; - String values = ""; - String table = ""; - long time = 0; - for (int i = 0; i < data.getPathNum(); i++) { - String path = data.getPath(i); - DataType dataType = data.getDataType(i); - table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Map tags = new HashMap<>(); - if (data.hasTagsList()) { - tags = data.getTags(i); - } - fields = fields + "," + field; // ,id1,id2 - createTimeSeriesIfNotExists(conn, table, field, tags, dataType); - - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - time = data.getKey(j); - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + boolean firstRound = true; + while (cnt < data.getTimeSize()) { + int size = Math.min(data.getTimeSize() - cnt, batchSize); + Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 + for (int i = 0; i < data.getPathNum(); i++) { + String path = data.getPath(i); + DataType dataType = data.getDataType(i); + String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String field = path.substring(path.lastIndexOf('.') + 1); + Map tags = new HashMap<>(); + if (data.hasTagsList()) { + tags = data.getTags(i); + } + + BitmapView bitmapView = data.getBitmapView(i); + + StringBuilder columnKeys = new StringBuilder(); + List columnValues = new ArrayList<>(); + if (tableToColumnEntries.containsKey(table)) { + columnKeys = new StringBuilder(tableToColumnEntries.get(table).k); + columnValues = tableToColumnEntries.get(table).v; + } + + int index = 0; + if (pathIndexToBitmapIndex.containsKey(i)) { + index = pathIndexToBitmapIndex.get(i); + } + for (int j = cnt; j < cnt + size; j++) { + String value = "null"; + if (bitmapView.get(j)) { + if (dataType == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + } else { + value = data.getValue(i, index).toString(); + } + index++; + if (tableHasData.containsKey(table)) { + tableHasData.get(table)[j - cnt] = true; + } else { + boolean[] hasData = new boolean[size]; + hasData[j - cnt] = true; + tableHasData.put(table, hasData); + } + } + + if (j - cnt < columnValues.size()) { + columnValues.set(j - cnt, columnValues.get(j - cnt) + value + ", "); } else { - value = data.getValue(i, index).toString(); + columnValues.add(data.getKey(j) + ", " + value + ", "); // 添加 key(time) 列 } + } + pathIndexToBitmapIndex.put(i, index); - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); + if (firstRound) { + columnKeys.append(field).append(", "); if (tags != null && !tags.isEmpty()) { for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(" "); - columnValues.append(tagEntry.getValue()).append(" "); + columnKeys.append(field).append(POSTGRESQL_SEPARATOR).append(tagEntry.getKey()).append(", "); + columnValues = columnValues.stream().map(x -> x + tagEntry.getValue() + ", ").collect(Collectors.toList()); } } - columnsKeys.append(field); - columnValues.append(value); - values = values + "," + value; //,123,456 - - cnt++; - index++; -// stmt.addBatch(String.format("INSERT INTO %s (time, %s) values (%d, %s)", table, columnsKeys, time, columnValues)); -// if (cnt % batchSize == 0) { -// stmt.executeBatch(); -// } } -// stmt.executeBatch(); + + tableToColumnEntries.put(table, new Pair<>(columnKeys.toString(), columnValues)); + } + + for (Map.Entry entry : tableHasData.entrySet()) { + String table = entry.getKey(); + boolean[] hasData = entry.getValue(); + String columnKeys = tableToColumnEntries.get(table).k; + List columnValues = tableToColumnEntries.get(table).v; + boolean needToInsert = false; + for (int i = hasData.length - 1; i >= 0; i--) { + if (!hasData[i]) { + columnValues.remove(i); + } else { + needToInsert = true; + } + } + if (needToInsert) { + tableToColumnEntries.put(table, new Pair<>(columnKeys, columnValues)); + } + } + + executeBatch(stmt, tableToColumnEntries); + for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + entry.getValue().v.clear(); } + + firstRound = false; + cnt += size; } - stmt.execute(String.format("INSERT INTO %s (time %s) values (%d %s)", table, fields, time, values)); } catch (SQLException e) { logger.info("error", e); return e; @@ -644,6 +748,24 @@ private Exception insertColumnRecords(Connection conn, ColumnDataView data) { return null; } + private void executeBatch(Statement stmt, Map>> tableToColumnEntries) throws SQLException { + for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + StringBuilder insertStatement = new StringBuilder(); + insertStatement.append("INSERT INTO "); + insertStatement.append(entry.getKey()); + insertStatement.append(" (time, "); + insertStatement.append(entry.getValue().k, 0, entry.getValue().k.length() - 2); + insertStatement.append(") VALUES"); + for (String value : entry.getValue().v) { + insertStatement.append(" ("); + insertStatement.append(value, 0, value.length() - 2); + insertStatement.append("), "); + } + stmt.addBatch(insertStatement.substring(0, insertStatement.toString().length() - 2)); + } + stmt.executeBatch(); + } + private TaskExecuteResult executeDeleteTask(Connection conn, Delete delete) { try { for (int i = 0; i < delete.getPatterns().size(); i++) { From 8d19a530649de3e1cdd1b35c96fab34af9e55f2c Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Thu, 9 Mar 2023 16:46:22 +0800 Subject: [PATCH 25/94] fix --- .../iginx/postgresql/PostgreSQLStorage.java | 107 ++++++++++++------ .../entity/PostgreSQLQueryRowStream.java | 2 +- 2 files changed, 72 insertions(+), 37 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index b946447d63..f9378276e6 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -320,12 +320,12 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); for (String path : project.getPatterns()) { - Statement stmt = conn.createStatement(); String tableName = path.substring(0, path.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = path.substring(path.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); DatabaseMetaData databaseMetaData = conn.getMetaData(); ResultSet tableSet = null; ResultSet columnSet = null; + if (path.equals("*.*")) { tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); } else if (columnName.equals("*")) { @@ -333,8 +333,15 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F } else { } + if (tableSet == null && columnSet == null) { - ResultSet rs = stmt.executeQuery(String.format("select time,%s from %s where %s", columnName, tableName, FilterTransformer.toString(filter))); + Statement stmt = conn.createStatement(); + ResultSet rs = null; + try { + rs = stmt.executeQuery(String.format("select time,%s from %s where %s", columnName, tableName, FilterTransformer.toString(filter))); + } catch (Exception e) { + continue; + } resultSets.add(rs); ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, columnName); String typeName = "INT"; @@ -345,11 +352,18 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); } else if (tableSet == null && columnSet != null) { +// ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); while (columnSet.next()) { + Statement stmt = conn.createStatement(); String field = columnSet.getString("COLUMN_NAME"); if (!field.equals("time")) { String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - ResultSet rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, tableName, FilterTransformer.toString(filter))); + ResultSet rs = null; + try { + rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, tableName, FilterTransformer.toString(filter))); + } catch (Exception e) { + continue; + } resultSets.add(rs); fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) @@ -361,10 +375,16 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F String table = tableSet.getString(3);//获取表名称 ResultSet columnSet1 = databaseMetaData.getColumns(null, null, table, null); while (columnSet1.next()) { + Statement stmt = conn.createStatement(); String field = columnSet1.getString("COLUMN_NAME"); if (!field.equals("time")) { + ResultSet rs = null; String typeName = columnSet1.getString("TYPE_NAME");//列字段类型 - ResultSet rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, table, FilterTransformer.toString(filter))); + try { + rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, table, FilterTransformer.toString(filter))); + } catch (Exception e) { + continue; + } resultSets.add(rs); fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) @@ -373,6 +393,8 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F } } } + + } RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields, false); return new TaskExecuteResult(rowStream); @@ -389,6 +411,10 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); for (String path : project.getPatterns()) { + String[] l = path.split("\\."); + if (l.length < 3) { + continue; + } String database_table = path.substring(0, path.lastIndexOf(".")); String dataBaseName = database_table.substring(0, database_table.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String tableName = database_table.substring(database_table.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); @@ -397,8 +423,6 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt if (conn == null) { continue; } - Statement stmt = conn.createStatement(); - DatabaseMetaData databaseMetaData = conn.getMetaData(); ResultSet tableSet = null; ResultSet columnSet = null; @@ -408,18 +432,30 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt columnSet = databaseMetaData.getColumns(null, null, tableName, null); } + + ResultSet columnSet_all = databaseMetaData.getColumns(null, null, tableName, null); + String hv = ""; + while (columnSet_all.next()) { + String columnName2 = columnSet_all.getString("COLUMN_NAME");//获取列名称 + hv = hv + columnName2 + ","; //c1,c2,c3, + } + if (hv.equals("")) { + continue; + } + hv = hv.substring(0, hv.lastIndexOf(",")); + + if (tableSet == null && columnSet == null) { - ResultSet columnSet2 = databaseMetaData.getColumns(null, null, tableName, null); - String hv = ""; - while (columnSet2.next()) { - String columnName2 = columnSet2.getString("COLUMN_NAME");//获取列名称 - hv = hv + columnName2 + ","; //c1,c2,c3, + Statement stmt = conn.createStatement(); + ResultSet rs = null; + try { + rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName)); + } catch (Exception e) { + continue; } - hv = hv.substring(0, hv.lastIndexOf(",")); - ResultSet rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName)); resultSets.add(rs); ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, columnName); - String typeName = "INT"; + String typeName = "TEXT"; if (columnSet_.next()) { typeName = columnSet_.getString("TYPE_NAME");//列字段类型 } @@ -427,18 +463,17 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); } else if (tableSet == null && columnSet != null) { - while (columnSet.next()) { - String hv = ""; - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); - while (columnSet_.next()) { - String columnName2 = columnSet_.getString("COLUMN_NAME");//获取列名称 - hv = hv + columnName2 + ","; //c1,c2,c3, + ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); + while (columnSet_.next()) { + Statement stmt = conn.createStatement(); + String typeName = columnSet_.getString("TYPE_NAME");//列字段类型 + String field = columnSet_.getString("COLUMN_NAME"); + ResultSet rs = null; + try { + rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, tableName)); + } catch (Exception e) { + continue; } - hv = hv.substring(0, hv.lastIndexOf(",")); - - String field = columnSet.getString("COLUMN_NAME"); - ResultSet rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, tableName)); resultSets.add(rs); fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) @@ -447,17 +482,17 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt } else { while (tableSet.next()) { String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet1 = databaseMetaData.getColumns(null, null, table, null); - String hv = ""; - while (columnSet1.next()) { - String columnName1 = columnSet1.getString("COLUMN_NAME");//获取列名称 - hv = hv + columnName1 + ","; //c1,c2,c3, - } - hv = hv.substring(0, hv.lastIndexOf(",")); - while (columnSet1.next()) { - String field = columnSet1.getString("COLUMN_NAME"); - String typeName = columnSet1.getString("TYPE_NAME");//列字段类型 - ResultSet rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, table)); + ResultSet columnSet2 = databaseMetaData.getColumns(null, null, table, null); + while (columnSet2.next()) { + Statement stmt = conn.createStatement(); + String field = columnSet2.getString("COLUMN_NAME"); + String typeName = columnSet2.getString("TYPE_NAME");//列字段类型 + ResultSet rs = null; + try { + rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, table)); + } catch (Exception e) { + continue; + } resultSets.add(rs); fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 5acb195c02..cbe3e0b72e 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -118,7 +118,7 @@ public Row next() throws PhysicalException { } } return new Row(header, timestamp, values); - } catch (SQLException e) { + } catch (Exception e) { logger.info("error:", e); throw new RowFetchException(e); } From a3212198abae0adf110085408da3fc2b10099de8 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Thu, 9 Mar 2023 18:51:36 +0800 Subject: [PATCH 26/94] fix --- .../iginx/postgresql/PostgreSQLStorage.java | 12 ++++++---- .../entity/PostgreSQLQueryRowStream.java | 24 +++++++++++++++++-- .../iginx/session/IoTDBSessionExample.java | 22 ++++++++--------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index f9378276e6..55984fb1bb 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -152,7 +152,7 @@ private Connection getConnection(String dbname, String url) { stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { //database exists - logger.info("database exists!", e); + //logger.info("database exists!", e); } try { if (connectionPoolMap.containsKey(dbname)) { @@ -449,6 +449,7 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt Statement stmt = conn.createStatement(); ResultSet rs = null; try { + //String s = String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName); rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName)); } catch (Exception e) { continue; @@ -459,7 +460,8 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt if (columnSet_.next()) { typeName = columnSet_.getString("TYPE_NAME");//列字段类型 } - fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + fields.add(new Field(dataBaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); } else if (tableSet == null && columnSet != null) { @@ -475,7 +477,8 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt continue; } resultSets.add(rs); - fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + fields.add(new Field(dataBaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); } @@ -494,7 +497,8 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt continue; } resultSets.add(rs); - fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + fields.add(new Field(dataBaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index cbe3e0b72e..147a076df4 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -40,7 +40,17 @@ public PostgreSQLQueryRowStream(List resultSets, List fields, } else { this.currTimestamps[i] = resultSet.getLong(1); } - this.currValues[i] = resultSet.getObject(2); + String typeName = ""; + if (resultSet.getObject(2) != null) { + typeName = resultSet.getObject(2).getClass().getTypeName(); + } else { + typeName = ""; + } + if (typeName.contains("String")) { + this.currValues[i] = resultSet.getObject(2).toString().getBytes(); + } else { + this.currValues[i] = resultSet.getObject(2); + } } } } catch (SQLException e) { @@ -109,7 +119,17 @@ public Row next() throws PhysicalException { } else { this.currTimestamps[i] = resultSet.getLong(1); } - this.currValues[i] = resultSet.getObject(2); + String typeName = ""; + if (resultSet.getObject(2) != null) { + typeName = resultSet.getObject(2).getClass().getTypeName(); + } else { + typeName = "null"; + } + if (typeName.contains("String")) { + this.currValues[i] = resultSet.getObject(2).toString().getBytes(); + } else { + this.currValues[i] = resultSet.getObject(2); + } } else { // 值已经取完 this.currTimestamps[i] = Long.MIN_VALUE; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java index 3763425726..4f0078fdce 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java @@ -51,27 +51,27 @@ public static void main(String[] args) throws SessionException, ExecutionExcepti session.openSession(); // 列式插入对齐数据 - insertColumnRecords(); - // 列式插入非对齐数据 - insertNonAlignedColumnRecords(); - // 行式插入对齐数据 - insertRowRecords(); - // 行式插入非对齐数据 - insertNonAlignedRowRecords(); +// insertColumnRecords(); +// // 列式插入非对齐数据 +// insertNonAlignedColumnRecords(); +// // 行式插入对齐数据 +// insertRowRecords(); +// // 行式插入非对齐数据 +// insertNonAlignedRowRecords(); // 查询序列 showTimeSeries(); // 查询数据 queryData(); // 聚合查询 - aggregateQuery(); +// aggregateQuery(); // Last 查询 lastQuery(); // 降采样聚合查询 - downsampleQuery(); +// downsampleQuery(); // 曲线匹配 - curveMatch(); +// curveMatch(); // 删除数据 - deleteDataInColumns(); +// deleteDataInColumns(); // 再次查询数据 queryData(); // 查看集群信息 From d8b469f492a984acc36572e6c6706227a6eb5ae3 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 10 Mar 2023 10:08:37 +0800 Subject: [PATCH 27/94] fix --- .../postgresql/tools/DataTypeTransformer.java | 6 ++--- .../iginx/session/IoTDBSessionExample.java | 22 +++++++++---------- .../edu/tsinghua/iginx/utils/ByteUtils.java | 15 ++++++++----- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java index ecd5fbed98..1f892eb484 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java @@ -5,9 +5,9 @@ import static cn.edu.tsinghua.iginx.thrift.DataType.*; public class DataTypeTransformer { - + public static DataType fromPostgreSQL(String dataType) { - if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { + if (dataType.contains("int") || dataType.contains("serial") || dataType.contains("timestamp")) { return LONG; } else if (dataType.contains("bool")) { return BOOLEAN; @@ -17,7 +17,7 @@ public static DataType fromPostgreSQL(String dataType) { return BINARY; } } - + public static String toPostgreSQL(DataType dataType) { switch (dataType) { case BOOLEAN: diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java index 4f0078fdce..3763425726 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java @@ -51,27 +51,27 @@ public static void main(String[] args) throws SessionException, ExecutionExcepti session.openSession(); // 列式插入对齐数据 -// insertColumnRecords(); -// // 列式插入非对齐数据 -// insertNonAlignedColumnRecords(); -// // 行式插入对齐数据 -// insertRowRecords(); -// // 行式插入非对齐数据 -// insertNonAlignedRowRecords(); + insertColumnRecords(); + // 列式插入非对齐数据 + insertNonAlignedColumnRecords(); + // 行式插入对齐数据 + insertRowRecords(); + // 行式插入非对齐数据 + insertNonAlignedRowRecords(); // 查询序列 showTimeSeries(); // 查询数据 queryData(); // 聚合查询 -// aggregateQuery(); + aggregateQuery(); // Last 查询 lastQuery(); // 降采样聚合查询 -// downsampleQuery(); + downsampleQuery(); // 曲线匹配 -// curveMatch(); + curveMatch(); // 删除数据 -// deleteDataInColumns(); + deleteDataInColumns(); // 再次查询数据 queryData(); // 查看集群信息 diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java index 6e21cecebf..338038b99e 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java @@ -239,11 +239,16 @@ public static ByteBuffer getRowByteBuffer(Object[] values, List dataTy buffer.putInt((int) value); break; case LONG: - try{ - buffer.putLong((long) value); - } - catch(Exception e){ - buffer.putLong(((Number)value).longValue()); + try { + buffer.putLong((long) value); + } catch (Exception e) { + if (value.getClass().getTypeName().contains("INT")) { //integer + buffer.putLong(((Number) value).longValue()); + } + if (value.getClass().getTypeName().contains("Timestamp")) { //pg timestamp + java.sql.Timestamp ts2 = java.sql.Timestamp.valueOf(value.toString()); + buffer.putLong(ts2.getTime()); + } } break; case FLOAT: From 81d33ba3eaaccf9772eb69b19630787ccb22088a Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 10 Mar 2023 11:54:22 +0800 Subject: [PATCH 28/94] fix --- conf/config.properties | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 6 + .../iginx/integration/SQLSessionIT.java | 206 +++++++++--------- .../iotdb/IoTDBSQLSessionPoolIT.java | 2 +- 4 files changed, 111 insertions(+), 105 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index ca4989c5bb..f691485682 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -18,7 +18,7 @@ password=root #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=true +storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 55984fb1bb..7230dbea05 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -229,6 +229,9 @@ public List getTimeSeries() { while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 String typeName = columnSet.getString("TYPE_NAME");//列字段类型 + if (columnName.equals("time") || columnName.contains("$")) { //tagKV的列不显示 ,time列就是key列，不显示 + continue; + } if (databaseName.startsWith(DATABASE_PREFIX)) { timeseries.add(new Timeseries( tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR @@ -825,6 +828,9 @@ private TaskExecuteResult executeDeleteTask(Connection conn, Delete delete) { try { for (int i = 0; i < delete.getPatterns().size(); i++) { String path = delete.getPatterns().get(i); + if (delete.getTimeRanges() == null) { + continue; + } TimeRange timeRange = delete.getTimeRanges().get(i); String table = path.substring(0, path.lastIndexOf('.')); table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java index 6f1b57053b..db63d3dc12 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java @@ -77,7 +77,7 @@ public static void tearDown() throws SessionException { session.closeSession(); } - @Before + // @Before public void insertData() throws ExecutionException, SessionException { String insertStrPrefix = "INSERT INTO us.d1 (key, s1, s2, s3, s4) values "; @@ -107,7 +107,7 @@ public void insertData() throws ExecutionException, SessionException { } } - @After + // @After public void clearData() throws ExecutionException, SessionException { if (!ifClearData) { return; @@ -173,7 +173,7 @@ public void capacityExpansion() { testTimeRangeQuery(); -// testValueFilter(); + testValueFilter(); testPathFilter(); @@ -2483,9 +2483,9 @@ public void testMultiSubQuery() { executeAndCompare(statement, expected); statement = "SELECT avg_s1, sum_s2 " + - "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + - "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + - "WHERE avg_s1 > 1020 AND sum_s2 < 10800;"; + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "WHERE avg_s1 > 1020 AND sum_s2 < 10800;"; expected = "ResultSets:\n" + "+----+------+------+\n" + "| key|avg_s1|sum_s2|\n" + @@ -2501,10 +2501,10 @@ public void testMultiSubQuery() { executeAndCompare(statement, expected); statement = "SELECT MAX(avg_s1), MIN(sum_s2) " + - "FROM (SELECT avg_s1, sum_s2 " + - "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + - "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + - "WHERE avg_s1 > 1020 AND sum_s2 < 10800);"; + "FROM (SELECT avg_s1, sum_s2 " + + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "WHERE avg_s1 > 1020 AND sum_s2 < 10800);"; expected = "ResultSets:\n" + "+-----------+-----------+\n" + "|max(avg_s1)|min(sum_s2)|\n" + @@ -2514,7 +2514,7 @@ public void testMultiSubQuery() { "Total line number = 1\n"; executeAndCompare(statement, expected); } - + @Test public void testFromSubQuery() { String insert = "INSERT INTO test(key, a.a, a.b) VALUES (1, 1, 1.1), (2, 3, 3.1), (3, 7, 7.1);"; @@ -2523,117 +2523,117 @@ public void testFromSubQuery() { execute(insert); insert = "INSERT INTO test(key, c.a, c.b) VALUES (2, \"eee\", false), (3, \"aaa\", true), (4, \"bbb\", false);"; execute(insert); - + String statement = "SELECT * FROM test.a, (SELECT * FROM test.b);"; String expected = "ResultSets:\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.b|test.b.key|\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "| 1| 1.1| 1| 2| aaa| 1|\n" + - "| 1| 1.1| 1| 4| ccc| 3|\n" + - "| 1| 1.1| 1| 6| eee| 5|\n" + - "| 3| 3.1| 2| 2| aaa| 1|\n" + - "| 3| 3.1| 2| 4| ccc| 3|\n" + - "| 3| 3.1| 2| 6| eee| 5|\n" + - "| 7| 7.1| 3| 2| aaa| 1|\n" + - "| 7| 7.1| 3| 4| ccc| 3|\n" + - "| 7| 7.1| 3| 6| eee| 5|\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "Total line number = 9\n"; + "+--------+--------+----------+--------+--------+----------+\n" + + "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.b|test.b.key|\n" + + "+--------+--------+----------+--------+--------+----------+\n" + + "| 1| 1.1| 1| 2| aaa| 1|\n" + + "| 1| 1.1| 1| 4| ccc| 3|\n" + + "| 1| 1.1| 1| 6| eee| 5|\n" + + "| 3| 3.1| 2| 2| aaa| 1|\n" + + "| 3| 3.1| 2| 4| ccc| 3|\n" + + "| 3| 3.1| 2| 6| eee| 5|\n" + + "| 7| 7.1| 3| 2| aaa| 1|\n" + + "| 7| 7.1| 3| 4| ccc| 3|\n" + + "| 7| 7.1| 3| 6| eee| 5|\n" + + "+--------+--------+----------+--------+--------+----------+\n" + + "Total line number = 9\n"; executeAndCompare(statement, expected); statement = "SELECT * FROM (SELECT * FROM test.b), test.a;"; expected = "ResultSets:\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.b|test.b.key|\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "| 1| 1.1| 1| 2| aaa| 1|\n" + - "| 3| 3.1| 2| 2| aaa| 1|\n" + - "| 7| 7.1| 3| 2| aaa| 1|\n" + - "| 1| 1.1| 1| 4| ccc| 3|\n" + - "| 3| 3.1| 2| 4| ccc| 3|\n" + - "| 7| 7.1| 3| 4| ccc| 3|\n" + - "| 1| 1.1| 1| 6| eee| 5|\n" + - "| 3| 3.1| 2| 6| eee| 5|\n" + - "| 7| 7.1| 3| 6| eee| 5|\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "Total line number = 9\n"; + "+--------+--------+----------+--------+--------+----------+\n" + + "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.b|test.b.key|\n" + + "+--------+--------+----------+--------+--------+----------+\n" + + "| 1| 1.1| 1| 2| aaa| 1|\n" + + "| 3| 3.1| 2| 2| aaa| 1|\n" + + "| 7| 7.1| 3| 2| aaa| 1|\n" + + "| 1| 1.1| 1| 4| ccc| 3|\n" + + "| 3| 3.1| 2| 4| ccc| 3|\n" + + "| 7| 7.1| 3| 4| ccc| 3|\n" + + "| 1| 1.1| 1| 6| eee| 5|\n" + + "| 3| 3.1| 2| 6| eee| 5|\n" + + "| 7| 7.1| 3| 6| eee| 5|\n" + + "+--------+--------+----------+--------+--------+----------+\n" + + "Total line number = 9\n"; executeAndCompare(statement, expected); statement = "SELECT * FROM test.a, (SELECT * FROM test.b WHERE test.b.a < 6) WHERE test.a.a > 1;"; expected = "ResultSets:\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.b|test.b.key|\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "| 3| 3.1| 2| 2| aaa| 1|\n" + - "| 3| 3.1| 2| 4| ccc| 3|\n" + - "| 7| 7.1| 3| 2| aaa| 1|\n" + - "| 7| 7.1| 3| 4| ccc| 3|\n" + - "+--------+--------+----------+--------+--------+----------+\n" + - "Total line number = 4\n"; + "+--------+--------+----------+--------+--------+----------+\n" + + "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.b|test.b.key|\n" + + "+--------+--------+----------+--------+--------+----------+\n" + + "| 3| 3.1| 2| 2| aaa| 1|\n" + + "| 3| 3.1| 2| 4| ccc| 3|\n" + + "| 7| 7.1| 3| 2| aaa| 1|\n" + + "| 7| 7.1| 3| 4| ccc| 3|\n" + + "+--------+--------+----------+--------+--------+----------+\n" + + "Total line number = 4\n"; executeAndCompare(statement, expected); statement = "SELECT * FROM (SELECT test.a.a, test.b.a FROM test.a, test.b WHERE test.b.a < 6 AND test.a.a > 1) AS sub_query;"; expected = "ResultSets:\n" + - "+------------------+------------------+\n" + - "|sub_query.test.a.a|sub_query.test.b.a|\n" + - "+------------------+------------------+\n" + - "| 3| 2|\n" + - "| 3| 4|\n" + - "| 7| 2|\n" + - "| 7| 4|\n" + - "+------------------+------------------+\n" + - "Total line number = 4\n"; + "+------------------+------------------+\n" + + "|sub_query.test.a.a|sub_query.test.b.a|\n" + + "+------------------+------------------+\n" + + "| 3| 2|\n" + + "| 3| 4|\n" + + "| 7| 2|\n" + + "| 7| 4|\n" + + "+------------------+------------------+\n" + + "Total line number = 4\n"; executeAndCompare(statement, expected); statement = "SELECT * FROM test.a INNER JOIN (SELECT a FROM test.b) ON test.a.a < test.b.a"; expected = "ResultSets:\n" + - "+--------+--------+----------+--------+----------+\n" + - "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.key|\n" + - "+--------+--------+----------+--------+----------+\n" + - "| 1| 1.1| 1| 2| 1|\n" + - "| 1| 1.1| 1| 4| 3|\n" + - "| 1| 1.1| 1| 6| 5|\n" + - "| 3| 3.1| 2| 4| 3|\n" + - "| 3| 3.1| 2| 6| 5|\n" + - "+--------+--------+----------+--------+----------+\n" + - "Total line number = 5\n"; + "+--------+--------+----------+--------+----------+\n" + + "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.key|\n" + + "+--------+--------+----------+--------+----------+\n" + + "| 1| 1.1| 1| 2| 1|\n" + + "| 1| 1.1| 1| 4| 3|\n" + + "| 1| 1.1| 1| 6| 5|\n" + + "| 3| 3.1| 2| 4| 3|\n" + + "| 3| 3.1| 2| 6| 5|\n" + + "+--------+--------+----------+--------+----------+\n" + + "Total line number = 5\n"; executeAndCompare(statement, expected); statement = "SELECT * FROM test.a LEFT OUTER JOIN (SELECT a FROM test.b) ON test.a.a < test.b.a"; expected = "ResultSets:\n" + - "+--------+--------+----------+--------+----------+\n" + - "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.key|\n" + - "+--------+--------+----------+--------+----------+\n" + - "| 1| 1.1| 1| 2| 1|\n" + - "| 1| 1.1| 1| 4| 3|\n" + - "| 1| 1.1| 1| 6| 5|\n" + - "| 3| 3.1| 2| 4| 3|\n" + - "| 3| 3.1| 2| 6| 5|\n" + - "| 7| 7.1| 3| null| null|\n" + - "+--------+--------+----------+--------+----------+\n" + - "Total line number = 6\n"; + "+--------+--------+----------+--------+----------+\n" + + "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.key|\n" + + "+--------+--------+----------+--------+----------+\n" + + "| 1| 1.1| 1| 2| 1|\n" + + "| 1| 1.1| 1| 4| 3|\n" + + "| 1| 1.1| 1| 6| 5|\n" + + "| 3| 3.1| 2| 4| 3|\n" + + "| 3| 3.1| 2| 6| 5|\n" + + "| 7| 7.1| 3| null| null|\n" + + "+--------+--------+----------+--------+----------+\n" + + "Total line number = 6\n"; executeAndCompare(statement, expected); statement = "SELECT * FROM test.a, (SELECT a FROM test.b WHERE test.b.a < 6), (SELECT b FROM test.c WHERE test.c.b = false);"; expected = "ResultSets:\n" + - "+--------+--------+----------+--------+----------+--------+----------+\n" + - "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.key|test.c.b|test.c.key|\n" + - "+--------+--------+----------+--------+----------+--------+----------+\n" + - "| 1| 1.1| 1| 2| 1| false| 2|\n" + - "| 1| 1.1| 1| 2| 1| false| 4|\n" + - "| 1| 1.1| 1| 4| 3| false| 2|\n" + - "| 1| 1.1| 1| 4| 3| false| 4|\n" + - "| 3| 3.1| 2| 2| 1| false| 2|\n" + - "| 3| 3.1| 2| 2| 1| false| 4|\n" + - "| 3| 3.1| 2| 4| 3| false| 2|\n" + - "| 3| 3.1| 2| 4| 3| false| 4|\n" + - "| 7| 7.1| 3| 2| 1| false| 2|\n" + - "| 7| 7.1| 3| 2| 1| false| 4|\n" + - "| 7| 7.1| 3| 4| 3| false| 2|\n" + - "| 7| 7.1| 3| 4| 3| false| 4|\n" + - "+--------+--------+----------+--------+----------+--------+----------+\n" + - "Total line number = 12\n"; + "+--------+--------+----------+--------+----------+--------+----------+\n" + + "|test.a.a|test.a.b|test.a.key|test.b.a|test.b.key|test.c.b|test.c.key|\n" + + "+--------+--------+----------+--------+----------+--------+----------+\n" + + "| 1| 1.1| 1| 2| 1| false| 2|\n" + + "| 1| 1.1| 1| 2| 1| false| 4|\n" + + "| 1| 1.1| 1| 4| 3| false| 2|\n" + + "| 1| 1.1| 1| 4| 3| false| 4|\n" + + "| 3| 3.1| 2| 2| 1| false| 2|\n" + + "| 3| 3.1| 2| 2| 1| false| 4|\n" + + "| 3| 3.1| 2| 4| 3| false| 2|\n" + + "| 3| 3.1| 2| 4| 3| false| 4|\n" + + "| 7| 7.1| 3| 2| 1| false| 2|\n" + + "| 7| 7.1| 3| 2| 1| false| 4|\n" + + "| 7| 7.1| 3| 4| 3| false| 2|\n" + + "| 7| 7.1| 3| 4| 3| false| 4|\n" + + "+--------+--------+----------+--------+----------+--------+----------+\n" + + "Total line number = 12\n"; executeAndCompare(statement, expected); } @@ -2809,9 +2809,9 @@ public void testInsertWithSubQuery() { executeAndCompare(query, expected); insert = "INSERT INTO us.d5(key, s1, s2) VALUES (SELECT avg_s1, sum_s2 " + - "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + - "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + - "WHERE avg_s1 > 1020 AND sum_s2 < 10800);"; + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "WHERE avg_s1 > 1020 AND sum_s2 < 10800);"; execute(insert); query = "SELECT s1, s2 FROM us.d5"; @@ -2830,10 +2830,10 @@ public void testInsertWithSubQuery() { executeAndCompare(query, expected); insert = "INSERT INTO us.d6(key, s1, s2) VALUES (SELECT MAX(avg_s1), MIN(sum_s2) " + - "FROM (SELECT avg_s1, sum_s2 " + - "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + - "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + - "WHERE avg_s1 > 1020 AND sum_s2 < 10800));"; + "FROM (SELECT avg_s1, sum_s2 " + + "FROM (SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 " + + "FROM us.d1 OVER (RANGE 10 IN [1000, 1100))) " + + "WHERE avg_s1 > 1020 AND sum_s2 < 10800));"; execute(insert); query = "SELECT s1, s2 FROM us.d6"; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/iotdb/IoTDBSQLSessionPoolIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/iotdb/IoTDBSQLSessionPoolIT.java index a17e8f6dfa..cb1a2de308 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/iotdb/IoTDBSQLSessionPoolIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/iotdb/IoTDBSQLSessionPoolIT.java @@ -6,7 +6,7 @@ public class IoTDBSQLSessionPoolIT extends SQLSessionPoolIT { public IoTDBSQLSessionPoolIT() { super(); - this.isAbleToDelete = true; + this.isAbleToDelete = false; this.isAbleToShowTimeSeries = true; } } From f8e17bf9e104bfc85ba8e3f85abb3fa60976c9de Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 10 Mar 2023 12:21:07 +0800 Subject: [PATCH 29/94] Fix delete partially --- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 62 +++++++++++++------ 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 2ac494f5e2..984551b044 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -674,7 +674,7 @@ private TaskExecuteResult executeDeleteTask(String storageUnit, Delete delete) { deletedPaths = determineDeletePathList(storageUnit, delete); } catch (PhysicalException e) { logger.warn("encounter error when delete path: " + e.getMessage()); - return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("execute delete path task in iotdb11 failure", e)); + return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("execute delete path task in iotdb12 failure", e)); } for (String path: deletedPaths) { try { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 55984fb1bb..a411281fad 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -38,6 +38,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.postgresql.entity.PostgreSQLQueryRowStream; @@ -45,6 +46,7 @@ import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; +import cn.edu.tsinghua.iginx.utils.StringUtils; import org.postgresql.ds.PGConnectionPoolDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +56,7 @@ import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import java.util.stream.Collectors; public class PostgreSQLStorage implements IStorage { @@ -197,7 +200,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { return executeInsertTask(conn, insert); } else if (op.getType() == OperatorType.Delete) { Delete delete = (Delete) op; - return executeDeleteTask(conn, delete); + return executeDeleteTask(conn, storageUnit, delete); } return new TaskExecuteResult( new NonExecutablePhysicalTaskException("unsupported physical task in postgresql ")); @@ -795,6 +798,7 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView firstRound = false; cnt += size; } + conn.close(); } catch (SQLException e) { logger.info("error", e); return e; @@ -821,28 +825,50 @@ private void executeBatch(Statement stmt, Map> stmt.executeBatch(); } - private TaskExecuteResult executeDeleteTask(Connection conn, Delete delete) { + private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { try { - for (int i = 0; i < delete.getPatterns().size(); i++) { - String path = delete.getPatterns().get(i); - TimeRange timeRange = delete.getTimeRanges().get(i); - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - // 查询序列类型 - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (columnSet.next()) { - String statement = String.format("delete from %s where (time>%d and time<%d)", table, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = conn.createStatement(); - stmt.execute(statement); + Statement stmt = conn.createStatement(); + List paths = delete.getPatterns(); + String table; + String field; + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet columnSet; + if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { + if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { + conn.close(); + Connection postgresConn = getConnection("postgres", getUrl("postgres")); // 正在使用的数据库无法被删除，因为需要切换到名为postgres的默认数据库 + if (postgresConn != null) { + stmt = postgresConn.createStatement(); + stmt.execute(String.format("drop database %s", storageUnit)); // 删除数据库 + postgresConn.close(); + return new TaskExecuteResult(null, null); + } else { + return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("cannot connect to database: postgres", new SQLException())); + } + } else { + for (String path : paths) { + table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + field = path.substring(path.lastIndexOf('.') + 1); + stmt.execute(String.format("alter table %s drop column if exists %s", table, field)); // 删除列 + } + } + } else { + for (int i = 0; i < paths.size(); i++) { + String path = paths.get(i); + TimeRange timeRange = delete.getTimeRanges().get(i); + table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + field = path.substring(path.lastIndexOf('.') + 1); + columnSet = databaseMetaData.getColumns(null, "%", table, field); + if (columnSet.next()) { + stmt.execute(String.format("update %s set %s = null where (time > %d and time < %d)", table, field, + timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP))); // 将目标列的目标范围的值置为空 + } } } + conn.close(); return new TaskExecuteResult(null, null); } catch (SQLException e) { - logger.info("error:", e); + logger.error(e.getMessage()); return new TaskExecuteResult( new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", e)); } From 7e9fe66f0a15b49c93f3ed58ee4b629e6332c33c Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 10 Mar 2023 12:26:49 +0800 Subject: [PATCH 30/94] fix --- .../edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java | 2 +- .../cn/edu/tsinghua/iginx/integration/SQLSessionIT.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 2195c88117..1dd3f1f182 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -232,7 +232,7 @@ public List getTimeSeries() { while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - if (columnName.equals("time") || columnName.contains("$")) { //tagKV的列不显示 ,time列就是key列，不显示 + if (columnName.equals("time") || columnName.contains("$")) { //tagKV 和 time 列不显示 continue; } if (databaseName.startsWith(DATABASE_PREFIX)) { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java index db63d3dc12..e3404ede87 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java @@ -205,11 +205,11 @@ public void capacityExpansion() { testErrorClause(); - testDelete(); +// testDelete(); - testMultiRangeDelete(); - - testCrossRangeDelete(); +// testMultiRangeDelete(); +// +// testCrossRangeDelete(); } @Test From ad467681997159bdc4d7ec3f23d2d8f67cb5d39d Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 10 Mar 2023 17:04:39 +0800 Subject: [PATCH 31/94] Fix query partially --- .../iginx/postgresql/PostgreSQLStorage.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 1dd3f1f182..3bb9da233a 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -249,6 +249,7 @@ public List getTimeSeries() { } } } + conn1.close(); } catch (Exception e) { logger.info("error:", e); } @@ -325,21 +326,23 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F try { List resultSets = new ArrayList<>(); List fields = new ArrayList<>(); - for (String path : project.getPatterns()) { - String tableName = path.substring(0, path.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String columnName = path.substring(path.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - ResultSet columnSet = null; + DatabaseMetaData databaseMetaData = conn.getMetaData();; + ResultSet tableSet = null; + ResultSet columnSet = null; + String tableName = ""; + String columnName = ""; - if (path.equals("*.*")) { + for (String path : project.getPatterns()) { + if (path.equals("*") || path.equals("*.*")) { tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else if (columnName.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, tableName, null); } else { + tableName = path.substring(0, path.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + columnName = path.substring(path.lastIndexOf(".") + 1); + if (columnName.equals("*")) { + columnSet = databaseMetaData.getColumns(null, null, tableName, null); + } } - if (tableSet == null && columnSet == null) { Statement stmt = conn.createStatement(); ResultSet rs = null; @@ -357,7 +360,7 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) , DataTypeTransformer.fromPostgreSQL(typeName))); - } else if (tableSet == null && columnSet != null) { + } else if (tableSet == null) { // ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); while (columnSet.next()) { Statement stmt = conn.createStatement(); @@ -399,10 +402,9 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F } } } - - } RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields, false); + conn.close(); return new TaskExecuteResult(rowStream); } catch (SQLException e) { logger.info("error: ", e); @@ -828,6 +830,8 @@ private void executeBatch(Statement stmt, Map> stmt.executeBatch(); } + // TODO 处理带*路径 + // TODO 处理tagKV private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { try { Statement stmt = conn.createStatement(); From 6f8e72203bf718f6cbc6f38ac333c2e40e61e730 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Sun, 12 Mar 2023 19:31:22 +0800 Subject: [PATCH 32/94] fix --- .../java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java index e3404ede87..e0687d1873 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/SQLSessionIT.java @@ -77,7 +77,7 @@ public static void tearDown() throws SessionException { session.closeSession(); } - // @Before + @Before public void insertData() throws ExecutionException, SessionException { String insertStrPrefix = "INSERT INTO us.d1 (key, s1, s2, s3, s4) values "; From da8b6ea9519c3279e4e2123f0aa435f91363516c Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Sun, 12 Mar 2023 21:11:41 +0800 Subject: [PATCH 33/94] fix query --- .../postgresql/entity/PostgreSQLQueryRowStream.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 147a076df4..9e629e6747 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -137,6 +137,15 @@ public Row next() throws PhysicalException { } } } + boolean f = false; //判断是否是全为空的行 + for (Object i : values) { + if (i != null) { + f = true; + } + } + if (!f && timestamp == 0) { + return next(); + } return new Row(header, timestamp, values); } catch (Exception e) { logger.info("error:", e); From 46680e8e5472ae3308e2423bb8705a90dfa2bc45 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Sun, 12 Mar 2023 22:01:15 +0800 Subject: [PATCH 34/94] fix query --- .../entity/PostgreSQLQueryRowStream.java | 55 ++++++++++++++++--- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 9e629e6747..9bc3c07316 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -78,6 +78,52 @@ public void close() { @Override public boolean hasNext() { + boolean f = false; //判断是否是全为空的行 + for (Object i : currValues) { + if (i != null) { + f = true; + } + } + long timestamp = Long.MAX_VALUE; + for (long currTimestamp : this.currTimestamps) { + if (currTimestamp != Long.MIN_VALUE) { + timestamp = Math.min(timestamp, currTimestamp); + } + } + if (!f && timestamp == 0) { + try { + for (int i = 0; i < this.currTimestamps.length; i++) { + ResultSet resultSet = this.resultSets.get(i); + if (resultSet.next()) { + if (isdummy) { + this.currTimestamps[i] = toHash(resultSet.getString(1)); + } else { + this.currTimestamps[i] = resultSet.getLong(1); + } + String typeName = ""; + if (resultSet.getObject(2) != null) { + typeName = resultSet.getObject(2).getClass().getTypeName(); + } else { + typeName = "null"; + } + if (typeName.contains("String")) { + this.currValues[i] = resultSet.getObject(2).toString().getBytes(); + } else { + this.currValues[i] = resultSet.getObject(2); + } + } else { + // 值已经取完 + this.currTimestamps[i] = Long.MIN_VALUE; + this.currValues[i] = null; + } + } + } catch (Exception e) { + //error + logger.error("error in postgresqlrowstream "); + } + return hasNext(); + } + for (long currTimestamp : this.currTimestamps) { if (currTimestamp != Long.MIN_VALUE) { return true; @@ -137,15 +183,6 @@ public Row next() throws PhysicalException { } } } - boolean f = false; //判断是否是全为空的行 - for (Object i : values) { - if (i != null) { - f = true; - } - } - if (!f && timestamp == 0) { - return next(); - } return new Row(header, timestamp, values); } catch (Exception e) { logger.info("error:", e); From ac158896cac7f1f6ad4c937918b2bf5dbec5e104 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Mon, 13 Mar 2023 17:02:30 +0800 Subject: [PATCH 35/94] Update insert and get time series with tagkv --- .../iginx/postgresql/PostgreSQLStorage.java | 164 ++++++++++-------- .../iginx/postgresql/tools/Constants.java | 8 + .../iginx/postgresql/tools/TagKVUtils.java | 44 +++++ .../iginx/session/IoTDBSessionExample.java | 22 +-- .../integration/func/sql/SQLSessionIT.java | 50 +++--- .../iginx/integration/func/tag/TagIT.java | 2 +- 6 files changed, 180 insertions(+), 110 deletions(-) create mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java create mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 3bb9da233a..80f3c3996d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -24,6 +24,7 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.StorageInitializationException; import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Timeseries; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.TimeRange; @@ -54,10 +55,13 @@ import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.*; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; -import java.util.stream.Collectors; + +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; +import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; public class PostgreSQLStorage implements IStorage { @@ -89,10 +93,6 @@ public class PostgreSQLStorage implements IStorage { private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - private static final String IGINX_SEPARATOR = "."; - - private static final String POSTGRESQL_SEPARATOR = "$"; - private static final String DATABASE_PREFIX = "unit"; private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; @@ -154,9 +154,9 @@ private Connection getConnection(String dbname, String url) { Statement stmt = connection.createStatement(); stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { - //database exists - //logger.info("database exists!", e); + logger.info("database {} exists!", dbname); } + try { if (connectionPoolMap.containsKey(dbname)) { return connectionPoolMap.get(dbname).getConnection(); @@ -166,7 +166,7 @@ private Connection getConnection(String dbname, String url) { connectionPoolMap.put(dbname, connectionPool); return connectionPool.getConnection(); } catch (SQLException e) { - logger.error("cannot get connection for database: {}", e); + logger.error("cannot get connection for database {}: {}", dbname, e.getMessage()); return null; } } @@ -183,6 +183,10 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { String storageUnit = task.getStorageUnit(); boolean isDummyStorageUnit = task.isDummyStorageUnit(); Connection conn = getConnection(storageUnit, getUrl(storageUnit)); + if (conn == null) { + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException(String.format("cannot connect to storage unit %s", storageUnit))); + } if (op.getType() == OperatorType.Project) { Project project = (Project) op; @@ -216,42 +220,48 @@ public List getTimeSeries() { while (databaseSet.next()) { try { String databaseName = databaseSet.getString(1);//获取数据库名称 - if (extraParams.get("has_data") != null && extraParams.get("has_data").equals("true")) { - } else { - if (databaseName.startsWith("unit")) { - } else { - continue; - } + if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { + continue; } - Connection conn1 = getConnection(databaseName, getUrl(databaseName)); - DatabaseMetaData databaseMetaData = conn1.getMetaData(); + + Connection conn = getConnection(databaseName, getUrl(databaseName)); + if (conn == null) { + logger.error("cannot get connection for database {}!", databaseName); + return timeseries; + } + DatabaseMetaData databaseMetaData = conn.getMetaData(); ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 + String tableName = tableSet.getString(3); // 获取表名称 ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - if (columnName.equals("time") || columnName.contains("$")) { //tagKV 和 time 列不显示 + String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 + String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 + if (columnName.equals("time")) { // time 列不显示 continue; } + Pair> nameAndTags = splitFullName(columnName); if (databaseName.startsWith(DATABASE_PREFIX)) { timeseries.add(new Timeseries( - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + nameAndTags.k, + DataTypeTransformer.fromPostgreSQL(typeName), + nameAndTags.v) + ); } else { timeseries.add(new Timeseries( - databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName))); + databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + nameAndTags.k, + DataTypeTransformer.fromPostgreSQL(typeName), + nameAndTags.v) + ); } } } - conn1.close(); - } catch (Exception e) { - logger.info("error:", e); + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); } } } catch (SQLException e) { @@ -559,30 +569,16 @@ private void createOrAlterTables(Connection conn, List paths, List tagEntry : tags.entrySet()) { - columnKeys.append(field).append(POSTGRESQL_SEPARATOR).append(tagEntry.getKey()).append(", "); - columnValues = columnValues.stream().map(x -> x + tagEntry.getValue() + ", ").collect(Collectors.toList()); - } - } - columnKeys.append(field).append(", "); + columnKeys.append(toFullName(field, tags)).append(", "); } if (i - cnt < columnValues.size()) { @@ -765,13 +755,7 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView pathIndexToBitmapIndex.put(i, index); if (firstRound) { - columnKeys.append(field).append(", "); - if (tags != null && !tags.isEmpty()) { - for (Entry tagEntry : tags.entrySet()) { - columnKeys.append(field).append(POSTGRESQL_SEPARATOR).append(tagEntry.getKey()).append(", "); - columnValues = columnValues.stream().map(x -> x + tagEntry.getValue() + ", ").collect(Collectors.toList()); - } - } + columnKeys.append(toFullName(field, tags)).append(", "); } tableToColumnEntries.put(table, new Pair<>(columnKeys.toString(), columnValues)); @@ -830,20 +814,21 @@ private void executeBatch(Statement stmt, Map> stmt.executeBatch(); } - // TODO 处理带*路径 - // TODO 处理tagKV private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { try { Statement stmt = conn.createStatement(); List paths = delete.getPatterns(); + List deletedPaths; String table; String field; DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet; ResultSet columnSet; + if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { conn.close(); - Connection postgresConn = getConnection("postgres", getUrl("postgres")); // 正在使用的数据库无法被删除，因为需要切换到名为postgres的默认数据库 + Connection postgresConn = getConnection("postgres", getUrl("postgres")); // 正在使用的数据库无法被删除，因此需要切换到名为postgres的默认数据库 if (postgresConn != null) { stmt = postgresConn.createStatement(); stmt.execute(String.format("drop database %s", storageUnit)); // 删除数据库 @@ -853,15 +838,20 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("cannot connect to database: postgres", new SQLException())); } } else { - for (String path : paths) { + deletedPaths = determineDeletedPathList(paths, delete.getTagFilter()); + for (String path : deletedPaths) { table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); field = path.substring(path.lastIndexOf('.') + 1); - stmt.execute(String.format("alter table %s drop column if exists %s", table, field)); // 删除列 + tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + if (tableSet.next()) { + stmt.execute(String.format("alter table %s drop column if exists %s", table, field)); // 删除列 + } } } } else { - for (int i = 0; i < paths.size(); i++) { - String path = paths.get(i); + deletedPaths = determineDeletedPathList(paths, delete.getTagFilter()); + for (int i = 0; i < deletedPaths.size(); i++) { + String path = deletedPaths.get(i); TimeRange timeRange = delete.getTimeRanges().get(i); table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); field = path.substring(path.lastIndexOf('.') + 1); @@ -881,6 +871,34 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, } } + private List determineDeletedPathList(List paths, TagFilter tagFilter) { + List timeSeries = getTimeSeries(); + List deletedPaths = new ArrayList<>(); + + if (tagFilter != null) { + for (Timeseries ts: timeSeries) { + for (String path : paths) { + if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath()) && + TagKVUtils.match(ts.getTags(), tagFilter)) { + deletedPaths.add(ts.getPath()); + break; + } + } + } + } else { + for (Timeseries ts: timeSeries) { + for (String path : paths) { + if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath())) { + deletedPaths.add(ts.getPath()); + break; + } + } + } + } + + return deletedPaths; + } + @Override public void release() throws PhysicalException { try { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java new file mode 100644 index 0000000000..5cb47b03b6 --- /dev/null +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -0,0 +1,8 @@ +package cn.edu.tsinghua.iginx.postgresql.tools; + +public class Constants { + + public static final String IGINX_SEPARATOR = "."; + + public static final String POSTGRESQL_SEPARATOR = "$"; +} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java new file mode 100644 index 0000000000..a36ca9ecff --- /dev/null +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -0,0 +1,44 @@ +package cn.edu.tsinghua.iginx.postgresql.tools; + +import cn.edu.tsinghua.iginx.utils.Pair; + +import java.util.*; +import java.util.stream.Collectors; + +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; + +// TODO 目前未考虑 annotation +public class TagKVUtils { + + public static Pair> splitFullName(String fullName) { + if (!fullName.contains(POSTGRESQL_SEPARATOR)) { + return new Pair<>(fullName, null); + } + + String[] parts = fullName.split("\\" + POSTGRESQL_SEPARATOR); + String name = parts[0]; + + Map tags = new HashMap<>(); + for (int i = 1; i < parts.length; i++) { + if (i % 2 != 0) { + continue; + } + String tagKey = parts[i - 1]; + String tagValue = parts[i]; + tags.put(tagKey, tagValue); + } + return new Pair<>(name, tags); + } + + public static String toFullName(String name, Map tags) { + if (tags != null && !tags.isEmpty()) { + TreeMap sortedTags = new TreeMap<>(tags); + StringBuilder pathBuilder = new StringBuilder(); + sortedTags.forEach((tagKey, tagValue) -> { + pathBuilder.append(POSTGRESQL_SEPARATOR).append(tagKey).append(POSTGRESQL_SEPARATOR).append(tagValue); + }); + name += pathBuilder.toString(); + } + return name; + } +} diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java index 3763425726..86aa42810a 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java @@ -55,27 +55,27 @@ public static void main(String[] args) throws SessionException, ExecutionExcepti // 列式插入非对齐数据 insertNonAlignedColumnRecords(); // 行式插入对齐数据 - insertRowRecords(); +// insertRowRecords(); // 行式插入非对齐数据 - insertNonAlignedRowRecords(); +// insertNonAlignedRowRecords(); // 查询序列 - showTimeSeries(); +// showTimeSeries(); // 查询数据 - queryData(); +// queryData(); // 聚合查询 - aggregateQuery(); +// aggregateQuery(); // Last 查询 - lastQuery(); +// lastQuery(); // 降采样聚合查询 - downsampleQuery(); +// downsampleQuery(); // 曲线匹配 - curveMatch(); +// curveMatch(); // 删除数据 - deleteDataInColumns(); +// deleteDataInColumns(); // 再次查询数据 - queryData(); +// queryData(); // 查看集群信息 - showClusterInfo(); +// showClusterInfo(); // 关闭 Session session.closeSession(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index f879bdc920..017082b3d4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -128,7 +128,7 @@ public void insertData() throws ExecutionException, SessionException { } } - // @After + @After public void clearData() throws ExecutionException, SessionException { String clearData = "CLEAR DATA;"; @@ -201,7 +201,7 @@ public void testCountPath() { executeAndCompare(statement, expected); } - @Test +// @Test public void testCountPoints() { if (ifScaleOutIn) return; String statement = "COUNT POINTS;"; @@ -209,7 +209,7 @@ public void testCountPoints() { executeAndCompare(statement, expected); } - @Test +// @Test public void testShowTimeSeries() { if (!isAbleToShowTimeSeries || ifScaleOutIn) { return; @@ -303,14 +303,14 @@ public void testShowTimeSeries() { executeAndCompare(statement, expected); } - @Test +// @Test public void testShowReplicaNum() { String statement = "SHOW REPLICA NUMBER;"; String expected = "Replica num: 1\n"; executeAndCompare(statement, expected); } - @Test +// @Test public void testTimeRangeQuery() { String statement = "SELECT s1 FROM us.d1 WHERE key > 100 AND key < 120;"; String expected = "ResultSets:\n" + @@ -431,7 +431,7 @@ public void testValueFilter() { executeAndCompare(query, expected); } - @Test +// @Test public void testPathFilter() { String insert = "INSERT INTO us.d9(key, a, b) VALUES (1, 1, 9), (2, 2, 8), (3, 3, 7), (4, 4, 6), (5, 5, 5), (6, 6, 4), (7, 7, 3), (8, 8, 2), (9, 9, 1);"; execute(insert); @@ -518,7 +518,7 @@ public void testPathFilter() { executeAndCompare(query, expected); } - @Test +// @Test public void testLimitAndOffsetQuery() { String statement = "SELECT s1 FROM us.d1 WHERE key > 0 AND key < 10000 limit 10;"; String expected = "ResultSets:\n" + @@ -559,7 +559,7 @@ public void testLimitAndOffsetQuery() { executeAndCompare(statement, expected); } - @Test +// @Test public void testOrderByQuery() { String insert = "INSERT INTO us.d2 (key, s1, s2, s3) values " + "(1, \"apple\", 871, 232.1), (2, \"peach\", 123, 132.5), (3, \"banana\", 356, 317.8)," @@ -682,7 +682,7 @@ public void testOrderByQuery() { executeAndCompare(orderByQuery, expected); } - @Test +// @Test public void testFirstLastQuery() { String statement = "SELECT FIRST(s2) FROM us.d1 WHERE key > 0;"; String expected = "ResultSets:\n" + @@ -805,7 +805,7 @@ public void testFirstLastQuery() { executeAndCompare(statement, expected); } - @Test +// @Test public void testAggregateQuery() { String statement = "SELECT %s(s1), %s(s2) FROM us.d1 WHERE key > 0 AND key < 1000;"; List funcTypeList = Arrays.asList( @@ -869,7 +869,7 @@ public void testAggregateQuery() { } } - @Test +// @Test public void testDownSampleQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000));"; List funcTypeList = Arrays.asList( @@ -996,7 +996,7 @@ public void testDownSampleQuery() { } } - @Test +// @Test public void testRangeDownSampleQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 600 AND s1 <= 900 OVER (RANGE 100 IN (0, 1000));"; List funcTypeList = Arrays.asList( @@ -1074,7 +1074,7 @@ public void testRangeDownSampleQuery() { } } - @Test +// @Test public void testSlideWindowByTimeQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000) STEP 50);"; List funcTypeList = Arrays.asList( @@ -1264,7 +1264,7 @@ public void testSlideWindowByTimeQuery() { } } - @Test +// @Test public void testRangeSlideWindowByTimeQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER (RANGE 100 IN (0, 1000) STEP 50);"; List funcTypeList = Arrays.asList( @@ -1627,7 +1627,7 @@ public void testCrossRangeDelete() { executeAndCompare(queryOverDeleteRange, expected); } - @Test +// @Test public void testGroupBy() { String insert = "insert into test(key, a, b, c, d) values (1, 3, 2, 3.1, \"val1\"), (2, 1, 3, 2.1, \"val2\"), (3, 2, 2, 1.1, \"val5\"), (4, 3, 2, 2.1, \"val2\"), (5, 1, 2, 3.1, \"val1\"), (6, 2, 2, 5.1, \"val3\");"; execute(insert); @@ -1748,7 +1748,7 @@ public void testGroupBy() { executeAndCompare(query, expected); } - @Test +// @Test public void testGroupByAndHaving() { String insert = "insert into test(key, a, b, c, d) values (1, 3, 2, 3.1, \"val1\"), (2, 1, 3, 2.1, \"val2\"), (3, 2, 2, 1.1, \"val5\"), (4, 3, 2, 2.1, \"val2\"), (5, 1, 2, 3.1, \"val1\"), (6, 2, 2, 5.1, \"val3\");"; execute(insert); @@ -2176,7 +2176,7 @@ public void testMultiJoin() { executeAndCompare(statement, expected); } - @Test +// @Test public void testBasicArithmeticExpr() { String insert = "INSERT INTO us.d3 (key, s1, s2, s3) values " + "(1, 1, 6, 1.5), (2, 2, 5, 2.5), (3, 3, 4, 3.5), (4, 4, 3, 4.5), (5, 5, 2, 5.5), (6, 6, 1, 6.5);"; @@ -2263,7 +2263,7 @@ public void testBasicArithmeticExpr() { executeAndCompare(statement, expected); } - @Test +// @Test public void testComplexArithmeticExpr() { String insert = "INSERT INTO us.d3 (key, s1, s2, s3) values " + "(1, 1, 6, 1.5), (2, 2, 5, 2.5), (3, 3, 4, 3.5), (4, 4, 3, 4.5), (5, 5, 2, 5.5), (6, 6, 1, 6.5);"; @@ -2350,7 +2350,7 @@ public void testComplexArithmeticExpr() { executeAndCompare(statement, expected); } - @Test +// @Test public void testAlias() { // time series alias String statement = "SELECT s1 AS rename_series, s2 FROM us.d1 WHERE s1 >= 1000 AND s1 < 1010;"; @@ -2413,7 +2413,7 @@ public void testAlias() { executeAndCompare(statement, expected); } - @Test +// @Test public void testAggregateSubQuery() { String statement = "SELECT %s_s1 FROM (SELECT %s(s1) AS %s_s1 FROM us.d1 OVER(RANGE 60 IN [1000, 1600)));"; List funcTypeList = Arrays.asList( @@ -2541,7 +2541,7 @@ public void testAggregateSubQuery() { } } - @Test +// @Test public void testValueFilterSubQuery() { String statement = "SELECT ts FROM (SELECT s1 AS ts FROM us.d1 WHERE us.d1.s1 >= 1000 AND us.d1.s1 < 1010);"; String expected = "ResultSets:\n" + @@ -2588,7 +2588,7 @@ public void testValueFilterSubQuery() { executeAndCompare(statement, expected); } - @Test +// @Test public void testMultiSubQuery() { String statement = "SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 FROM us.d1 OVER (RANGE 10 IN [1000, 1100));"; String expected = "ResultSets:\n" + @@ -2642,7 +2642,7 @@ public void testMultiSubQuery() { executeAndCompare(statement, expected); } - @Test +// @Test public void testFromSubQuery() { String insert = "INSERT INTO test(key, a.a, a.b) VALUES (1, 1, 1.1), (2, 3, 3.1), (3, 7, 7.1);"; execute(insert); @@ -3304,7 +3304,7 @@ public void testErrorClause() { executeAndCompareErrMsg(errClause, "Group by can not use SetToSet and RowToRow functions."); } - @Test +// @Test public void testExplain() { if (ifScaleOutIn) return; String explain = "explain select max(s2), min(s1) from us.d1;"; @@ -3338,7 +3338,7 @@ public void testExplain() { executeAndCompare(explain, expected); } - @Test +// @Test public void testDeleteTimeSeries() { if (!isAbleToDelete || ifScaleOutIn) { return; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java index 2dae680371..8107b42c09 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java @@ -34,7 +34,7 @@ public TagIT() throws IOException { @BeforeClass public static void setUp() throws SessionException { - ifClearData = true; + ifClearData = false; session = new Session("127.0.0.1", 6888, "root", "root"); session.openSession(); } From 067c720389e42dddfcadd7859b6faf78ecf78cf9 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 16 Mar 2023 09:42:39 +0800 Subject: [PATCH 36/94] Update all kinds of operations without expansion --- .../iginx/postgresql/PostgreSQLStorage.java | 286 +++++++++-------- .../entity/PostgreSQLQueryRowStream.java | 297 +++++++++++------- .../postgresql/tools/FilterTransformer.java | 5 +- .../iginx/postgresql/tools/TagKVUtils.java | 2 +- .../integration/func/sql/SQLSessionIT.java | 56 ++-- 5 files changed, 366 insertions(+), 280 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 80f3c3996d..f175e9da86 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -28,6 +28,7 @@ import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.TimeRange; +import cn.edu.tsinghua.iginx.engine.shared.data.read.ClearEmptyRowStreamWrapper; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; @@ -62,6 +63,7 @@ import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; +import static java.sql.ResultSet.*; public class PostgreSQLStorage implements IStorage { @@ -95,7 +97,7 @@ public class PostgreSQLStorage implements IStorage { private static final String DATABASE_PREFIX = "unit"; - private static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + private static final long MAX_TIMESTAMP = Long.MAX_VALUE; private final StorageEngineMeta meta; @@ -154,7 +156,7 @@ private Connection getConnection(String dbname, String url) { Statement stmt = connection.createStatement(); stmt.execute(String.format("create database %s", dbname)); } catch (SQLException e) { - logger.info("database {} exists!", dbname); +// logger.info("database {} exists!", dbname); } try { @@ -331,99 +333,94 @@ public Pair getBoundaryOfStorage(String prefix) t new TimeInterval(minTime, maxTime + 1)); } + private Map splitAndMergeQueryPatterns(Connection conn, List patterns) throws SQLException { + // table name -> column names + // 1 -> n + Map tableNameToColumnNames = new HashMap<>(); + String tableName; + String columnNames; - private TaskExecuteResult executeProjectTask(Connection conn, Project project, Filter filter) { - try { - List resultSets = new ArrayList<>(); - List fields = new ArrayList<>(); - DatabaseMetaData databaseMetaData = conn.getMetaData();; - ResultSet tableSet = null; - ResultSet columnSet = null; - String tableName = ""; - String columnName = ""; + for (String pattern : patterns) { - for (String path : project.getPatterns()) { - if (path.equals("*") || path.equals("*.*")) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else { - tableName = path.substring(0, path.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - columnName = path.substring(path.lastIndexOf(".") + 1); - if (columnName.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, tableName, null); + if (pattern.equals("*") || pattern.equals("*.*")) { + tableName = "%"; + columnNames = "%"; + } else { + tableName = pattern.substring(0, pattern.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + columnNames = pattern.substring(pattern.lastIndexOf(".") + 1); + boolean columnEqualsStar = columnNames.equals("*"); + boolean tableContainsStar = tableName.contains("*"); + if (columnEqualsStar || tableContainsStar) { + tableName = tableName.replace('*', '%'); + if (columnEqualsStar) { + columnNames = "%"; + if (!tableName.endsWith("%")) { + tableName += "%"; + } } } + } - if (tableSet == null && columnSet == null) { - Statement stmt = conn.createStatement(); - ResultSet rs = null; - try { - rs = stmt.executeQuery(String.format("select time,%s from %s where %s", columnName, tableName, FilterTransformer.toString(filter))); - } catch (Exception e) { - continue; - } - resultSets.add(rs); - ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, columnName); - String typeName = "INT"; - if (columnSet_.next()) { - typeName = columnSet_.getString("TYPE_NAME");//列字段类型 - } - fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - } else if (tableSet == null) { -// ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); - while (columnSet.next()) { - Statement stmt = conn.createStatement(); - String field = columnSet.getString("COLUMN_NAME"); - if (!field.equals("time")) { - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - ResultSet rs = null; - try { - rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, tableName, FilterTransformer.toString(filter))); - } catch (Exception e) { - continue; - } - resultSets.add(rs); - fields.add(new Field(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - } - } - } else { - while (tableSet.next()) { - String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet1 = databaseMetaData.getColumns(null, null, table, null); - while (columnSet1.next()) { - Statement stmt = conn.createStatement(); - String field = columnSet1.getString("COLUMN_NAME"); - if (!field.equals("time")) { - ResultSet rs = null; - String typeName = columnSet1.getString("TYPE_NAME");//列字段类型 - try { - rs = stmt.executeQuery(String.format("select time,%s from %s where %s", field, table, FilterTransformer.toString(filter))); - } catch (Exception e) { - continue; - } - resultSets.add(rs); - fields.add(new Field(table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - } - } - } + if (!columnNames.endsWith("%")) { + columnNames += "%"; // 匹配 tagKV + } + ResultSet rs = conn.getMetaData().getColumns(null, null, tableName, columnNames); + while (rs.next()) { + tableName = rs.getString("TABLE_NAME"); + columnNames = rs.getString("COLUMN_NAME"); + if (columnNames.equals("time")) { + continue; + } + if (tableNameToColumnNames.containsKey(tableName)) { + columnNames = tableNameToColumnNames.get(tableName) + ", " + columnNames; + } + tableNameToColumnNames.put(tableName, columnNames); + } + rs.close(); + } + + return tableNameToColumnNames; + } + + private TaskExecuteResult executeProjectTask(Connection conn, Project project, Filter filter) { + try { + List resultSets = new ArrayList<>(); + ResultSet rs; + Statement stmt; + + Map tableNameToColumnNames = splitAndMergeQueryPatterns(conn, project.getPatterns()); + for (Map.Entry entry : tableNameToColumnNames.entrySet()) { + String tableName = entry.getKey(); + String columnNames = entry.getValue(); + StringBuilder statement = new StringBuilder(); + try { + stmt = conn.createStatement(); + statement.append("select time, "); + statement.append(columnNames); + statement.append(" from "); + statement.append(tableName); + statement.append(" where "); + statement.append(FilterTransformer.toString(filter)); + rs = stmt.executeQuery(statement.toString()); + logger.info("[Query] execute query: {}", statement); + } catch (SQLException e) { + logger.error("meet error when executing query {}: {}", statement, e.getMessage()); + continue; } + resultSets.add(rs); } - RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields, false); + + RowStream rowStream = new ClearEmptyRowStreamWrapper( + new PostgreSQLQueryRowStream(resultSets, false, project.getTagFilter())); conn.close(); return new TaskExecuteResult(rowStream); } catch (SQLException e) { - logger.info("error: ", e); + logger.error(e.getMessage()); return new TaskExecuteResult( new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); } } - private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { try { List resultSets = new ArrayList<>(); @@ -523,10 +520,10 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt } } } - RowStream rowStream = new PostgreSQLQueryRowStream(resultSets, fields, true); + RowStream rowStream = new ClearEmptyRowStreamWrapper(new PostgreSQLQueryRowStream(resultSets, true, project.getTagFilter())); return new TaskExecuteResult(rowStream); } catch (SQLException e) { - logger.info("error: ", e); + logger.info(e.getMessage()); return new TaskExecuteResult( new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); } @@ -612,7 +609,7 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) String field = path.substring(path.lastIndexOf('.') + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { - tags = data.getTags(i); + tags = data.getTags(j); } StringBuilder columnKeys = new StringBuilder(); @@ -798,18 +795,45 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView private void executeBatch(Statement stmt, Map>> tableToColumnEntries) throws SQLException { for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + String tableName = entry.getKey(); + String columnNames = entry.getValue().k.substring(0, entry.getValue().k.length() - 2); + List values = entry.getValue().v; + StringBuilder insertStatement = new StringBuilder(); insertStatement.append("INSERT INTO "); - insertStatement.append(entry.getKey()); + insertStatement.append(tableName); insertStatement.append(" (time, "); - insertStatement.append(entry.getValue().k, 0, entry.getValue().k.length() - 2); + insertStatement.append(columnNames); insertStatement.append(") VALUES"); - for (String value : entry.getValue().v) { + for (String value : values) { insertStatement.append(" ("); insertStatement.append(value, 0, value.length() - 2); insertStatement.append("), "); } - stmt.addBatch(insertStatement.substring(0, insertStatement.toString().length() - 2)); + insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.toString().length() - 2)); + insertStatement.append(" ON CONFLICT (time) DO UPDATE SET "); + String[] parts = columnNames.split(", "); + if (parts.length != 1) { + insertStatement.append("("); // 只有一列不加括号 + } + insertStatement.append(columnNames); + if (parts.length != 1) { + insertStatement.append(")"); // 只有一列不加括号 + } + insertStatement.append(" = "); + if (parts.length != 1) { + insertStatement.append("("); // 只有一列不加括号 + } + for (String part : columnNames.split(", ")) { + insertStatement.append("excluded."); + insertStatement.append(part); + insertStatement.append(", "); + } + insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.toString().length() - 2)); + if (parts.length != 1) { + insertStatement.append(")"); // 只有一列不加括号 + } + stmt.addBatch(insertStatement.toString()); } stmt.executeBatch(); } @@ -817,13 +841,14 @@ private void executeBatch(Statement stmt, Map> private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { try { Statement stmt = conn.createStatement(); + String statement; List paths = delete.getPatterns(); - List deletedPaths; - String table; - String field; + List> deletedPaths; // table name -> column name + String tableName; + String columnName; DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet; - ResultSet columnSet; + ResultSet tableSet = null; + ResultSet columnSet = null; if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { @@ -831,37 +856,50 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Connection postgresConn = getConnection("postgres", getUrl("postgres")); // 正在使用的数据库无法被删除，因此需要切换到名为postgres的默认数据库 if (postgresConn != null) { stmt = postgresConn.createStatement(); - stmt.execute(String.format("drop database %s", storageUnit)); // 删除数据库 + statement = String.format("drop database %s", storageUnit); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 删除数据库 postgresConn.close(); return new TaskExecuteResult(null, null); } else { return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("cannot connect to database: postgres", new SQLException())); } } else { - deletedPaths = determineDeletedPathList(paths, delete.getTagFilter()); - for (String path : deletedPaths) { - table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - field = path.substring(path.lastIndexOf('.') + 1); - tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); + deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); + for (Pair pair : deletedPaths) { + tableName = pair.k; + columnName = pair.v; + tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); if (tableSet.next()) { - stmt.execute(String.format("alter table %s drop column if exists %s", table, field)); // 删除列 + statement = String.format("alter table %s drop column if exists %s", tableName, columnName); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 删除列 } } } } else { - deletedPaths = determineDeletedPathList(paths, delete.getTagFilter()); - for (int i = 0; i < deletedPaths.size(); i++) { - String path = deletedPaths.get(i); - TimeRange timeRange = delete.getTimeRanges().get(i); - table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - field = path.substring(path.lastIndexOf('.') + 1); - columnSet = databaseMetaData.getColumns(null, "%", table, field); + deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); + for (Pair pair : deletedPaths) { + tableName = pair.k; + columnName = pair.v; + columnSet = databaseMetaData.getColumns(null, "%", tableName, columnName); if (columnSet.next()) { - stmt.execute(String.format("update %s set %s = null where (time > %d and time < %d)", table, field, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP))); // 将目标列的目标范围的值置为空 + for (TimeRange timeRange : delete.getTimeRanges()) { + statement = String.format("update %s set %s = null where (time >= %d and time < %d)", tableName, columnName, + timeRange.getBeginTime(), timeRange.getEndTime()); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 将目标列的目标范围的值置为空 + } } } } + if (tableSet != null) { + tableSet.close(); + } + if (columnSet != null) { + columnSet.close(); + } + stmt.close(); conn.close(); return new TaskExecuteResult(null, null); } catch (SQLException e) { @@ -871,27 +909,21 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, } } - private List determineDeletedPathList(List paths, TagFilter tagFilter) { + private List> determineDeletedPaths(List paths, TagFilter tagFilter) { List timeSeries = getTimeSeries(); - List deletedPaths = new ArrayList<>(); - - if (tagFilter != null) { - for (Timeseries ts: timeSeries) { - for (String path : paths) { - if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath()) && - TagKVUtils.match(ts.getTags(), tagFilter)) { - deletedPaths.add(ts.getPath()); - break; - } - } - } - } else { - for (Timeseries ts: timeSeries) { - for (String path : paths) { - if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath())) { - deletedPaths.add(ts.getPath()); - break; + List> deletedPaths = new ArrayList<>(); + + for (Timeseries ts: timeSeries) { + for (String path : paths) { + if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath())) { + if (tagFilter != null && !TagKVUtils.match(ts.getTags(), tagFilter)) { + continue; } + String fullPath = ts.getPath(); + String tableName = fullPath.substring(0, fullPath.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String columnName = toFullName(fullPath.substring(fullPath.lastIndexOf('.') + 1), ts.getTags()); + deletedPaths.add(new Pair<>(tableName, columnName)); + break; } } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java index 9bc3c07316..205d872bf1 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java @@ -2,67 +2,112 @@ import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; +import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.ResultSet; +import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.List; +import java.util.*; + +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; public class PostgreSQLQueryRowStream implements RowStream { - private final List resultSets; - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - private final long[] currTimestamps; + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLQueryRowStream.class); - private final Object[] currValues; + private final List resultSets; private final Header header; - private boolean isdummy; - public PostgreSQLQueryRowStream(List resultSets, List fields, boolean isdummy) throws SQLException { + private final boolean isDummy; + + private boolean[] gotNext; // 标记每个结果集是否已经获取到下一行，如果是，则在下次调用 next() 时无需再调用该结果集的 next() + + private long[] cachedTimestamps; // 缓存每个结果集当前的 time 列的值 + + private Object[] cachedValues; // 缓存每列当前的值 + + private int[] resultSetSizes; // 记录每个结果集的列数 + + private Map fieldToColumnName; // 记录匹配 tagFilter 的列名 + + private Row cachedRow; + + private boolean hasCachedRow; + + public PostgreSQLQueryRowStream(List resultSets, boolean isDummy, TagFilter tagFilter) throws SQLException { this.resultSets = resultSets; - this.isdummy = isdummy; - this.header = new Header(Field.KEY, fields); - this.currTimestamps = new long[resultSets.size()]; - this.currValues = new Object[resultSets.size()]; - try { - for (int i = 0; i < this.currTimestamps.length; i++) { - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - if (isdummy) { - this.currTimestamps[i] = toHash(resultSet.getString(1)); - } else { - this.currTimestamps[i] = resultSet.getLong(1); - } - String typeName = ""; - if (resultSet.getObject(2) != null) { - typeName = resultSet.getObject(2).getClass().getTypeName(); - } else { - typeName = ""; - } - if (typeName.contains("String")) { - this.currValues[i] = resultSet.getObject(2).toString().getBytes(); - } else { - this.currValues[i] = resultSet.getObject(2); - } + this.isDummy = isDummy; + + if (resultSets.isEmpty()) { + this.header = new Header(Field.KEY, Collections.emptyList()); + return; + } + + boolean filterByTags = tagFilter != null; + + Field time = null; + List fields = new ArrayList<>(); + this.resultSetSizes = new int[resultSets.size()]; + this.fieldToColumnName = new HashMap<>(); + + for (int i = 0; i < resultSets.size(); i++) { + ResultSetMetaData resultSetMetaData = resultSets.get(i).getMetaData(); + String tableName = resultSetMetaData.getTableName(1); + int cnt = 0; + for (int j = 1; j <= resultSetMetaData.getColumnCount(); j++) { + String columnName = resultSetMetaData.getColumnName(j); + String typeName = resultSetMetaData.getColumnTypeName(j); + if (j == 1 && columnName.equals("time")) { + time = Field.KEY; + continue; + } + + Pair> namesAndTags = splitFullName(columnName); + Field field = new Field( + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + namesAndTags.k.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v + ); + + if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { + continue; } + fieldToColumnName.put(field, columnName); + fields.add(field); + cnt++; } - } catch (SQLException e) { - e.printStackTrace(); - // pass + resultSetSizes[i] = cnt; } + this.header = new Header(time, fields); + + this.gotNext = new boolean[resultSets.size()]; + Arrays.fill(gotNext, false); + this.cachedTimestamps = new long[resultSets.size()]; + Arrays.fill(cachedTimestamps, Long.MAX_VALUE); + this.cachedValues = new Object[fields.size()]; + Arrays.fill(cachedValues, null); + this.cachedRow = null; + this.hasCachedRow = false; } @Override public Header getHeader() { - return this.header; + return header; } @Override @@ -72,64 +117,119 @@ public void close() { resultSet.close(); } } catch (SQLException e) { - // pass + logger.error(e.getMessage()); } } @Override public boolean hasNext() { - boolean f = false; //判断是否是全为空的行 - for (Object i : currValues) { - if (i != null) { - f = true; + if (resultSets.isEmpty()) { + return false; + } + + try { + if (!hasCachedRow) { + cacheOneRow(); } + } catch (SQLException e) { + logger.error(e.getMessage()); } - long timestamp = Long.MAX_VALUE; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); + + return cachedRow != null; + } + + @Override + public Row next() throws PhysicalException { + try { + Row row; + if (!hasCachedRow) { + cacheOneRow(); } + row = cachedRow; + hasCachedRow = false; + cachedRow = null; + return row; + } catch (SQLException e) { + logger.error(e.getMessage()); + throw new RowFetchException(e); } - if (!f && timestamp == 0) { - try { - for (int i = 0; i < this.currTimestamps.length; i++) { - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - if (isdummy) { - this.currTimestamps[i] = toHash(resultSet.getString(1)); - } else { - this.currTimestamps[i] = resultSet.getLong(1); - } - String typeName = ""; - if (resultSet.getObject(2) != null) { - typeName = resultSet.getObject(2).getClass().getTypeName(); - } else { - typeName = "null"; - } - if (typeName.contains("String")) { - this.currValues[i] = resultSet.getObject(2).toString().getBytes(); + } + + private void cacheOneRow() throws SQLException { + boolean hasNext = false; + long timestamp; + Object[] values = new Object[header.getFieldSize()]; + + int startIndex = 0; + int endIndex = 0; + for (int i = 0; i < resultSets.size(); i++) { + ResultSet resultSet = resultSets.get(i); + if (resultSetSizes[i] == 0) { + continue; + } + endIndex += resultSetSizes[i]; + if (!gotNext[i]) { + boolean tempHasNext = resultSet.next(); + hasNext |= tempHasNext; + gotNext[i] = true; + + if (tempHasNext) { + long tempTimestamp; + Object tempValue; + + if (isDummy) { + tempTimestamp = toHash(resultSet.getString("time")); + } else { + tempTimestamp = resultSet.getLong("time"); + } + cachedTimestamps[i] = tempTimestamp; + + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int j = 0; j < resultSetSizes[i]; j++) { + Object value = resultSet.getObject(fieldToColumnName.get(header.getField(startIndex + j))); + if (header.getField(startIndex + j).getType() == DataType.BINARY && value != null) { + tempValue = value.toString().getBytes(); } else { - this.currValues[i] = resultSet.getObject(2); + tempValue = value; } - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; + cachedValues[startIndex + j] = tempValue; + } + } else { + cachedTimestamps[i] = Long.MAX_VALUE; + for (int j = startIndex; j < endIndex; j++) { + cachedValues[j] = null; } } - } catch (Exception e) { - //error - logger.error("error in postgresqlrowstream "); + } else { + hasNext = true; } - return hasNext(); + startIndex = endIndex; } - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - return true; + if (hasNext) { + timestamp = Arrays.stream(cachedTimestamps).min().getAsLong(); + startIndex = 0; + endIndex = 0; + for (int i = 0; i < resultSets.size(); i++) { + endIndex += resultSetSizes[i]; + if (cachedTimestamps[i] == timestamp) { + for (int j = 0; j < resultSetSizes[i]; j++) { + values[startIndex + j] = cachedValues[startIndex + j]; + } + gotNext[i] = false; + } else { + for (int j = 0; j < resultSetSizes[i]; j++) { + values[startIndex + j] = null; + } + gotNext[i] = true; + } + startIndex = endIndex; } + cachedRow = new Row(header, timestamp, values); + } else { + cachedRow = null; } - return false; + hasCachedRow = true; } private long toHash(String s) { @@ -144,49 +244,4 @@ private long toHash(String s) { } return hv; } - - @Override - public Row next() throws PhysicalException { - try { - long timestamp = Long.MAX_VALUE; - Object[] values = new Object[this.resultSets.size()]; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); - } - } - for (int i = 0; i < this.currTimestamps.length; i++) { - if (this.currTimestamps[i] == timestamp) { - values[i] = this.currValues[i]; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - if (isdummy) { - this.currTimestamps[i] = toHash(resultSet.getString(1)); - } else { - this.currTimestamps[i] = resultSet.getLong(1); - } - String typeName = ""; - if (resultSet.getObject(2) != null) { - typeName = resultSet.getObject(2).getClass().getTypeName(); - } else { - typeName = "null"; - } - if (typeName.contains("String")) { - this.currValues[i] = resultSet.getObject(2).toString().getBytes(); - } else { - this.currValues[i] = resultSet.getObject(2); - } - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; - } - } - } - return new Row(header, timestamp, values); - } catch (Exception e) { - logger.info("error:", e); - throw new RowFetchException(e); - } - } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index 011a0762ac..a4ab5b1836 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -24,7 +24,7 @@ public class FilterTransformer { - public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; + public static final long MAX_TIMESTAMP = Long.MAX_VALUE; public static String toString(Filter filter) { if (filter == null) { @@ -55,7 +55,7 @@ private static String toString(NotFilter filter) { } private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + Math.min(filter.getValue(), MAX_TIMESTAMP); + return "time " + Op.op2Str(filter.getOp()) + " " + Math.min(filter.getValue(), MAX_TIMESTAMP); } private static String toString(ValueFilter filter) { @@ -66,5 +66,4 @@ private static String toString(OrFilter filter) { return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); } - } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java index a36ca9ecff..cc9b3bcbac 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -3,8 +3,8 @@ import cn.edu.tsinghua.iginx.utils.Pair; import java.util.*; -import java.util.stream.Collectors; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; // TODO 目前未考虑 annotation diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index 017082b3d4..79f9daecdb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -201,7 +201,7 @@ public void testCountPath() { executeAndCompare(statement, expected); } -// @Test + @Test public void testCountPoints() { if (ifScaleOutIn) return; String statement = "COUNT POINTS;"; @@ -209,7 +209,7 @@ public void testCountPoints() { executeAndCompare(statement, expected); } -// @Test + @Test public void testShowTimeSeries() { if (!isAbleToShowTimeSeries || ifScaleOutIn) { return; @@ -303,14 +303,14 @@ public void testShowTimeSeries() { executeAndCompare(statement, expected); } -// @Test + @Test public void testShowReplicaNum() { String statement = "SHOW REPLICA NUMBER;"; String expected = "Replica num: 1\n"; executeAndCompare(statement, expected); } -// @Test + @Test public void testTimeRangeQuery() { String statement = "SELECT s1 FROM us.d1 WHERE key > 100 AND key < 120;"; String expected = "ResultSets:\n" + @@ -431,7 +431,7 @@ public void testValueFilter() { executeAndCompare(query, expected); } -// @Test + @Test public void testPathFilter() { String insert = "INSERT INTO us.d9(key, a, b) VALUES (1, 1, 9), (2, 2, 8), (3, 3, 7), (4, 4, 6), (5, 5, 5), (6, 6, 4), (7, 7, 3), (8, 8, 2), (9, 9, 1);"; execute(insert); @@ -518,7 +518,7 @@ public void testPathFilter() { executeAndCompare(query, expected); } -// @Test + @Test public void testLimitAndOffsetQuery() { String statement = "SELECT s1 FROM us.d1 WHERE key > 0 AND key < 10000 limit 10;"; String expected = "ResultSets:\n" + @@ -559,7 +559,7 @@ public void testLimitAndOffsetQuery() { executeAndCompare(statement, expected); } -// @Test + @Test public void testOrderByQuery() { String insert = "INSERT INTO us.d2 (key, s1, s2, s3) values " + "(1, \"apple\", 871, 232.1), (2, \"peach\", 123, 132.5), (3, \"banana\", 356, 317.8)," @@ -682,7 +682,7 @@ public void testOrderByQuery() { executeAndCompare(orderByQuery, expected); } -// @Test + @Test public void testFirstLastQuery() { String statement = "SELECT FIRST(s2) FROM us.d1 WHERE key > 0;"; String expected = "ResultSets:\n" + @@ -805,7 +805,7 @@ public void testFirstLastQuery() { executeAndCompare(statement, expected); } -// @Test + @Test public void testAggregateQuery() { String statement = "SELECT %s(s1), %s(s2) FROM us.d1 WHERE key > 0 AND key < 1000;"; List funcTypeList = Arrays.asList( @@ -869,7 +869,7 @@ public void testAggregateQuery() { } } -// @Test + @Test public void testDownSampleQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000));"; List funcTypeList = Arrays.asList( @@ -996,7 +996,7 @@ public void testDownSampleQuery() { } } -// @Test + @Test public void testRangeDownSampleQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 600 AND s1 <= 900 OVER (RANGE 100 IN (0, 1000));"; List funcTypeList = Arrays.asList( @@ -1074,7 +1074,7 @@ public void testRangeDownSampleQuery() { } } -// @Test + @Test public void testSlideWindowByTimeQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 OVER (RANGE 100 IN (0, 1000) STEP 50);"; List funcTypeList = Arrays.asList( @@ -1264,7 +1264,7 @@ public void testSlideWindowByTimeQuery() { } } -// @Test + @Test public void testRangeSlideWindowByTimeQuery() { String statement = "SELECT %s(s1), %s(s4) FROM us.d1 WHERE key > 300 AND s1 <= 600 OVER (RANGE 100 IN (0, 1000) STEP 50);"; List funcTypeList = Arrays.asList( @@ -1627,7 +1627,7 @@ public void testCrossRangeDelete() { executeAndCompare(queryOverDeleteRange, expected); } -// @Test + @Test public void testGroupBy() { String insert = "insert into test(key, a, b, c, d) values (1, 3, 2, 3.1, \"val1\"), (2, 1, 3, 2.1, \"val2\"), (3, 2, 2, 1.1, \"val5\"), (4, 3, 2, 2.1, \"val2\"), (5, 1, 2, 3.1, \"val1\"), (6, 2, 2, 5.1, \"val3\");"; execute(insert); @@ -1748,7 +1748,7 @@ public void testGroupBy() { executeAndCompare(query, expected); } -// @Test + @Test public void testGroupByAndHaving() { String insert = "insert into test(key, a, b, c, d) values (1, 3, 2, 3.1, \"val1\"), (2, 1, 3, 2.1, \"val2\"), (3, 2, 2, 1.1, \"val5\"), (4, 3, 2, 2.1, \"val2\"), (5, 1, 2, 3.1, \"val1\"), (6, 2, 2, 5.1, \"val3\");"; execute(insert); @@ -2176,7 +2176,7 @@ public void testMultiJoin() { executeAndCompare(statement, expected); } -// @Test + @Test public void testBasicArithmeticExpr() { String insert = "INSERT INTO us.d3 (key, s1, s2, s3) values " + "(1, 1, 6, 1.5), (2, 2, 5, 2.5), (3, 3, 4, 3.5), (4, 4, 3, 4.5), (5, 5, 2, 5.5), (6, 6, 1, 6.5);"; @@ -2263,7 +2263,7 @@ public void testBasicArithmeticExpr() { executeAndCompare(statement, expected); } -// @Test + @Test public void testComplexArithmeticExpr() { String insert = "INSERT INTO us.d3 (key, s1, s2, s3) values " + "(1, 1, 6, 1.5), (2, 2, 5, 2.5), (3, 3, 4, 3.5), (4, 4, 3, 4.5), (5, 5, 2, 5.5), (6, 6, 1, 6.5);"; @@ -2350,7 +2350,7 @@ public void testComplexArithmeticExpr() { executeAndCompare(statement, expected); } -// @Test + @Test public void testAlias() { // time series alias String statement = "SELECT s1 AS rename_series, s2 FROM us.d1 WHERE s1 >= 1000 AND s1 < 1010;"; @@ -2413,7 +2413,7 @@ public void testAlias() { executeAndCompare(statement, expected); } -// @Test + @Test public void testAggregateSubQuery() { String statement = "SELECT %s_s1 FROM (SELECT %s(s1) AS %s_s1 FROM us.d1 OVER(RANGE 60 IN [1000, 1600)));"; List funcTypeList = Arrays.asList( @@ -2541,7 +2541,7 @@ public void testAggregateSubQuery() { } } -// @Test + @Test public void testValueFilterSubQuery() { String statement = "SELECT ts FROM (SELECT s1 AS ts FROM us.d1 WHERE us.d1.s1 >= 1000 AND us.d1.s1 < 1010);"; String expected = "ResultSets:\n" + @@ -2588,7 +2588,7 @@ public void testValueFilterSubQuery() { executeAndCompare(statement, expected); } -// @Test + @Test public void testMultiSubQuery() { String statement = "SELECT AVG(s1) AS avg_s1, SUM(s2) AS sum_s2 FROM us.d1 OVER (RANGE 10 IN [1000, 1100));"; String expected = "ResultSets:\n" + @@ -2642,7 +2642,7 @@ public void testMultiSubQuery() { executeAndCompare(statement, expected); } -// @Test + @Test public void testFromSubQuery() { String insert = "INSERT INTO test(key, a.a, a.b) VALUES (1, 1, 1.1), (2, 3, 3.1), (3, 7, 7.1);"; execute(insert); @@ -3010,7 +3010,7 @@ public void testDateFormat() { execute(String.format(insert, dateFormats.get(i), i)); } - String query = "SELECT date FROM us.d2;"; + String query = "SELECT date FROM us.d2 ORDER BY key;"; String expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3032,7 +3032,7 @@ public void testDateFormat() { + "Total line number = 12\n"; executeAndCompare(query, expected); - query = "SELECT date FROM us.d2 WHERE key >= 2021-08-26 16:15:27 AND key <= 2021.08.26T16:15:32.001;"; + query = "SELECT date FROM us.d2 WHERE key >= 2021-08-26 16:15:27 AND key <= 2021.08.26T16:15:32.001 ORDER BY key;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3054,7 +3054,7 @@ public void testDateFormat() { + "Total line number = 12\n"; executeAndCompare(query, expected); - query = "SELECT date FROM us.d2 WHERE key >= 2021.08.26 16:15:29 AND key <= 2021-08-26T16:15:30.001;"; + query = "SELECT date FROM us.d2 WHERE key >= 2021.08.26 16:15:29 AND key <= 2021-08-26T16:15:30.001 ORDER BY key;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3068,7 +3068,7 @@ public void testDateFormat() { + "Total line number = 4\n"; executeAndCompare(query, expected); - query = "SELECT date FROM us.d2 WHERE key >= 2021/08/26 16:15:28 AND key <= 2021/08/26T16:15:31.001;"; + query = "SELECT date FROM us.d2 WHERE key >= 2021/08/26 16:15:28 AND key <= 2021/08/26T16:15:31.001 ORDER BY key;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3304,7 +3304,7 @@ public void testErrorClause() { executeAndCompareErrMsg(errClause, "Group by can not use SetToSet and RowToRow functions."); } -// @Test + @Test public void testExplain() { if (ifScaleOutIn) return; String explain = "explain select max(s2), min(s1) from us.d1;"; @@ -3338,7 +3338,7 @@ public void testExplain() { executeAndCompare(explain, expected); } -// @Test + @Test public void testDeleteTimeSeries() { if (!isAbleToDelete || ifScaleOutIn) { return; From d2c18a2891060090c83b4695e6d44bf441fc166c Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 16 Mar 2023 09:56:53 +0800 Subject: [PATCH 37/94] Remove unused files --- conf/config.properties | 12 +- dataSources/dmdb/test.java | 319 ------------------ .../iginx/session/IoTDBSessionExample.java | 22 +- .../iginx/integration/func/tag/TagIT.java | 2 +- tools-viewPartitions/fragment.png | Bin 6457 -> 0 bytes 5 files changed, 21 insertions(+), 334 deletions(-) delete mode 100644 dataSources/dmdb/test.java delete mode 100644 tools-viewPartitions/fragment.png diff --git a/conf/config.properties b/conf/config.properties index 2943add3cd..9ad6455d79 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,11 +14,11 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 -#storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres +#storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456 +#storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 @@ -62,6 +62,12 @@ reshardFragmentTimeMargin=60 # parquet是否为本地存储 isLocalParquetStorage=true +# thrift线程池最小线程数量 +minThriftWorkerThreadNum = 20 + +# thrift线程池最大线程数量 +maxThriftWrokerThreadNum = 2147483647; + #################### ### Migration 相关配置 #################### diff --git a/dataSources/dmdb/test.java b/dataSources/dmdb/test.java deleted file mode 100644 index 1b9675a14f..0000000000 --- a/dataSources/dmdb/test.java +++ /dev/null @@ -1,319 +0,0 @@ -/*该例程实现插入数据，修改数据，删除数据，数据查询等基本操作。*/ - -import java.awt.Color; -import java.awt.Font; -import java.awt.Graphics2D; -import java.awt.font.FontRenderContext; -import java.awt.geom.Rectangle2D; -import java.awt.image.BufferedImage; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.math.BigDecimal; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.Date; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; -import javax.imageio.ImageIO; - -public class test { - // 定义DM JDBC驱动串 - String jdbcString = "dm.jdbc.driver.DmDriver"; - // 定义DM URL连接串 - String urlString = "jdbc:dm://localhost:5236"; - // 定义连接用户名 - String userName = "SYSDBA"; - // 定义连接用户口令 - String password = "SYSDBA001"; - // 定义连接对象 - Connection conn = null; - - /* 加载JDBC驱动程序 - * @throws SQLException 异常 */ - public void loadJdbcDriver() throws SQLException { - try { - System.out.println("Loading JDBC Driver..."); - // 加载JDBC驱动程序 - Class.forName(jdbcString); - } catch (ClassNotFoundException e) { - throw new SQLException("Load JDBC Driver Error : " + e.getMessage()); - } catch (Exception ex) { - throw new SQLException("Load JDBC Driver Error : " - + ex.getMessage()); - } - } - - /* 连接DM数据库 - * @throws SQLException 异常 */ - public void connect() throws SQLException { - try { - System.out.println("Connecting to DM Server..."); - // 连接DM数据库 - conn = DriverManager.getConnection(urlString, userName, password); - } catch (SQLException e) { - throw new SQLException("Connect to DM Server Error : " - + e.getMessage()); - } - } - - /* 关闭连接 - * @throws SQLException 异常 */ - public void disConnect() throws SQLException { - try { - // 关闭连接 - conn.close(); - } catch (SQLException e) { - throw new SQLException("close connection error : " + e.getMessage()); - } - } - - /* 往产品信息表插入数据 - * @throws SQLException 异常 */ - public void insertTable() throws SQLException { - // 插入数据语句 - String sql = "INSERT INTO production.product(name,author,publisher,publishtime," - + "product_subcategoryid,productno,satetystocklevel,originalprice,nowprice,discount," - + "description,photo,type,papertotal,wordtotal,sellstarttime,sellendtime) " - + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);"; - // 创建语句对象 - PreparedStatement pstmt = conn.prepareStatement(sql); - // 为参数赋值 - pstmt.setString(1, "三国演义"); - pstmt.setString(2, "罗贯中"); - pstmt.setString(3, "中华书局"); - pstmt.setDate(4, Date.valueOf("2005-04-01")); - pstmt.setInt(5, 4); - pstmt.setString(6, "9787101046121"); - pstmt.setInt(7, 10); - pstmt.setBigDecimal(8, new BigDecimal(19.0000)); - pstmt.setBigDecimal(9, new BigDecimal(15.2000)); - pstmt.setBigDecimal(10, new BigDecimal(8.0)); - pstmt.setString(11, "《三国演义》是中国第一部长篇章回体小说，中国小说由短篇发展至长篇的原因与说书有关。"); - // 设置大字段参数 - try { - // 创建一个图片用于插入大字段 - String filePath = "c:\\三国演义.jpg"; - CreateImage(filePath); - File file = new File(filePath); - InputStream in = new BufferedInputStream(new FileInputStream(file)); - pstmt.setBinaryStream(12, in, (int) file.length()); - } catch (FileNotFoundException e) { - System.out.println(e.getMessage()); - // 如果没有图片设置为NULL - pstmt.setNull(12, java.sql.Types.BINARY); - } catch (IOException e) { - System.out.println(e.getMessage()); - } - pstmt.setString(13, "25"); - pstmt.setInt(14, 943); - pstmt.setInt(15, 93000); - pstmt.setDate(16, Date.valueOf("2006-03-20")); - pstmt.setDate(17, Date.valueOf("1900-01-01")); - // 执行语句 - pstmt.executeUpdate(); - // 关闭语句 - pstmt.close(); - } - - /* 查询产品信息表 - * @throws SQLException 异常 */ - public void queryProduct() throws SQLException { - // 查询语句 - String sql = "SELECT productid,name,author,description,photo FROM production.product WHERE productid=11"; - // 创建语句对象 - Statement stmt = conn.createStatement(); - // 执行查询 - ResultSet rs = stmt.executeQuery(sql); - // 显示结果集 - displayResultSet(rs); - // 关闭结果集 - rs.close(); - // 关闭语句 - stmt.close(); - } - - /* 修改产品信息表数据 - * @throws SQLException 异常 */ - public void updateTable() throws SQLException { - // 更新数据语句 - String sql = "UPDATE production.product SET name = ?" - + "WHERE productid = 11;"; - // 创建语句对象 - PreparedStatement pstmt = conn.prepareStatement(sql); - // 为参数赋值 - pstmt.setString(1, "三国演义（上）"); - // 执行语句 - pstmt.executeUpdate(); - // 关闭语句 - pstmt.close(); - } - - /* 删除产品信息表数据 - * @throws SQLException 异常 */ - public void deleteTable() throws SQLException { - // 删除数据语句 - String sql = "DELETE FROM production.product WHERE productid = 11;"; - // 创建语句对象 - Statement stmt = conn.createStatement(); - // 执行语句 - stmt.executeUpdate(sql); - // 关闭语句 - stmt.close(); - } - - /* 查询产品信息表 - * @throws SQLException 异常 */ - public void queryTable() throws SQLException { - // 查询语句 - String sql = "SELECT productid,name,author,publisher FROM production.product"; - // 创建语句对象 - Statement stmt = conn.createStatement(); - // 执行查询 - ResultSet rs = stmt.executeQuery(sql); - // 显示结果集 - displayResultSet(rs); - // 关闭结果集 - rs.close(); - // 关闭语句 - stmt.close(); - } - - /* 调用存储过程修改产品信息表数据 - * @throws SQLException 异常 */ - public void updateProduct() throws SQLException { - // 更新数据语句 - String sql = "{ CALL production.updateProduct(?,?) }"; - // 创建语句对象 - CallableStatement cstmt = conn.prepareCall(sql); - // 为参数赋值 - cstmt.setInt(1, 1); - cstmt.setString(2, "红楼梦（上）"); - // 执行语句 - cstmt.execute(); - // 关闭语句 - cstmt.close(); - } - - /* 显示结果集 - * @param rs 结果集对象 - * @throws SQLException 异常 */ - private void displayResultSet(ResultSet rs) throws SQLException { - // 取得结果集元数据 - ResultSetMetaData rsmd = rs.getMetaData(); - // 取得结果集所包含的列数 - int numCols = rsmd.getColumnCount(); - // 显示列标头 - for (int i = 1; i <= numCols; i++) { - if (i > 1) { - System.out.print(","); - } - System.out.print(rsmd.getColumnLabel(i)); - } - System.out.println(""); - // 显示结果集中所有数据 - while (rs.next()) { - for (int i = 1; i <= numCols; i++) { - if (i > 1) { - System.out.print(","); - } - // 处理大字段 - if ("IMAGE".equals(rsmd.getColumnTypeName(i))) { - byte[] data = rs.getBytes(i); - if (data != null && data.length > 0) { - FileOutputStream fos; - try { - fos = new FileOutputStream("c:\\三国演义1.jpg"); - fos.write(data); - fos.close(); - } catch (FileNotFoundException e) { - System.out.println(e.getMessage()); - } catch (IOException e) { - System.out.println(e.getMessage()); - } - } - System.out.print("字段内容已写入文件c:\\三国演义1.jpg，长度" + data.length); - } else { - // 普通字段 - System.out.print(rs.getString(i)); - } - } - System.out.println(""); - } - } - - /* 创建一个图片用于插入大字段 - * @throws IOException 异常 */ - private void CreateImage(String path) throws IOException { - int width = 100; - int height = 100; - String s = "三国演义"; - File file = new File(path); - Font font = new Font("Serif", Font.BOLD, 10); - BufferedImage bi = new BufferedImage(width, height, - BufferedImage.TYPE_INT_RGB); - Graphics2D g2 = (Graphics2D) bi.getGraphics(); - g2.setBackground(Color.WHITE); - g2.clearRect(0, 0, width, height); - g2.setPaint(Color.RED); - FontRenderContext context = g2.getFontRenderContext(); - Rectangle2D bounds = font.getStringBounds(s, context); - double x = (width - bounds.getWidth()) / 2; - double y = (height - bounds.getHeight()) / 2; - double ascent = -bounds.getY(); - double baseY = y + ascent; - g2.drawString(s, (int) x, (int) baseY); - ImageIO.write(bi, "jpg", file); - } - - //类主方法 @param args 参数 - public static void main(String args[]) { - try { - // 定义类对象 - BasicApp basicApp = new BasicApp(); - // 加载驱动程序 - basicApp.loadJdbcDriver(); - // 连接DM数据库 - basicApp.connect(); - // 插入数据 - System.out.println("--- 插入产品信息 ---"); - basicApp.insertTable(); - // 查询含有大字段的产品信息 - System.out.println("--- 显示插入结果 ---"); - basicApp.queryProduct(); - // 在修改前查询产品信息表 - System.out.println("--- 在修改前查询产品信息 ---"); - basicApp.queryTable(); - // 修改产品信息表 - System.out.println("--- 修改产品信息 ---"); - basicApp.updateTable(); - // 在修改后查询产品信息表 - System.out.println("--- 在修改后查询产品信息 ---"); - basicApp.queryTable(); - // 删除产品信息表 - System.out.println("--- 删除产品信息 ---"); - basicApp.deleteTable(); - // 在删除后查询产品信息表 - System.out.println("--- 在删除后查询产品信息 ---"); - basicApp.queryTable(); - // 调用存储过程修改产品信息表 - System.out.println("--- 调用存储过程修改产品信息 ---"); - basicApp.updateProduct(); - // 在存储过程更新后查询产品信息表 - System.out.println("--- 调用存储过程后查询产品信息 ---"); - basicApp.queryTable(); - // 关闭连接 - basicApp.disConnect(); - } catch (SQLException e) { - System.out.println(e.getMessage()); - } - } -} diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java index 86aa42810a..3763425726 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/IoTDBSessionExample.java @@ -55,27 +55,27 @@ public static void main(String[] args) throws SessionException, ExecutionExcepti // 列式插入非对齐数据 insertNonAlignedColumnRecords(); // 行式插入对齐数据 -// insertRowRecords(); + insertRowRecords(); // 行式插入非对齐数据 -// insertNonAlignedRowRecords(); + insertNonAlignedRowRecords(); // 查询序列 -// showTimeSeries(); + showTimeSeries(); // 查询数据 -// queryData(); + queryData(); // 聚合查询 -// aggregateQuery(); + aggregateQuery(); // Last 查询 -// lastQuery(); + lastQuery(); // 降采样聚合查询 -// downsampleQuery(); + downsampleQuery(); // 曲线匹配 -// curveMatch(); + curveMatch(); // 删除数据 -// deleteDataInColumns(); + deleteDataInColumns(); // 再次查询数据 -// queryData(); + queryData(); // 查看集群信息 -// showClusterInfo(); + showClusterInfo(); // 关闭 Session session.closeSession(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java index 8107b42c09..2dae680371 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/tag/TagIT.java @@ -34,7 +34,7 @@ public TagIT() throws IOException { @BeforeClass public static void setUp() throws SessionException { - ifClearData = false; + ifClearData = true; session = new Session("127.0.0.1", 6888, "root", "root"); session.openSession(); } diff --git a/tools-viewPartitions/fragment.png b/tools-viewPartitions/fragment.png deleted file mode 100644 index 6c278386e941f5cdb31659ee9158dabaeb68a165..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6457 zcmeHMcTiK?x~C&bIf{sMk0_vkM37D>Dgsg!6+%hEp-Jx@5~N-aMGVqQKm$kq&Z++j|&&^EqxK0b6 zW?*38GSI*Kkb!|w8+fNqF##4ub3z0Ig8+lUU7bh4x$DFT@5j4&)UBX%m*Dz;q%u37>=Vy~(%k_6x+^mrIZ$E#jt$&KB?TbX(V8SrU9Ob9+@i7OAhyjFrzO{ zQpC;KAKg(PbogK7sW4;aWu)>MsV1BaybUIFUJe!u)L}J>ye+jcC-v~;!aQH2$i}ld z5dV|gOxzvPrMM%FT$O~gqA}mRg?b~k+BAZNlW(iLul>F8#Z0zTMydF&0mY8fEwsrjhKnlyHePRz@hN-3#l4R7#MrtB4ZRhco}cKGrwVd%PmmOkI zM-czN_D^2*7X?IUaEx~u0)?4sukIiEwBd32k+@UHll*^84G^>D{<%$0*nkfI+cV0u zMb(v6ODRK)ToK^-$YfD)lc%i~Cg#{ei+(5B=!+ds_~|+jt0!9lHSUf%qN*qk5s}s@>UU z9&7=Vg!b>}@BAm>klg4B#4>g3N7&xl!OpO$o10syQI_v^pE7emKtL7l@H)tEvG=BZ z7lU+lyt2}BxWw4e(NTHY-`v0ee>KDAW7z&i zTwL4-`!7)B@orsRU5=7R)AH+t#lD>zE38XR^3 z%_?YjZ7s$W&K=Gv*6vPa*=w^>In^3{W$oGLYJMv?TR11VO=g_a*jtu%E?I}UH!Ae) zGgj4T2SnW+=I~Pui{a87@dd&&Q9*NT!Dsi{MF!d8;jF?@RUK7xC++?!Gt-y&hsIX4 ztc;C~$83Y9(`Bs%g@iJ0*f#ks^_yxQIP~YLiXmxRdSVf`04#r52J*pfW0m0G%~>*H zAL}5GaURJ3oWM_>#x@ieJ9Z@s{A#aGZdPy`EeBvxR#dbNTd!-SEy%qPQ6C)~1OfKL z!^7#)eTFtwo)_%x&ZV|OdP$+%W+NKlzk6&12Uq}A0`vEyYuyo2w=N5`b3q@}k}HD`ZmUc3u~29sQ;N+z>7Vbog;jm$zrHjtMFet410EnP z{mf&kUe>m0=X-+O#@d=j;F_h5j?UtxEDD8UWo32l`CS(m7ehls7ni-Ox$=(PZwyi+ z0hc8PjM=uuaPSHW3RYqkv&$nSO>!+CKi>VsYx?NP6KLp4@lKu!BaeC=h{dXqD-LaC z&UrNFR(%n{DLGe`C}LeeCHPL1#Xc*&FbP*^W_JX&F6p?=-fR>(J?G7XEX~8AQ$(8? zn&g$FzK&n=I~I$#>(s=?+==6K@$m3)a}&~})figZ*fb+g=v5KxZVCKy^sN`jaLPQs zH2;v~+*({*+(-qQJlUJLDy*_uvKVmM4Sw(eejsRGn=blUN$lFz;s>9rnh zIA?h8Z>$-r06hR>l?{e6MWXMmP1Mw|8F> zb5*tg=>hyUqAyz!y}d$3eF|X}W#YMWjnJWB#v%mgns3+h7ITjuyToB@#-c4=9;LX+ zBoiO5@6^m$ViX^1#vW_NlxiaOM!=~f$Bg5>9lLP%c#y0{7(G97IgV3+vf0U#k)A#} zI;y$dtDvi^E2QLZS>t33h0Xwc^(RH_dysI)bz@LNiBwx#TcClqz5S1OkKd1$L$*84 z$jQKP8d_RfSwbh)C1`lv0@#Ui0yz@9ye| zdf=Lh|1wf;ZnDk|9kJJ&0SY2XqQfbL@uE#MN{<(2A)Ox66umKXjd) zmjG(Xk2Tc)dP>A306JWLiO~rQ3j=yfKUtej@JsN zkSxmf_AuJXJdPW@evVu4RRo8*7v%lBclGWQDE%wlJw1Af%Zulnl2aA+5A6^L#G&{t zkoRnxZbJXwxnEzKzluG&MOp&5yZmB%2+~u8q(8NVe_3A$^; zObeVtBEXDgq#l4$5ny%$bgEb3zsz$+V8Q@Z_*<#Ei&6bwBS&CY&OL{n9FAq3Q7b~w zGAx@S{r&wP0pg>qJT^LNg4T?nQMW28DijqJf$)O`vHKoooF%`IDE{x`2ZmdUiWRU{ zTtNYRu{X=UD{-PJ;3uKXoSjRMFh1T0=riDq+dim>b_;kO&-(iMKGAPrbFOo#KM(7> z(0%u^&*p5q1vKR9?(`BGN&obm*I9V3on1^A2!=cwrj>vZwP~r(-V*fKCVSi&9>)4at z-Q7*-=t>d-WN<=(`!NK93Ef>4Q4bVW^Ovey!~?3hTo`|&l}54l#4LQ~hvjM^=@@Vp z?uVA-Bbb`+yuN{f!J(ae*4o;dFbJ+KpGB;58=Gkf>Arcx8onCD6)fpV~l>^{U{NH+F=@&?X-DYrZzw2-HE7l5rD0v=nlMW2& z=cqSG#(C4%pV)p0fJh!$&_<4lt@*n!Z`84#78licCil}DEJnS>iCIYL;}(l#`iJu5 zCW{chOnmRnMAiPS={3&+I~jYqCCYBOT*0MX*@EOIWB2PtYZ9K<3+GQ)zAX;q&AIXY zbx5yJS2cnBH{`sSIs0lsQTD1+wp&?V|ISw8fXKXbcyZ?Ue5HOtErXAl-?`NuVi{VO z_Nyqa0*(f#>3q6Svk#vIxVO^)<(HYt=ly$ZSz)qYI8T`7!myQVDQLmTvtPuOftjA) zTJ?I>T+_JXfSt~$(mO}q%nK;TMyqekau=kHrj;9wObxR%*`mY#W;AW`F+-Xvgh<5= zLhf)mc^}FbA9Er7q#N>Q@2<|7QaJ#`Uu;uN4_rp^7akA>NSZX*?r_7Ph4*Lfz}lf~ zsGqoUGW;$ppYg2-G#Q!d)o<5dq%nEdT*zbJd%%L)ZP~@`xXg4RJlad6jk)b?vW)YA z@FKB|>%C=NM$6H}NdykbA7pGl>6^YJ+LX5!*66I#xg;NSRhsvTFS|==gt!oPy0LU? z7+d?TK6jxe3!J|tG|uqo!7eWXMRzR;kyMuiF@6>I0!8y)`RJ4>>MGFlr<7F(DxiGo zb0x7bG&ByoEa-h$MMiJcgf&Xm5r+a|t;>$!~{MRxg*7^?~*9_$rzniGN@R12(nr=N>oFW8Hy1hLRB^iR( zT$r4-K0qZ@lYLrj+IM0_J5qZ61bSz&7;H*MM({N>O>8q-A{#oFhWw7EN1y0>)u&Yq zY!Ml`Y0@W?>-fMypOokfZ@fUM$ZajIiglFMVl?~ zTbjClNqJID7pZSUYtB@@L963s0!9VcrWX&+>aM_Utk>=j&4J>IFT{baY;zfs^A}g7 zF_e3rMpYqk&hlX!e_!5258s+DBWsx{ChgG;hu=3ZzDTm`4IL%hnYu7JIdDIvM7bD->R))Q(u6K$S>FW{e~?k&8`KvV@Q`LeG#{vd9dYu(EdYR2c3Es!Wv zXA`MjboFiMj?}$0b2>LUJgMb}{)WHl(vX3(Yc!cMO>mBh7G{wjsC_lW zt=0sdcw!}&8M@NI*xLO*XF?;<4q(+1P1VfEM?48&S)8KKcy*j&$b7^ny?r0pVJh@$ z+IvV#v`D-l-K|Q-$TpWcySFTcYEvAxCvdGoiSR# zsKc+7E)Hkqt0PKsFb6LA(9Ud@=LF}kJ05RpU5t*D=^lLV1)o~BF~Mcs&-as&e9Wu0s&n{H?ep#Z4$h0+dk$f)g^ClC+3tPQ?-eE{a@{uq+xYN&N2sGv z(s80Z`-<)&^-Ss+Kjo)}^#NDZL&e@)oj^^Ns6mcxC>SDFHp*BUC9Rg4r-#z(*VwH| z)tB&<9PBwP_gd($FE!Xha18OQxp~Us(`S~DG9l&r9!)L6KD)yfaQJW0L+HGrA@an- z6EMg^^8g}d{{HP`M>@-M-|k^DB5blna+-T6p?6DCehWs;=sme|uc{$q=)RPJET_Zl zCAI~Vf{a}V>2>_XqRGd7!@>2OZ5_^TwI`MBoT9z6`M4vQfs&aM z7>I(xZ=9IFc|#{h9qC^>zIpSXFSP%nSC9Yg;{D&w^XNn~{Q3o!ramh0ON7Bd*W_;H IZFtmw0J{7X$N&HU From 7c37215c068d7863c67bbc0a70e6c677313ff936 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 16 Mar 2023 11:53:57 +0800 Subject: [PATCH 38/94] Add actions --- .github/actions/dbRunner/action.yml | 15 +++++++++++++++ .github/postgresql.sh | 11 +++++++++++ .github/postgresql_macos.sh | 13 +++++++++++++ .github/workflows/standalone-test.yml | 2 +- conf/config.properties | 4 ++-- .../edu/tsinghua/iginx/conf/ConfigDescriptor.java | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 2 +- .../integration/controller/testConfig.properties | 5 +++-- .../tsinghua/iginx/integration/tool/DBConf.java | 5 ++++- 9 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 .github/postgresql.sh create mode 100644 .github/postgresql_macos.sh diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml index 5a8f838dc0..c3be040925 100644 --- a/.github/actions/dbRunner/action.yml +++ b/.github/actions/dbRunner/action.yml @@ -52,4 +52,19 @@ runs: else echo "$RUNNER_OS is not supported" exit 1 + fi + + - if: inputs.version=='PostgreSQL' + name: Run DB + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/postgresql.sh" + "${GITHUB_WORKSPACE}/.github/postgresql.sh" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh" + "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh" + else + echo "$RUNNER_OS is not supported" + exit 1 fi \ No newline at end of file diff --git a/.github/postgresql.sh b/.github/postgresql.sh new file mode 100644 index 0000000000..b0e2d72433 --- /dev/null +++ b/.github/postgresql.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties + +sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties + +sh -c "sudo apt-get -y install postgresql-12" + +sh -c "sudo su postgres" diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh new file mode 100644 index 0000000000..3e5d626bc3 --- /dev/null +++ b/.github/postgresql_macos.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +set -e + +sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties + +sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties + +sh -c "wget https://sbp.enterprisedb.com/getfile.jsp?fileid=1258319" + +sh -c "unzip postgresql-15.2-1-osx-binaries.zip" + +sh -c "cd pgsql; sudo mkdir -p /usr/local/var/postgresql@15; ./bin/initdb -D /usr/local/var/postgresql@15" diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index 6b16c883e6..9fe27ebab9 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -19,7 +19,7 @@ jobs: java: [ 8 ] python-version: [ "3.7" ] os: [ ubuntu-latest, macos-latest ] - DB-name: ["IoTDB12", "InfluxDB", "Parquet"] + DB-name: ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/conf/config.properties b/conf/config.properties index 9ad6455d79..7874556453 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -17,8 +17,8 @@ password=root storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 -#storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456 -#storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456 +#storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password= +#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password= #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java index 209bf6eeb4..5a7ead3adf 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java @@ -67,7 +67,7 @@ private void loadPropsFromFile() { config.setAsyncExecuteThreadPool(Integer.parseInt(properties.getProperty("asyncExecuteThreadPool", "20"))); config.setReplicaNum(Integer.parseInt(properties.getProperty("replicaNum", "1"))); - config.setDatabaseClassNames(properties.getProperty("databaseClassNames", "iotdb=cn.edu.tsinghua.iginx.iotdb.IoTDBPlanExecutor,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBPlanExecutor,parquet=cn.edu.tsinghua.iginx.parquet.parquetStorage")); + config.setDatabaseClassNames(properties.getProperty("databaseClassNames", "iotdb=cn.edu.tsinghua.iginx.iotdb.IoTDBPlanExecutor,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBPlanExecutor,parquet=cn.edu.tsinghua.iginx.parquet.parquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage")); //,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage config.setPolicyClassName(properties.getProperty("policyClassName", "cn.edu.tsinghua.iginx.policy.naive.NativePolicy")); diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index f175e9da86..6ca931b02d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -81,7 +81,7 @@ public class PostgreSQLStorage implements IStorage { private static final String DEFAULT_USERNAME = "postgres"; - private static final String DEFAULT_PASSWORD = "postgres"; + private static final String DEFAULT_PASSWORD = ""; private static final String DEFAULT_DBNAME = "timeseries"; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties index ee9d0dac2b..0769e6efa6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties @@ -1,11 +1,12 @@ # the storage engine that you want to test #storageEngineList=iotdb12 -storageEngineList=IoTDB12,InfluxDB,Parquet +storageEngineList=IoTDB12,InfluxDB,Parquet,PostgreSQL # the info of the engine that you list in storageEngineList IoTDB12=127.0.0.1#6668#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false InfluxDB=127.0.0.1#8087#influxdb#url=http://localhost:8087/#username=user#password=12345678#sessionPoolSize=20#has_data=false#is_read_only=false#token=testToken#organization=testOrg Parquet=127.0.0.1#6668#parquet#dir=parquetData2#sessionPoolSize=20#has_data=false#is_read_only=false +PostgreSQL=127.0.0.1#5432#postgresql#username=postgres#password=#has_data=false#is_read_only=false # the test for every engine # the normal test TagIT,RestAnnotationIT,RestIT,TransformIT,SessionV2IT @@ -16,6 +17,6 @@ influxdb-test-list={}SQLSessionIT,{}SQLSessionPoolIT,RestAnnotationIT,TransformI IoTDB12-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=true,isSupportTagKV=true InfluxDB-config=isAbleToClearData=false,isAbleToDelete=false,isAbleToShowTimeSeries=false,isSupportSpecialPath=false,isSupportTagKV=false Parquet-config=isAbleToClearData=false,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=false,isSupportTagKV=true - +PostgreSQL-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=true,isSupportTagKV=true diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java index bfa474a91f..5dcb927b57 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java @@ -4,7 +4,8 @@ public final class DBConf { public enum DBType { parquet, iotdb12, - influxdb + influxdb, + postgresql } public static DBType getDBType(String dbName) { @@ -15,6 +16,8 @@ public static DBType getDBType(String dbName) { return DBType.influxdb; case "parquet": return DBType.parquet; + case "postgresql": + return DBType.postgresql; default: throw new IllegalArgumentException("Invalid DBName value provided"); } From d6b781d7c5b389785113d2b9b9dbc1a22c24f77c Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 16 Mar 2023 12:07:05 +0800 Subject: [PATCH 39/94] Update linux startup shell --- .github/postgresql.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/postgresql.sh b/.github/postgresql.sh index b0e2d72433..5eba5b78fa 100644 --- a/.github/postgresql.sh +++ b/.github/postgresql.sh @@ -6,6 +6,12 @@ sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1# sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties +sh -c "sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" + +sh -c "wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -" + +sh -c "sudo apt-get update" + sh -c "sudo apt-get -y install postgresql-12" sh -c "sudo su postgres" From b6a605a7351c564be02083aacde066abb3ccb9a9 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 16 Mar 2023 12:12:02 +0800 Subject: [PATCH 40/94] Update macos startup shell --- .github/postgresql_macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh index 3e5d626bc3..f9fd50b8f9 100644 --- a/.github/postgresql_macos.sh +++ b/.github/postgresql_macos.sh @@ -8,6 +8,6 @@ sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127. sh -c "wget https://sbp.enterprisedb.com/getfile.jsp?fileid=1258319" -sh -c "unzip postgresql-15.2-1-osx-binaries.zip" +sh -c "sudo unzip postgresql-15.2-1-osx-binaries.zip" sh -c "cd pgsql; sudo mkdir -p /usr/local/var/postgresql@15; ./bin/initdb -D /usr/local/var/postgresql@15" From 851095bc07fad0b85b2f2241ee68765254a27709 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Sun, 19 Mar 2023 21:37:47 +0800 Subject: [PATCH 41/94] Fix actions (#1) * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update linux startup shell * Update macos startup shell * Update macos startup shell * Fix special characters * Fix rest * Fix data type * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell * Update macos startup shell --- .github/postgresql.sh | 18 +++- .github/postgresql_macos.sh | 32 +++++- conf/config.properties | 6 +- .../iginx/compaction/CompactionManager.java | 4 +- .../storage/zk/ZooKeeperMetaStorage.java | 4 +- .../iginx/monitor/MonitorManager.java | 4 +- dataSources/postgresql/pom.xml | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 100 +++++++++++------- .../iginx/postgresql/tools/Constants.java | 2 +- .../postgresql/tools/DataTypeTransformer.java | 13 ++- .../edu/tsinghua/iginx/utils/StringUtils.java | 2 +- .../controller/testConfig.properties | 6 +- .../iginx/integration/tool/ConfLoder.java | 5 +- 13 files changed, 134 insertions(+), 64 deletions(-) diff --git a/.github/postgresql.sh b/.github/postgresql.sh index 5eba5b78fa..44a157b876 100644 --- a/.github/postgresql.sh +++ b/.github/postgresql.sh @@ -6,12 +6,24 @@ sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1# sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties -sh -c "sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" +sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" sh -c "wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -" sh -c "sudo apt-get update" -sh -c "sudo apt-get -y install postgresql-12" +sh -c "sudo apt-get -y install postgresql-15" -sh -c "sudo su postgres" +sh -c "sudo rm -rf /var/lib/postgresql/15/main" + +sh -c "sudo mkdir -p /var/lib/postgresql/15/main" + +sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" + +sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" + +sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/initdb -D /var/lib/postgresql/15/main --auth-local peer --auth-host scram-sha-256 --no-instructions'" + +sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/15/main start'" + +sh -c "sudo su - postgres -c 'psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh index f9fd50b8f9..7e5bcf148a 100644 --- a/.github/postgresql_macos.sh +++ b/.github/postgresql_macos.sh @@ -6,8 +6,34 @@ sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0 sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties -sh -c "wget https://sbp.enterprisedb.com/getfile.jsp?fileid=1258319" +sh -c "wget https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" -sh -c "sudo unzip postgresql-15.2-1-osx-binaries.zip" +sh -c "sudo unzip -q postgresql-15.2-1-osx-binaries.zip" -sh -c "cd pgsql; sudo mkdir -p /usr/local/var/postgresql@15; ./bin/initdb -D /usr/local/var/postgresql@15" +sh -c "sudo dscl . -create /Users/postgres" + +sh -c "sudo dscl . -create /Users/postgres UserShell /bin/bash" + +sh -c "sudo dscl . -create /Users/postgres RealName \"PostgreSQL\"" + +sh -c "sudo dscl . -create /Users/postgres UniqueID 666" + +sh -c "sudo dscl . -create /Users/postgres PrimaryGroupID 20" + +sh -c "sudo dscl . -create /Users/postgres NFSHomeDirectory /Users/postgres" + +sh -c "sudo dscl . -passwd /Users/postgres postgres" + +sh -c "sudo dscl . -append /Groups/admin GroupMembership postgres" + +sh -c "sudo mkdir -p /var/lib/postgresql/15/main" + +sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" + +sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" + +sh -c "cd pgsql/bin; sudo -u postgres ./initdb -D /var/lib/postgresql/15/main" + +sh -c "cd pgsql/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql/15/main start" + +sh -c "sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\"" diff --git a/conf/config.properties b/conf/config.properties index 7874556453..3f766e6fdb 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,11 +14,11 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 -#storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password= -#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password= +#storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres +storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java index 5aee84807b..65a1f39dbb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/compaction/CompactionManager.java @@ -28,7 +28,7 @@ public static CompactionManager getInstance() { } public void clearFragment() throws Exception { - logger.info("start to compact fragments"); +// logger.info("start to compact fragments"); if (ConfigDescriptor.getInstance().getConfig().isEnableInstantCompaction()) { new InstantCompaction(PhysicalEngineImpl.getInstance(), DefaultMetaManager.getInstance()).compact(); } else if (ConfigDescriptor.getInstance().getConfig().isEnableFragmentCompaction()) { @@ -38,6 +38,6 @@ public void clearFragment() throws Exception { } } } - logger.info("end compact fragments"); +// logger.info("end compact fragments"); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java index 3d851c04c1..0e3ea77061 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/storage/zk/ZooKeeperMetaStorage.java @@ -1836,10 +1836,10 @@ public void incrementFragmentHeatCounter() throws MetaStorageException { this.client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT) .forPath(STATISTICS_FRAGMENT_HEAT_COUNTER_PREFIX, JsonUtils.toJson(1)); } else { - logger.error("inc FragmentHeatCounter"); +// logger.error("inc FragmentHeatCounter"); int counter = JsonUtils.fromJson( this.client.getData().forPath(STATISTICS_FRAGMENT_HEAT_COUNTER_PREFIX), Integer.class); - logger.error("counter=" + counter); +// logger.error("counter=" + counter); this.client.setData() .forPath(STATISTICS_FRAGMENT_HEAT_COUNTER_PREFIX, JsonUtils.toJson(counter + 1)); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java index d1a65459f8..9528395d15 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/MonitorManager.java @@ -37,10 +37,10 @@ public void run() { while (true) { try { //清空节点信息 - logger.info("start to clear monitors"); +// logger.info("start to clear monitors"); compactionManager.clearFragment(); metaManager.clearMonitors(); - logger.error("end clear monitors"); +// logger.error("end clear monitors"); Thread.sleep(interval * 1000L); //上传本地统计数据 diff --git a/dataSources/postgresql/pom.xml b/dataSources/postgresql/pom.xml index 7ee0ca6f0e..314f8883eb 100644 --- a/dataSources/postgresql/pom.xml +++ b/dataSources/postgresql/pom.xml @@ -29,7 +29,7 @@ org.postgresql postgresql - LATEST + 42.5.4 diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 6ca931b02d..01b58f38d0 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -63,7 +63,6 @@ import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; -import static java.sql.ResultSet.*; public class PostgreSQLStorage implements IStorage { @@ -81,7 +80,7 @@ public class PostgreSQLStorage implements IStorage { private static final String DEFAULT_USERNAME = "postgres"; - private static final String DEFAULT_PASSWORD = ""; + private static final String DEFAULT_PASSWORD = "postgres"; private static final String DEFAULT_DBNAME = "timeseries"; @@ -341,21 +340,25 @@ private Map splitAndMergeQueryPatterns(Connection conn, List entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); String columnNames = entry.getValue(); + String[] parts = columnNames.split(", "); StringBuilder statement = new StringBuilder(); try { stmt = conn.createStatement(); statement.append("select time, "); - statement.append(columnNames); + for (String part : parts) { + statement.append(getCompleteName(part)); + statement.append(", "); + } + statement = new StringBuilder(statement.substring(0, statement.length() - 2)); statement.append(" from "); - statement.append(tableName); + statement.append(getCompleteName(tableName)); statement.append(" where "); statement.append(FilterTransformer.toString(filter)); rs = stmt.executeQuery(statement.toString()); @@ -567,15 +575,19 @@ private void createOrAlterTables(Connection conn, List paths, List> columnEntries : tableToColumnEntries.values()) { columnEntries.v.clear(); } @@ -776,7 +788,7 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView } } - executeBatch(stmt, tableToColumnEntries); + executeBatchInsert(stmt, tableToColumnEntries); for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { entry.getValue().v.clear(); } @@ -793,46 +805,57 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView return null; } - private void executeBatch(Statement stmt, Map>> tableToColumnEntries) throws SQLException { + private void executeBatchInsert(Statement stmt, Map>> tableToColumnEntries) throws SQLException { for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { String tableName = entry.getKey(); String columnNames = entry.getValue().k.substring(0, entry.getValue().k.length() - 2); List values = entry.getValue().v; + String[] parts = columnNames.split(", "); + boolean hasMultipleRows = parts.length != 1; StringBuilder insertStatement = new StringBuilder(); insertStatement.append("INSERT INTO "); - insertStatement.append(tableName); + insertStatement.append(getCompleteName(tableName)); insertStatement.append(" (time, "); - insertStatement.append(columnNames); + for (String part : parts) { + insertStatement.append(getCompleteName(part)); + insertStatement.append(", "); + } + insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.length() - 2)); insertStatement.append(") VALUES"); for (String value : values) { insertStatement.append(" ("); insertStatement.append(value, 0, value.length() - 2); insertStatement.append("), "); } - insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.toString().length() - 2)); + insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.length() - 2)); + insertStatement.append(" ON CONFLICT (time) DO UPDATE SET "); - String[] parts = columnNames.split(", "); - if (parts.length != 1) { + if (hasMultipleRows) { insertStatement.append("("); // 只有一列不加括号 } - insertStatement.append(columnNames); - if (parts.length != 1) { + for (String part : parts) { + insertStatement.append(getCompleteName(part)); + insertStatement.append(", "); + } + insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.length() - 2)); + if (hasMultipleRows) { insertStatement.append(")"); // 只有一列不加括号 } insertStatement.append(" = "); - if (parts.length != 1) { + if (hasMultipleRows) { insertStatement.append("("); // 只有一列不加括号 } - for (String part : columnNames.split(", ")) { + for (String part : parts) { insertStatement.append("excluded."); - insertStatement.append(part); + insertStatement.append(getCompleteName(part)); insertStatement.append(", "); } - insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.toString().length() - 2)); - if (parts.length != 1) { + insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.length() - 2)); + if (hasMultipleRows) { insertStatement.append(")"); // 只有一列不加括号 } + stmt.addBatch(insertStatement.toString()); } stmt.executeBatch(); @@ -871,7 +894,7 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, columnName = pair.v; tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); if (tableSet.next()) { - statement = String.format("alter table %s drop column if exists %s", tableName, columnName); + statement = String.format("alter table %s drop column if exists %s", getCompleteName(tableName), getCompleteName(columnName)); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 删除列 } @@ -885,7 +908,7 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, columnSet = databaseMetaData.getColumns(null, "%", tableName, columnName); if (columnSet.next()) { for (TimeRange timeRange : delete.getTimeRanges()) { - statement = String.format("update %s set %s = null where (time >= %d and time < %d)", tableName, columnName, + statement = String.format("update %s set %s = null where (time >= %d and time < %d)", getCompleteName(tableName), getCompleteName(columnName), timeRange.getBeginTime(), timeRange.getEndTime()); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 将目标列的目标范围的值置为空 @@ -931,6 +954,11 @@ private List> determineDeletedPaths(List paths, Tag return deletedPaths; } + private String getCompleteName(String name) { + return "\"" + name + "\""; +// return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; + } + @Override public void release() throws PhysicalException { try { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 5cb47b03b6..22c3dcc4c0 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -4,5 +4,5 @@ public class Constants { public static final String IGINX_SEPARATOR = "."; - public static final String POSTGRESQL_SEPARATOR = "$"; + public static final String POSTGRESQL_SEPARATOR = "\u2E82"; } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java index 1f892eb484..1e439beecc 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java @@ -7,11 +7,16 @@ public class DataTypeTransformer { public static DataType fromPostgreSQL(String dataType) { - if (dataType.contains("int") || dataType.contains("serial") || dataType.contains("timestamp")) { - return LONG; - } else if (dataType.contains("bool")) { + if (dataType.equalsIgnoreCase("bool")) { return BOOLEAN; - } else if (dataType.contains("float")) { + } else if (dataType.equalsIgnoreCase("int") || dataType.equalsIgnoreCase("int2") || dataType.equalsIgnoreCase("int4") || + dataType.equalsIgnoreCase("serial2") || dataType.equalsIgnoreCase("serial4")) { + return INTEGER; + } else if (dataType.equalsIgnoreCase("int8") || dataType.equalsIgnoreCase("serial8")) { + return LONG; + } else if (dataType.equalsIgnoreCase(("float4"))) { + return FLOAT; + } else if (dataType.equalsIgnoreCase("decimal") || dataType.equalsIgnoreCase("float8")) { return DOUBLE; } else { return BINARY; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java index 8e89b65600..452f62af4a 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java @@ -117,7 +117,7 @@ public static String reformatColumnName(String name) { } public static boolean isContainSpecialChar(String str) { - String regEx = "[~!@#$%&()+=|{}':;',<>?~]|\r|\n|\t|[\u2E80\u2E81\u2E83\u2E84]"; + String regEx = "[~!@#$%&()+=|{}':;',<>?~]|\r|\n|\t|[\u2E80\u2E81\u2E82\u2E83\u2E84]"; Pattern p = Pattern.compile(regEx); Matcher m = p.matcher(str); return m.find(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties index 0769e6efa6..da5b2bb4f6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties @@ -6,11 +6,11 @@ storageEngineList=IoTDB12,InfluxDB,Parquet,PostgreSQL IoTDB12=127.0.0.1#6668#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false InfluxDB=127.0.0.1#8087#influxdb#url=http://localhost:8087/#username=user#password=12345678#sessionPoolSize=20#has_data=false#is_read_only=false#token=testToken#organization=testOrg Parquet=127.0.0.1#6668#parquet#dir=parquetData2#sessionPoolSize=20#has_data=false#is_read_only=false -PostgreSQL=127.0.0.1#5432#postgresql#username=postgres#password=#has_data=false#is_read_only=false +PostgreSQL=127.0.0.1#5432#postgresql#username=postgres#password=postgres # the test for every engine # the normal test TagIT,RestAnnotationIT,RestIT,TransformIT,SessionV2IT -test-list={}SQLSessionIT,{}SQLSessionPoolIT,TagIT,RestAnnotationIT,RestIT,TransformIT,SessionV2IT,SessionV2IT,{}SessionIT,{}SessionPoolIT +test-list=SQLSessionIT,SQLSessionPoolIT,TagIT,RestAnnotationIT,RestIT,TransformIT,SessionV2IT,SessionV2IT,SessionIT,SessionPoolIT influxdb-test-list={}SQLSessionIT,{}SQLSessionPoolIT,RestAnnotationIT,TransformIT,{}SessionIT,{}SessionPoolIT # the DB config @@ -18,5 +18,3 @@ IoTDB12-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries InfluxDB-config=isAbleToClearData=false,isAbleToDelete=false,isAbleToShowTimeSeries=false,isSupportSpecialPath=false,isSupportTagKV=false Parquet-config=isAbleToClearData=false,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=false,isSupportTagKV=true PostgreSQL-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=true,isSupportTagKV=true - - diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoder.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoder.java index ab09994275..6743d7994c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoder.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoder.java @@ -77,10 +77,11 @@ public void loadTestConf() throws IOException { // load the task list for (String storageEngine : storageEngines) { String tasks = null; - if (storageEngine.toLowerCase().equals("influxdb")) + if (storageEngine.equalsIgnoreCase("influxdb")) { tasks = properties.getProperty("influxdb-" + TESTTASK); - else + } else { tasks = properties.getProperty(TESTTASK); + } logInfo("the task of {} is :", storageEngine); List oriTaskList = Arrays.asList(tasks.split(",")), taskList = new ArrayList<>(); for(String taskName : oriTaskList) { From 9d815a618693980235ec50bc04a00f0b3d9b84dd Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Sun, 19 Mar 2023 21:49:02 +0800 Subject: [PATCH 42/94] Update config --- conf/config.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 3f766e6fdb..4845f5678d 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,11 +14,11 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres +#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData # 写入的副本个数 From b78f01d855c847f73af212c99713431b36dcd9b1 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Mon, 20 Mar 2023 09:23:24 +0800 Subject: [PATCH 43/94] Fix actions (#2) * Add logs for testing * Remove logs --- .../cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java | 3 ++- .../iginx/integration/controller/testConfig.properties | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 01b58f38d0..011648d05b 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -586,7 +586,7 @@ private void createOrAlterTables(Connection conn, List paths, List Date: Tue, 21 Mar 2023 10:56:12 +0800 Subject: [PATCH 44/94] Support expansion (#3) * Support expansion * Fix databaseMetadata bug --- .github/workflows/standalone-test.yml | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 542 ++++++++---------- .../entity/PostgreSQLQueryRowStream.java | 32 +- .../iginx/postgresql/tools/Constants.java | 32 ++ .../iginx/postgresql/tools/HashUtils.java | 17 + .../iginx/postgresql/tools/TagKVUtils.java | 2 - 6 files changed, 310 insertions(+), 317 deletions(-) rename dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/{ => query}/entity/PostgreSQLQueryRowStream.java (91%) create mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index 9fe27ebab9..19ad1fd4ea 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -12,7 +12,7 @@ concurrency: jobs: Union-DB-Test: - timeout-minutes: 15 + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 011648d05b..182b7875dd 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -29,7 +29,6 @@ import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; import cn.edu.tsinghua.iginx.engine.shared.TimeRange; import cn.edu.tsinghua.iginx.engine.shared.data.read.ClearEmptyRowStreamWrapper; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; @@ -43,7 +42,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.metadata.entity.*; -import cn.edu.tsinghua.iginx.postgresql.entity.PostgreSQLQueryRowStream; +import cn.edu.tsinghua.iginx.postgresql.query.entity.PostgreSQLQueryRowStream; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; @@ -59,8 +58,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; +import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; @@ -68,43 +67,11 @@ public class PostgreSQLStorage implements IStorage { private static final Logger logger = LoggerFactory.getLogger(PostgreSQLStorage.class); - private static final int BATCH_SIZE = 10000; - - private static final String STORAGE_ENGINE = "postgresql"; - - private static final String USERNAME = "username"; - - private static final String PASSWORD = "password"; - - private static final String DBNAME = "dbname"; - - private static final String DEFAULT_USERNAME = "postgres"; - - private static final String DEFAULT_PASSWORD = "postgres"; - - private static final String DEFAULT_DBNAME = "timeseries"; - - private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; - - private static final String FIRST_QUERY = "select first(%s, time) from %s"; - - private static final String LAST_QUERY = "select last(%s, time) from %s"; - - private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; - - private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - - private static final String DATABASE_PREFIX = "unit"; - - private static final long MAX_TIMESTAMP = Long.MAX_VALUE; - private final StorageEngineMeta meta; private final Map connectionPoolMap = new ConcurrentHashMap<>(); - private Connection connection; - - private String D_URL = ""; + private final Connection connection; public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { this.meta = meta; @@ -120,7 +87,7 @@ public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationExc try { connection = DriverManager.getConnection(connUrl); } catch (SQLException e) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); + throw new StorageInitializationException("cannot connect to " + meta); } } @@ -140,34 +107,42 @@ private boolean testConnection() { } } - private String getUrl(String dbname) { + private String getUrl(String databaseName) { Map extraParams = meta.getExtraParams(); String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), dbname, + .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), databaseName, username, password); return connUrl; } - private Connection getConnection(String dbname, String url) { + private Connection getConnection(String databaseName) { + if (databaseName.startsWith("dummy")) { + return null; + } + if (databaseName.equalsIgnoreCase("template0") || databaseName.equalsIgnoreCase("template1")) { + return null; + } + try { Statement stmt = connection.createStatement(); - stmt.execute(String.format("create database %s", dbname)); + stmt.execute(String.format(CREATE_DATABASE_STATEMENT, databaseName)); + stmt.close(); } catch (SQLException e) { -// logger.info("database {} exists!", dbname); +// logger.info("database {} exists!", databaseName); } try { - if (connectionPoolMap.containsKey(dbname)) { - return connectionPoolMap.get(dbname).getConnection(); + if (connectionPoolMap.containsKey(databaseName)) { + return connectionPoolMap.get(databaseName).getConnection(); } PGConnectionPoolDataSource connectionPool = new PGConnectionPoolDataSource(); - connectionPool.setUrl(url); - connectionPoolMap.put(dbname, connectionPool); + connectionPool.setUrl(getUrl(databaseName)); + connectionPoolMap.put(databaseName, connectionPool); return connectionPool.getConnection(); } catch (SQLException e) { - logger.error("cannot get connection for database {}: {}", dbname, e.getMessage()); + logger.error("cannot get connection for database {}: {}", databaseName, e.getMessage()); return null; } } @@ -183,8 +158,8 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { Operator op = operators.get(0); String storageUnit = task.getStorageUnit(); boolean isDummyStorageUnit = task.isDummyStorageUnit(); - Connection conn = getConnection(storageUnit, getUrl(storageUnit)); - if (conn == null) { + Connection conn = getConnection(storageUnit); + if (!isDummyStorageUnit && conn == null) { return new TaskExecuteResult( new NonExecutablePhysicalTaskException(String.format("cannot connect to storage unit %s", storageUnit))); } @@ -199,16 +174,16 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); } - return isDummyStorageUnit ? executeHistoryProjectTask(project, filter) : executeProjectTask(conn, project, filter); + return isDummyStorageUnit ? executeHistoryProjectTask(project, filter) : executeProjectTask(conn, storageUnit, project, filter); } else if (op.getType() == OperatorType.Insert) { Insert insert = (Insert) op; - return executeInsertTask(conn, insert); + return executeInsertTask(conn, storageUnit, insert); } else if (op.getType() == OperatorType.Delete) { Delete delete = (Delete) op; return executeDeleteTask(conn, storageUnit, delete); } return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task in postgresql ")); + new NonExecutablePhysicalTaskException("unsupported physical task in postgresql")); } @Override @@ -217,24 +192,22 @@ public List getTimeSeries() { Map extraParams = meta.getExtraParams(); try { Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); while (databaseSet.next()) { try { - String databaseName = databaseSet.getString(1);//获取数据库名称 + String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { continue; } - - Connection conn = getConnection(databaseName, getUrl(databaseName)); + Connection conn = getConnection(databaseName); if (conn == null) { - logger.error("cannot get connection for database {}!", databaseName); - return timeseries; + continue; } DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); while (tableSet.next()) { - String tableName = tableSet.getString(3); // 获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); + String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 @@ -259,71 +232,62 @@ public List getTimeSeries() { ); } } + columnSet.close(); } + tableSet.close(); conn.close(); } catch (SQLException e) { logger.error(e.getMessage()); } } + databaseSet.close(); + stmt.close(); } catch (SQLException e) { throw new RuntimeException(e); } return timeseries; } - - private long toHash(String s) { - char c[] = s.toCharArray(); - long hv = 0; - long base = 131; - for (int i = 0; i < c.length; i++) { - hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 - } - if (hv < 0) { - return -1 * hv; - } - return hv; - } - @Override public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { long minTime = Long.MAX_VALUE, maxTime = 0; List paths = new ArrayList<>(); try { Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取database名称 - Connection conn2 = getConnection(databaseName, getUrl(databaseName)); - if (conn2 == null) { + String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 + Connection conn = getConnection(databaseName); + if (conn == null) { continue; } - DatabaseMetaData databaseMetaData = conn2.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - String fields = ""; + String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + StringBuilder columnNames = new StringBuilder(); while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - paths.add(tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR)); - fields = fields + columnName + ","; //c1,c2,c3, + String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 + paths.add(databaseName + IGINX_SEPARATOR + tableName + IGINX_SEPARATOR + columnName); + columnNames.append(columnName); + columnNames.append(", "); // c1, c2, c3, } - fields = fields.substring(0, fields.lastIndexOf(",")); //c1,c2,c3 - - // 获取first - String firstQueryStatement = String.format("select concat(%s) from %s", fields, tableName); - Statement firstQueryStmt = conn2.createStatement(); - ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); - while (firstQuerySet.next()) { - String s = firstQuerySet.getString(1); - long logic_time = toHash(s); - minTime = Math.min(logic_time, minTime); - maxTime = Math.max(logic_time, maxTime); + columnNames = new StringBuilder(columnNames.substring(0, columnNames.length() - 2)); // c1, c2, c3 + + // 获取 key 的范围 + String statement = String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); + Statement concatStmt = conn.createStatement(); + ResultSet concatSet = concatStmt.executeQuery(statement); + while (concatSet.next()) { + String concatValue = concatSet.getString("concat"); + long time = toHash(concatValue); + minTime = Math.min(time, minTime); + maxTime = Math.max(time, maxTime); } } } + stmt.close(); } catch (SQLException e) { throw new RuntimeException(e); } @@ -332,7 +296,7 @@ public Pair getBoundaryOfStorage(String prefix) t new TimeInterval(minTime, maxTime + 1)); } - private Map splitAndMergeQueryPatterns(Connection conn, List patterns) throws SQLException { + private Map splitAndMergeQueryPatterns(String databaseName, Connection conn, List patterns) throws SQLException { // table name -> column names // 1 -> n Map tableNameToColumnNames = new HashMap<>(); @@ -367,7 +331,7 @@ private Map splitAndMergeQueryPatterns(Connection conn, List splitAndMergeQueryPatterns(Connection conn, List resultSets = new ArrayList<>(); ResultSet rs; Statement stmt; - Map tableNameToColumnNames = splitAndMergeQueryPatterns(conn, project.getPatterns()); + Map tableNameToColumnNames = splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); - String columnNames = entry.getValue(); - String[] parts = columnNames.split(", "); - StringBuilder statement = new StringBuilder(); + String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); + String statement = String.format(QUERY_STATEMENT, columnNames, getCompleteName(tableName), FilterTransformer.toString(filter)); try { stmt = conn.createStatement(); - statement.append("select time, "); - for (String part : parts) { - statement.append(getCompleteName(part)); - statement.append(", "); - } - statement = new StringBuilder(statement.substring(0, statement.length() - 2)); - statement.append(" from "); - statement.append(getCompleteName(tableName)); - statement.append(" where "); - statement.append(FilterTransformer.toString(filter)); - rs = stmt.executeQuery(statement.toString()); + rs = stmt.executeQuery(statement); logger.info("[Query] execute query: {}", statement); } catch (SQLException e) { logger.error("meet error when executing query {}: {}", statement, e.getMessage()); @@ -419,7 +372,7 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F } RowStream rowStream = new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream(resultSets, false, project.getTagFilter())); + new PostgreSQLQueryRowStream(resultSets, false, filter, project.getTagFilter())); conn.close(); return new TaskExecuteResult(rowStream); } catch (SQLException e) { @@ -429,126 +382,139 @@ private TaskExecuteResult executeProjectTask(Connection conn, Project project, F } } - private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { - try { - List resultSets = new ArrayList<>(); - List fields = new ArrayList<>(); - for (String path : project.getPatterns()) { - String[] l = path.split("\\."); - if (l.length < 3) { + private Map> splitAndMergeHistoryQueryPatterns(List patterns) throws SQLException { + // > + Map> splitResults = new HashMap<>(); + String databaseName; + String tableName; + String columnNames; + + for (String pattern : patterns) { + if (pattern.equals("*") || pattern.equals("*.*")) { + databaseName = "%"; + tableName = "%"; + columnNames = "%"; + } else { + String[] parts = pattern.split("\\" + IGINX_SEPARATOR); + if (parts.length > 3) { // 大于三级，不合法 + logger.error("wrong pattern: {}", pattern); continue; } - String database_table = path.substring(0, path.lastIndexOf(".")); - String dataBaseName = database_table.substring(0, database_table.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String tableName = database_table.substring(database_table.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String columnName = path.substring(path.lastIndexOf(".") + 1).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - Connection conn = getConnection(dataBaseName, getUrl(dataBaseName)); - if (conn == null) { + if (parts.length == 2 && !parts[1].equals("*")) { // 只有两级且第二级不为 *，则此 pattern 不合法，无法拆分出三级 continue; } - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - ResultSet columnSet = null; - if (path.equals("*.*")) { - tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - } else if (columnName.equals("*")) { - columnSet = databaseMetaData.getColumns(null, null, tableName, null); - } + databaseName = parts[0]; + tableName = parts[1].equals("*") ? "%" : parts[1]; + columnNames = parts[2].equals("*") ? "%" : parts[2]; + } - ResultSet columnSet_all = databaseMetaData.getColumns(null, null, tableName, null); - String hv = ""; - while (columnSet_all.next()) { - String columnName2 = columnSet_all.getString("COLUMN_NAME");//获取列名称 - hv = hv + columnName2 + ","; //c1,c2,c3, + if (databaseName.equals("%")) { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + while (databaseSet.next()) { + String tempDatabaseName = databaseSet.getString("DATNAME"); + if (tempDatabaseName.startsWith(DATABASE_PREFIX)) { + continue; + } + Connection conn = getConnection(tempDatabaseName); + if (conn == null) { + continue; + } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(tempDatabaseName, "public", tableName, new String[]{"TABLE"}); + while (tableSet.next()) { + String tempTableName = tableSet.getString("TABLE_NAME"); + ResultSet columnSet = databaseMetaData.getColumns(tempDatabaseName, "public", tempTableName, columnNames); + while (columnSet.next()) { + String tempColumnNames = columnSet.getString("COLUMN_NAME"); + Map tableNameToColumnNames = new HashMap<>(); + if (splitResults.containsKey(tempDatabaseName)) { + tableNameToColumnNames = splitResults.get(tempDatabaseName); + tempColumnNames = tableNameToColumnNames.get(tempTableName) + ", " + tempColumnNames; + } + tableNameToColumnNames.put(tempTableName, tempColumnNames); + splitResults.put(tempDatabaseName, tableNameToColumnNames); + } + } } - if (hv.equals("")) { + } else { + Connection conn = getConnection(databaseName); + if (conn == null) { continue; } - hv = hv.substring(0, hv.lastIndexOf(",")); + ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); + while (rs.next()) { + tableName = rs.getString("TABLE_NAME"); + columnNames = rs.getString("COLUMN_NAME"); + Map tableNameToColumnNames = new HashMap<>(); + if (splitResults.containsKey(databaseName)) { + tableNameToColumnNames = splitResults.get(databaseName); + columnNames = tableNameToColumnNames.get(tableName) + ", " + columnNames; + } + tableNameToColumnNames.put(tableName, columnNames); + splitResults.put(databaseName, tableNameToColumnNames); + } + } + } + + return splitResults; + } + private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { + try { + List resultSets = new ArrayList<>(); + ResultSet rs; + Connection conn = null; + Statement stmt; - if (tableSet == null && columnSet == null) { - Statement stmt = conn.createStatement(); - ResultSet rs = null; + Map> splitResults = splitAndMergeHistoryQueryPatterns(project.getPatterns()); + for (Map.Entry> splitEntry : splitResults.entrySet()) { + String databaseName = splitEntry.getKey(); + conn = getConnection(databaseName); + if (conn == null) { + continue; + } + for (Map.Entry entry : splitEntry.getValue().entrySet()) { + String tableName = entry.getKey(); + String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); + String statement = String.format(QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, columnNames, columnNames, getCompleteName(tableName)); try { - //String s = String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName); - rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, columnName, tableName)); - } catch (Exception e) { + stmt = conn.createStatement(); + rs = stmt.executeQuery(statement); + logger.info("[Query] execute query: {}", statement); + } catch (SQLException e) { + logger.error("meet error when executing query {}: {}", statement, e.getMessage()); continue; } resultSets.add(rs); - ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, columnName); - String typeName = "TEXT"; - if (columnSet_.next()) { - typeName = columnSet_.getString("TYPE_NAME");//列字段类型 - } - fields.add(new Field(dataBaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - } else if (tableSet == null && columnSet != null) { - ResultSet columnSet_ = databaseMetaData.getColumns(null, null, tableName, null); - while (columnSet_.next()) { - Statement stmt = conn.createStatement(); - String typeName = columnSet_.getString("TYPE_NAME");//列字段类型 - String field = columnSet_.getString("COLUMN_NAME"); - ResultSet rs = null; - try { - rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, tableName)); - } catch (Exception e) { - continue; - } - resultSets.add(rs); - fields.add(new Field(dataBaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - } - } else { - while (tableSet.next()) { - String table = tableSet.getString(3);//获取表名称 - ResultSet columnSet2 = databaseMetaData.getColumns(null, null, table, null); - while (columnSet2.next()) { - Statement stmt = conn.createStatement(); - String field = columnSet2.getString("COLUMN_NAME"); - String typeName = columnSet2.getString("TYPE_NAME");//列字段类型 - ResultSet rs = null; - try { - rs = stmt.executeQuery(String.format("select concat(%s) as time,%s from %s", hv, field, table)); - } catch (Exception e) { - continue; - } - resultSets.add(rs); - fields.add(new Field(dataBaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + table.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromPostgreSQL(typeName))); - } - } } } - RowStream rowStream = new ClearEmptyRowStreamWrapper(new PostgreSQLQueryRowStream(resultSets, true, project.getTagFilter())); + + RowStream rowStream = new ClearEmptyRowStreamWrapper( + new PostgreSQLQueryRowStream(resultSets, true, filter, project.getTagFilter())); + if (conn != null) { + conn.close(); + } return new TaskExecuteResult(rowStream); } catch (SQLException e) { - logger.info(e.getMessage()); + logger.error(e.getMessage()); return new TaskExecuteResult( new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); } } - - private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { + private TaskExecuteResult executeInsertTask(Connection conn, String storageUnit, Insert insert) { DataView dataView = insert.getData(); Exception e = null; switch (dataView.getRawDataType()) { case Row: case NonAlignedRow: - e = insertNonAlignedRowRecords(conn, (RowDataView) dataView); + e = insertNonAlignedRowRecords(conn, storageUnit, (RowDataView) dataView); break; case Column: case NonAlignedColumn: - e = insertNonAlignedColumnRecords(conn, (ColumnDataView) dataView); + e = insertNonAlignedColumnRecords(conn, storageUnit, (ColumnDataView) dataView); break; } if (e != null) { @@ -558,7 +524,7 @@ private TaskExecuteResult executeInsertTask(Connection conn, Insert insert) { return new TaskExecuteResult(null, null); } - private void createOrAlterTables(Connection conn, List paths, List> tagsList, List dataTypeList) { + private void createOrAlterTables(Connection conn, String storageUnit, List paths, List> tagsList, List dataTypeList) { for (int i = 0; i < paths.size(); i++) { String path = paths.get(i); Map tags = new HashMap<>(); @@ -566,43 +532,42 @@ private void createOrAlterTables(Connection conn, List paths, List>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> @@ -617,8 +582,8 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) for (int j = 0; j < data.getPathNum(); j++) { String path = data.getPath(j); DataType dataType = data.getDataType(j); - String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); + String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String columnName = path.substring(path.lastIndexOf('.') + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { tags = data.getTags(j); @@ -626,9 +591,9 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) StringBuilder columnKeys = new StringBuilder(); List columnValues = new ArrayList<>(); - if (tableToColumnEntries.containsKey(table)) { - columnKeys = new StringBuilder(tableToColumnEntries.get(table).k); - columnValues = tableToColumnEntries.get(table).v; + if (tableToColumnEntries.containsKey(tableName)) { + columnKeys = new StringBuilder(tableToColumnEntries.get(tableName).k); + columnValues = tableToColumnEntries.get(tableName).v; } String value = "null"; @@ -639,17 +604,17 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) value = data.getValue(i, index).toString(); } index++; - if (tableHasData.containsKey(table)) { - tableHasData.get(table)[i - cnt] = true; + if (tableHasData.containsKey(tableName)) { + tableHasData.get(tableName)[i - cnt] = true; } else { boolean[] hasData = new boolean[size]; hasData[i - cnt] = true; - tableHasData.put(table, hasData); + tableHasData.put(tableName, hasData); } } if (firstRound) { - columnKeys.append(toFullName(field, tags)).append(", "); + columnKeys.append(toFullName(columnName, tags)).append(", "); } if (i - cnt < columnValues.size()) { @@ -658,17 +623,17 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) columnValues.add(data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 } - tableToColumnEntries.put(table, new Pair<>(columnKeys.toString(), columnValues)); + tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); } firstRound = false; } for (Map.Entry entry : tableHasData.entrySet()) { - String table = entry.getKey(); + String tableName = entry.getKey(); boolean[] hasData = entry.getValue(); - String columnKeys = tableToColumnEntries.get(table).k; - List columnValues = tableToColumnEntries.get(table).v; + String columnKeys = tableToColumnEntries.get(tableName).k; + List columnValues = tableToColumnEntries.get(tableName).v; boolean needToInsert = false; for (int i = hasData.length - 1; i >= 0; i--) { if (!hasData[i]) { @@ -678,7 +643,7 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) } } if (needToInsert) { - tableToColumnEntries.put(table, new Pair<>(columnKeys, columnValues)); + tableToColumnEntries.put(tableName, new Pair<>(columnKeys, columnValues)); } } @@ -689,6 +654,7 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) cnt += size; } + stmt.close(); conn.close(); } catch (SQLException e) { logger.error(e.getMessage()); @@ -698,13 +664,13 @@ private Exception insertNonAlignedRowRecords(Connection conn, RowDataView data) return null; } - private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView data) { + private Exception insertNonAlignedColumnRecords(Connection conn, String storageUnit, ColumnDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); // 创建表 - createOrAlterTables(conn, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + createOrAlterTables(conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); // 插入数据 Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> @@ -717,8 +683,8 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); DataType dataType = data.getDataType(i); - String table = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); + String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String columnName = path.substring(path.lastIndexOf('.') + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { tags = data.getTags(i); @@ -728,9 +694,9 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView StringBuilder columnKeys = new StringBuilder(); List columnValues = new ArrayList<>(); - if (tableToColumnEntries.containsKey(table)) { - columnKeys = new StringBuilder(tableToColumnEntries.get(table).k); - columnValues = tableToColumnEntries.get(table).v; + if (tableToColumnEntries.containsKey(tableName)) { + columnKeys = new StringBuilder(tableToColumnEntries.get(tableName).k); + columnValues = tableToColumnEntries.get(tableName).v; } int index = 0; @@ -746,12 +712,12 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView value = data.getValue(i, index).toString(); } index++; - if (tableHasData.containsKey(table)) { - tableHasData.get(table)[j - cnt] = true; + if (tableHasData.containsKey(tableName)) { + tableHasData.get(tableName)[j - cnt] = true; } else { boolean[] hasData = new boolean[size]; hasData[j - cnt] = true; - tableHasData.put(table, hasData); + tableHasData.put(tableName, hasData); } } @@ -764,17 +730,17 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView pathIndexToBitmapIndex.put(i, index); if (firstRound) { - columnKeys.append(toFullName(field, tags)).append(", "); + columnKeys.append(toFullName(columnName, tags)).append(", "); } - tableToColumnEntries.put(table, new Pair<>(columnKeys.toString(), columnValues)); + tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); } for (Map.Entry entry : tableHasData.entrySet()) { - String table = entry.getKey(); + String tableName = entry.getKey(); boolean[] hasData = entry.getValue(); - String columnKeys = tableToColumnEntries.get(table).k; - List columnValues = tableToColumnEntries.get(table).v; + String columnKeys = tableToColumnEntries.get(tableName).k; + List columnValues = tableToColumnEntries.get(tableName).v; boolean needToInsert = false; for (int i = hasData.length - 1; i >= 0; i--) { if (!hasData[i]) { @@ -784,7 +750,7 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView } } if (needToInsert) { - tableToColumnEntries.put(table, new Pair<>(columnKeys, columnValues)); + tableToColumnEntries.put(tableName, new Pair<>(columnKeys, columnValues)); } } @@ -796,9 +762,10 @@ private Exception insertNonAlignedColumnRecords(Connection conn, ColumnDataView firstRound = false; cnt += size; } + stmt.close(); conn.close(); } catch (SQLException e) { - logger.info("error", e); + logger.error(e.getMessage()); return e; } @@ -813,51 +780,33 @@ private void executeBatchInsert(Statement stmt, Map a + ", " + b).orElse("")); + statement.append(") VALUES"); + statement.append(values.stream().map(x -> " (" + x.substring(0, x.length() - 2) + ")").reduce((a, b) -> a + ", " + b).orElse("")); + statement.append(" ON CONFLICT (time) DO UPDATE SET "); if (hasMultipleRows) { - insertStatement.append("("); // 只有一列不加括号 - } - for (String part : parts) { - insertStatement.append(getCompleteName(part)); - insertStatement.append(", "); + statement.append("("); // 只有一列不加括号 } - insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.length() - 2)); + statement.append(Arrays.stream(parts).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("")); if (hasMultipleRows) { - insertStatement.append(")"); // 只有一列不加括号 + statement.append(")"); // 只有一列不加括号 } - insertStatement.append(" = "); + statement.append(" = "); if (hasMultipleRows) { - insertStatement.append("("); // 只有一列不加括号 - } - for (String part : parts) { - insertStatement.append("excluded."); - insertStatement.append(getCompleteName(part)); - insertStatement.append(", "); + statement.append("("); // 只有一列不加括号 } - insertStatement = new StringBuilder(insertStatement.substring(0, insertStatement.length() - 2)); + statement.append(Arrays.stream(parts).map(x -> "excluded." + getCompleteName(x)).reduce((a, b) -> a + ", " + b).orElse("")); if (hasMultipleRows) { - insertStatement.append(")"); // 只有一列不加括号 + statement.append(")"); // 只有一列不加括号 } + statement.append(";"); -// logger.info("[Insert] execute insert: {}", insertStatement); - stmt.addBatch(insertStatement.toString()); +// logger.info("[Insert] execute insert: {}", statement); + stmt.addBatch(statement.toString()); } stmt.executeBatch(); } @@ -877,12 +826,13 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { conn.close(); - Connection postgresConn = getConnection("postgres", getUrl("postgres")); // 正在使用的数据库无法被删除，因此需要切换到名为postgres的默认数据库 + Connection postgresConn = getConnection("postgres"); // 正在使用的数据库无法被删除，因此需要切换到名为 postgres 的默认数据库 if (postgresConn != null) { stmt = postgresConn.createStatement(); - statement = String.format("drop database %s", storageUnit); + statement = String.format(DROP_DATABASE_STATEMENT, storageUnit); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 删除数据库 + stmt.close(); postgresConn.close(); return new TaskExecuteResult(null, null); } else { @@ -893,9 +843,9 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, for (Pair pair : deletedPaths) { tableName = pair.k; columnName = pair.v; - tableSet = databaseMetaData.getTables(null, "%", tableName, new String[]{"TABLE"}); + tableSet = databaseMetaData.getTables(storageUnit, "public", tableName, new String[]{"TABLE"}); if (tableSet.next()) { - statement = String.format("alter table %s drop column if exists %s", getCompleteName(tableName), getCompleteName(columnName)); + statement = String.format(DROP_COLUMN_STATEMENT, getCompleteName(tableName), getCompleteName(columnName)); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 删除列 } @@ -906,10 +856,10 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, for (Pair pair : deletedPaths) { tableName = pair.k; columnName = pair.v; - columnSet = databaseMetaData.getColumns(null, "%", tableName, columnName); + columnSet = databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); if (columnSet.next()) { for (TimeRange timeRange : delete.getTimeRanges()) { - statement = String.format("update %s set %s = null where (time >= %d and time < %d)", getCompleteName(tableName), getCompleteName(columnName), + statement = String.format(UPDATE_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), timeRange.getBeginTime(), timeRange.getEndTime()); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 将目标列的目标范围的值置为空 diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java similarity index 91% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java rename to dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index 205d872bf1..83ad3f74de 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -1,4 +1,4 @@ -package cn.edu.tsinghua.iginx.postgresql.entity; +package cn.edu.tsinghua.iginx.postgresql.query.entity; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; @@ -7,6 +7,7 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; @@ -19,8 +20,10 @@ import java.sql.SQLException; import java.util.*; +import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; public class PostgreSQLQueryRowStream implements RowStream { @@ -33,6 +36,8 @@ public class PostgreSQLQueryRowStream implements RowStream { private final boolean isDummy; + private final Filter filter; + private boolean[] gotNext; // 标记每个结果集是否已经获取到下一行，如果是，则在下次调用 next() 时无需再调用该结果集的 next() private long[] cachedTimestamps; // 缓存每个结果集当前的 time 列的值 @@ -47,9 +52,10 @@ public class PostgreSQLQueryRowStream implements RowStream { private boolean hasCachedRow; - public PostgreSQLQueryRowStream(List resultSets, boolean isDummy, TagFilter tagFilter) throws SQLException { + public PostgreSQLQueryRowStream(List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter) throws SQLException { this.resultSets = resultSets; this.isDummy = isDummy; + this.filter = filter; if (resultSets.isEmpty()) { this.header = new Header(Field.KEY, Collections.emptyList()); @@ -131,7 +137,7 @@ public boolean hasNext() { if (!hasCachedRow) { cacheOneRow(); } - } catch (SQLException e) { + } catch (SQLException | PhysicalException e) { logger.error(e.getMessage()); } @@ -149,13 +155,13 @@ public Row next() throws PhysicalException { hasCachedRow = false; cachedRow = null; return row; - } catch (SQLException e) { + } catch (SQLException | PhysicalException e) { logger.error(e.getMessage()); throw new RowFetchException(e); } } - private void cacheOneRow() throws SQLException { + private void cacheOneRow() throws SQLException, PhysicalException { boolean hasNext = false; long timestamp; Object[] values = new Object[header.getFieldSize()]; @@ -226,22 +232,12 @@ private void cacheOneRow() throws SQLException { startIndex = endIndex; } cachedRow = new Row(header, timestamp, values); + if (isDummy && !validate(filter, cachedRow)) { + cacheOneRow(); + } } else { cachedRow = null; } hasCachedRow = true; } - - private long toHash(String s) { - char c[] = s.toCharArray(); - long hv = 0; - long base = 131; - for (int i = 0; i < c.length; i++) { - hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 - } - if (hv < 0) { - return -1 * hv; - } - return hv; - } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 22c3dcc4c0..5fda6ff469 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -5,4 +5,36 @@ public class Constants { public static final String IGINX_SEPARATOR = "."; public static final String POSTGRESQL_SEPARATOR = "\u2E82"; + + public static final int BATCH_SIZE = 10000; + + public static final String USERNAME = "username"; + + public static final String PASSWORD = "password"; + + public static final String DEFAULT_USERNAME = "postgres"; + + public static final String DEFAULT_PASSWORD = "postgres"; + + public static final String DATABASE_PREFIX = "unit"; + + public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; + + public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; + + public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; + + public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; + + public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; + + public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; + + public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; + + public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; + + public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; + + public static final String UPDATE_STATEMENT = "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java new file mode 100644 index 0000000000..8428ebd1c2 --- /dev/null +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java @@ -0,0 +1,17 @@ +package cn.edu.tsinghua.iginx.postgresql.tools; + +public class HashUtils { + + public static long toHash(String s) { + char c[] = s.toCharArray(); + long hv = 0; + long base = 131; + for (int i = 0; i < c.length; i++) { + hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 + } + if (hv < 0) { + return -1 * hv; + } + return hv; + } +} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java index cc9b3bcbac..e2af27de7b 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -4,10 +4,8 @@ import java.util.*; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; -// TODO 目前未考虑 annotation public class TagKVUtils { public static Pair> splitFullName(String fullName) { From 35a43b4ab653237deb0b99ea249e58bfbc690a39 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 21 Mar 2023 17:28:40 +0800 Subject: [PATCH 45/94] Add expansion test (#4) * Add expansion test * Fix clear data * Add pg to pom * Run another pg --- .github/actions/dbWriter/action.yml | 8 +- .github/postgresql.sh | 24 ++- .github/postgresql_macos.sh | 20 ++- .github/workflows/DB-CE.yml | 6 +- test/pom.xml | 5 + ...tgreSQLHistoryDataCapacityExpansionIT.java | 146 ++++++++++++++++++ .../PostgreSQLHistoryDataGeneratorTest.java | 110 +++++++++++++ 7 files changed, 313 insertions(+), 6 deletions(-) create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java diff --git a/.github/actions/dbWriter/action.yml b/.github/actions/dbWriter/action.yml index c186e546e2..37b985467a 100644 --- a/.github/actions/dbWriter/action.yml +++ b/.github/actions/dbWriter/action.yml @@ -22,4 +22,10 @@ runs: name: Write history Data shell: bash run: | - mvn test -q -Dtest=InfluxDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false \ No newline at end of file + mvn test -q -Dtest=InfluxDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false + + - if: inputs.version=='PostgreSQL' + name: Write history Data + shell: bash + run: | + mvn test -q -Dtest=PostgreSQLHistoryDataGeneratorTest#${{inputs.Test-Way}} -DfailIfNoTests=false \ No newline at end of file diff --git a/.github/postgresql.sh b/.github/postgresql.sh index 44a157b876..6512413c01 100644 --- a/.github/postgresql.sh +++ b/.github/postgresql.sh @@ -22,8 +22,28 @@ sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" -sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/initdb -D /var/lib/postgresql/15/main --auth-local peer --auth-host scram-sha-256 --no-instructions'" +sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/initdb -D /var/lib/postgresql/15/main --auth trust --no-instructions'" sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/15/main start'" -sh -c "sudo su - postgres -c 'psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" +sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" + +sh -c "sudo mkdir -p /usr/lib/postgresql2" + +sh -c "sudo chmod -R 777 /usr/lib/postgresql/15" + +sh -c "sudo chmod -R 777 /usr/lib/postgresql2" + +sh -c "sudo cp -R /usr/lib/postgresql/15 /usr/lib/postgresql2" + +sh -c "sudo mkdir -p /var/lib/postgresql2/15/main" + +sh -c "sudo chown -R postgres /var/lib/postgresql2/15/main" + +sh -c "sudo chmod -R 777 /var/lib/postgresql2/15/main" + +sh -c "sudo su - postgres -c '/usr/lib/postgresql2/15/bin/initdb -D /var/lib/postgresql2/15/main --auth trust --no-instructions'" + +sh -c "sudo su - postgres -c '/usr/lib/postgresql2/15/bin/pg_ctl -D /var/lib/postgresql2/15/main -o \"-F -p 5433\" start'" + +sh -c "sudo su - postgres -c '/usr/lib/postgresql2/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh index 7e5bcf148a..6caa70a1f6 100644 --- a/.github/postgresql_macos.sh +++ b/.github/postgresql_macos.sh @@ -32,8 +32,24 @@ sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" -sh -c "cd pgsql/bin; sudo -u postgres ./initdb -D /var/lib/postgresql/15/main" +sh -c "cd pgsql/bin; sudo -u postgres ./initdb -D /var/lib/postgresql/15/main --auth trust --no-instructions" sh -c "cd pgsql/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql/15/main start" -sh -c "sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\"" +sh -c "cd pgsql/bin; sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\"" + +sh -c "sudo mkdir pgsql2" + +sh -c "sudo cp -R pgsql pgsql2" + +sh -c "sudo mkdir -p /var/lib/postgresql2/15/main" + +sh -c "sudo chown -R postgres /var/lib/postgresql2/15/main" + +sh -c "sudo chmod -R 777 /var/lib/postgresql2/15/main" + +sh -c "cd pgsql2/bin; sudo -u postgres ./initdb -D /var/lib/postgresql2/15/main --auth trust --no-instructions" + +sh -c "cd pgsql2/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql2/15/main start" + +sh -c "cd pgsql2/bin; sudo -u postgres ./psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\"" diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 3b4ced6dec..ef1bd50271 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -21,7 +21,7 @@ jobs: java: [ 8 ] python-version: [ "3.7" ] os: [ ubuntu-latest, macos-latest ] - DB-name: ["IoTDB12", "InfluxDB"] + DB-name: ["IoTDB12", "InfluxDB", "PostgreSQL"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 @@ -120,6 +120,10 @@ jobs: mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false + elif [ "${{matrix.DB-name}}" == "PostgreSQL" ]; then + mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false elif [ "${{matrix.DB-name}}" == "InfluxDB" ]; then mvn test -q -Dtest=InfluxDBHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false else diff --git a/test/pom.xml b/test/pom.xml index 520d213fe2..925a6c5702 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -46,6 +46,11 @@ 0.5.7 true + + org.postgresql + postgresql + 42.5.4 + diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java new file mode 100644 index 0000000000..7da4092211 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java @@ -0,0 +1,146 @@ +package cn.edu.tsinghua.iginx.integration.expansion.postgresql; + +import cn.edu.tsinghua.iginx.integration.expansion.CapacityExpansionIT; +import cn.edu.tsinghua.iginx.integration.expansion.unit.SQLTestTools; +import cn.edu.tsinghua.iginx.thrift.RemovedStorageEngineInfo; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class PostgreSQLHistoryDataCapacityExpansionIT extends CapacityExpansionIT { + public PostgreSQLHistoryDataCapacityExpansionIT() { + super("postgresql"); + } + @Test + public void schemaPrefix() throws Exception { + testSchemaPrefix(); + } + @Test + public void testSchemaPrefix() throws Exception { + session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, schema_prefix:expansion, is_read_only:true\");"); + + String statement = "select * from expansion.ln.wf03"; + String expect = "ResultSets:\n" + + "+---+-----------------------------+----------------------------------+\n" + + "|key|expansion.ln.wf03.wt01.status|expansion.ln.wf03.wt01.temperature|\n" + + "+---+-----------------------------+----------------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+-----------------------------+----------------------------------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + statement = "count points"; + expect = "Points num: 3\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + } + + @Test + public void testDataPrefix() throws Exception { + session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:true\");"); + + String statement = "select * from test"; + String expect = "ResultSets:\n" + + "+---+---------------------+--------------------------+\n" + + "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" + + "+---+---------------------+--------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+---------------------+--------------------------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + statement = "select * from ln"; + expect = "ResultSets:\n" + + "+---+\n" + + "|key|\n" + + "+---+\n" + + "+---+\n" + + "Empty set.\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + } + + @Test + public void testAddSameDataPrefixWithDiffSchemaPrefix_AND_testRemoveHistoryDataSource() throws Exception { + session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p1, is_read_only:true\");"); + session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p2, is_read_only:true\");"); + + String statement = "select * from p1.test"; + String expect = "ResultSets:\n" + + "+---+------------------------+-----------------------------+\n" + + "|key|p1.test.wf03.wt01.status|p1.test.wf03.wt01.temperature|\n" + + "+---+------------------------+-----------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+------------------------+-----------------------------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + statement = "select * from p2.test"; + expect = "ResultSets:\n" + + "+---+------------------------+-----------------------------+\n" + + "|key|p2.test.wf03.wt01.status|p2.test.wf03.wt01.temperature|\n" + + "+---+------------------------+-----------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+------------------------+-----------------------------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + statement = "select * from test"; + expect = "ResultSets:\n" + + "+---+\n" + + "|key|\n" + + "+---+\n" + + "+---+\n" + + "Empty set.\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:false\");"); + statement = "select * from test"; + expect = "ResultSets:\n" + + "+---+---------------------+--------------------------+\n" + + "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" + + "+---+---------------------+--------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+---------------------+--------------------------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + List removedStorageEngineList = new ArrayList<>(); + removedStorageEngineList.add(new RemovedStorageEngineInfo("127.0.0.1", 5433, "", "test")); + session.removeHistoryDataSource(removedStorageEngineList); + statement = "select * from test"; + expect = "ResultSets:\n" + + "+---+\n" + + "|key|\n" + + "+---+\n" + + "+---+\n" + + "Empty set.\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + removedStorageEngineList.set(0, new RemovedStorageEngineInfo("127.0.0.1", 5433, "p2", "test")); + sessionPool.removeHistoryDataSource(removedStorageEngineList); + statement = "select * from p2.test"; + expect = "ResultSets:\n" + + "+---+\n" + + "|key|\n" + + "+---+\n" + + "+---+\n" + + "Empty set.\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + + session.executeSql("remove historydataresource (\"127.0.0.1\", 5433, \"p1\", \"test\")"); + statement = "select * from p1.test"; + expect = "ResultSets:\n" + + "+---+\n" + + "|key|\n" + + "+---+\n" + + "+---+\n" + + "Empty set.\n"; + SQLTestTools.executeAndCompare(session, statement, expect); + } + + +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java new file mode 100644 index 0000000000..51b23c36aa --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -0,0 +1,110 @@ +package cn.edu.tsinghua.iginx.integration.expansion.postgresql; + +import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator { + + private static final Logger logger = LoggerFactory.getLogger(PostgreSQLHistoryDataGeneratorTest.class); + + private static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; + + private static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s, PRIMARY KEY(time));"; + + private static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; + + private static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; + + private static final int PORT_A = 5432; + + private static final int PORT_B = 5433; + + private static final String USERNAME = "postgres"; + + private static final String PASSWORD = "postgres"; + + private static final String DATABASE_NAME = "ln"; + + private Connection connect(int port) { + try { + String url = String.format("jdbc:postgresql://127.0.0.1:%d/", port); + Class.forName("org.postgresql.Driver"); + return DriverManager.getConnection(url, USERNAME, PASSWORD); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Test + public void clearData() { + try { + Connection connA = connect(PORT_A); + Statement stmtA = connA.createStatement(); + stmtA.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); + stmtA.close(); + connA.close(); + + Connection connB = connect(PORT_B); + Statement stmtB = connB.createStatement(); + stmtB.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); + stmtB.close(); + stmtB.close(); + } catch (Exception e) { + logger.error(e.getMessage()); + } + logger.info("clear data success!"); + } + + @Test + public void writeHistoryDataToA() throws Exception { + Connection conn = connect(PORT_A); + if (conn == null) { + logger.error("cannot connect to 5432!"); + return; + } + + Statement stmt = conn.createStatement(); + try { + stmt.execute(String.format(CREATE_DATABASE_STATEMENT, DATABASE_NAME)); + } catch (SQLException e) { + logger.info("database ln exists!"); + } + + stmt.execute(String.format(CREATE_TABLE_STATEMENT, "wf01", "status boolean, temperature float8")); + + stmt.execute(String.format(INSERT_STATEMENT, "wf01", "(100, true, null), (200, false, 20.71)")); + + stmt.close(); + conn.close(); + logger.info("write data to 127.0.0.1:5432 success!"); + } + + @Test + public void writeHistoryDataToB() throws Exception { + Connection conn = connect(PORT_B); + if (conn == null) { + logger.error("cannot connect to 5433!"); + return; + } + + Statement stmt = conn.createStatement(); + try { + stmt.execute(String.format(CREATE_DATABASE_STATEMENT, DATABASE_NAME)); + } catch (SQLException e) { + logger.info("database ln exists!"); + } + + stmt.execute(String.format(CREATE_TABLE_STATEMENT, "wf03", "status boolean, temperature float8")); + + stmt.execute(String.format(INSERT_STATEMENT, "wf03", "(77, true, null), (200, false, 77.71)")); + + logger.info("write data to 127.0.0.1:5433 success!"); + } +} From d9c3ff1fec65cb153fb5af167e849eeadd0de63c Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 21 Mar 2023 17:38:19 +0800 Subject: [PATCH 46/94] Restore ByteUtils --- .../java/cn/edu/tsinghua/iginx/utils/ByteUtils.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java index 338038b99e..829bf422f9 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/ByteUtils.java @@ -239,17 +239,7 @@ public static ByteBuffer getRowByteBuffer(Object[] values, List dataTy buffer.putInt((int) value); break; case LONG: - try { - buffer.putLong((long) value); - } catch (Exception e) { - if (value.getClass().getTypeName().contains("INT")) { //integer - buffer.putLong(((Number) value).longValue()); - } - if (value.getClass().getTypeName().contains("Timestamp")) { //pg timestamp - java.sql.Timestamp ts2 = java.sql.Timestamp.valueOf(value.toString()); - buffer.putLong(ts2.getTime()); - } - } + buffer.putLong((long) value); break; case FLOAT: buffer.putFloat((float) value); From 6f53b88419654afcbc7de67f4eca1e0ecf701990 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 21 Mar 2023 21:46:36 +0800 Subject: [PATCH 47/94] Fix actions --- .github/postgresql_macos.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh index 6caa70a1f6..f7b6bb7116 100644 --- a/.github/postgresql_macos.sh +++ b/.github/postgresql_macos.sh @@ -38,8 +38,6 @@ sh -c "cd pgsql/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql/15/main st sh -c "cd pgsql/bin; sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\"" -sh -c "sudo mkdir pgsql2" - sh -c "sudo cp -R pgsql pgsql2" sh -c "sudo mkdir -p /var/lib/postgresql2/15/main" From a0dd5b7b71fe4e47a547c6234ca7a57b3d505315 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 21 Mar 2023 21:53:22 +0800 Subject: [PATCH 48/94] Fix actions --- .github/postgresql_macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh index f7b6bb7116..9d371128cd 100644 --- a/.github/postgresql_macos.sh +++ b/.github/postgresql_macos.sh @@ -48,6 +48,6 @@ sh -c "sudo chmod -R 777 /var/lib/postgresql2/15/main" sh -c "cd pgsql2/bin; sudo -u postgres ./initdb -D /var/lib/postgresql2/15/main --auth trust --no-instructions" -sh -c "cd pgsql2/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql2/15/main start" +sh -c "cd pgsql2/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql2/15/main -o \"-F -p 5433\" start" sh -c "cd pgsql2/bin; sudo -u postgres ./psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\"" From 268855beb6d7e24faf0cb3b126d50a49fd5b2ad4 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Tue, 4 Apr 2023 22:15:57 +0800 Subject: [PATCH 49/94] relationdb --- conf/config.properties | 6 +- dataSources/relationdb/pom.xml | 114 +++ .../iginx/relationdb/RelationDBStorage.java | 949 ++++++++++++++++++ .../iginx/relationdb/conf/RelationDBConf.java | 17 + .../iginx/relationdb/conf/dmdbConf.java | 19 + .../iginx/relationdb/conf/mysqlConf.java | 20 + .../entity/RelationDBQueryRowStream.java | 243 +++++ .../iginx/relationdb/tools/Constants.java | 39 + .../relationdb/tools/DataTypeTransformer.java | 43 + .../relationdb/tools/FilterTransformer.java | 69 ++ .../iginx/relationdb/tools/HashUtils.java | 17 + .../relationdb/tools/TagFilterUtils.java | 54 + .../iginx/relationdb/tools/TagKVUtils.java | 42 + pom.xml | 1 + 14 files changed, 1631 insertions(+), 2 deletions(-) create mode 100644 dataSources/relationdb/pom.xml create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java create mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java diff --git a/conf/config.properties b/conf/config.properties index 4845f5678d..8133d8d135 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,18 +14,20 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres #storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData +#storageEngineList=127.0.0.1#5236#dmdb#username=SYSDBA#password=SYSDBA001#sessionPoolSize=20#has_data=false#is_read_only=false +storageEngineList=127.0.0.1#3306#mysql#username=root#sessionPoolSize=20#has_data=false#is_read_only=false # 写入的副本个数 replicaNum=0 # 底层数据库类名 -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mysql=cn.edu.tsinghua.iginx.relationdb.RelationDBStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage # 内存任务执行线程池 diff --git a/dataSources/relationdb/pom.xml b/dataSources/relationdb/pom.xml new file mode 100644 index 0000000000..f586cf2aaf --- /dev/null +++ b/dataSources/relationdb/pom.xml @@ -0,0 +1,114 @@ + + + 4.0.0 + + + iginx + cn.edu.tsinghua + ${revision} + ../../pom.xml + + + relationdb + IGinX RelationDB + + + + + + + + + + + + + + + + + + + + + + + + 8 + 8 + + + + + cn.edu.tsinghua + iginx-core + ${project.version} + provided + + + org.postgresql + postgresql + 42.5.4 + + + com.mysql + mysql-connector-j + 8.0.32 + + + com.huawei.dmdb + dmdb-jdbc + 1.7.1.1 + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies + package + + copy-dependencies + + + ../../core/target/iginx-core-${project.version}/driver/relationdb/ + + provided + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.7 + + + copy-native-libraries + package + + run + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java new file mode 100644 index 0000000000..8eca8616bd --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java @@ -0,0 +1,949 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package cn.edu.tsinghua.iginx.relationdb; + +import cn.edu.tsinghua.iginx.engine.physical.exception.NonExecutablePhysicalTaskException; +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; +import cn.edu.tsinghua.iginx.engine.physical.exception.StorageInitializationException; +import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; +import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Timeseries; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; +import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; +import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; +import cn.edu.tsinghua.iginx.engine.shared.TimeRange; +import cn.edu.tsinghua.iginx.engine.shared.data.read.ClearEmptyRowStreamWrapper; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; +import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; +import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; +import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; +import cn.edu.tsinghua.iginx.engine.shared.operator.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; +import cn.edu.tsinghua.iginx.metadata.entity.*; +import cn.edu.tsinghua.iginx.relationdb.query.entity.RelationDBQueryRowStream; +import cn.edu.tsinghua.iginx.relationdb.tools.DataTypeTransformer; +import cn.edu.tsinghua.iginx.relationdb.tools.FilterTransformer; +import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.utils.Pair; +import cn.edu.tsinghua.iginx.utils.StringUtils; +import cn.edu.tsinghua.iginx.relationdb.conf.mysqlConf; +import cn.edu.tsinghua.iginx.relationdb.conf.dmdbConf; +//import org.postgresql.ds.PGConnectionPoolDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.sql.*; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; +import com.huawei.dmdb.jdbc.DmdbConnectionPoolDataSource; +import com.mysql.cj.jdbc.MysqlConnectionPoolDataSource; + +//import static cn.edu.tsinghua.iginx.relationDB.tools.Constants.*; +import static cn.edu.tsinghua.iginx.relationdb.tools.HashUtils.toHash; +import static cn.edu.tsinghua.iginx.relationdb.tools.TagKVUtils.splitFullName; +import static cn.edu.tsinghua.iginx.relationdb.tools.TagKVUtils.toFullName; + +public class RelationDBStorage implements IStorage { + + private static final Logger logger = LoggerFactory.getLogger(RelationDBStorage.class); + + private final StorageEngineMeta meta; + + private final Map connectionPoolMap = new ConcurrentHashMap<>(); + + private final Connection connection; + + public final RelationDBConf r; //关系数据库配置 + + public final String IGINX_SEPARATOR; + public final String RELATIONDB_SEPARATOR; + + public RelationDBStorage(StorageEngineMeta meta) throws StorageInitializationException { + this.meta = meta; + if (!testConnection()) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); + } + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + + //找到具体的关系字类并创建实例 + String engineName = meta.getStorageEngine(); + String className = engineName + "Conf"; //关系子类的配置 类名 + Class clr = (Class)Class.forName("cn.edu.tsinghua.iginx.relationDB.conf."+className); + r = clr.getConstructor().newInstance(); //子类实例 + + this.IGINX_SEPARATOR = r.IGINX_SEPARATOR; + this.RELATIONDB_SEPARATOR = r.RELATIONDB_SEPARATOR; + + String connUrl = String + .format(r.CONNECTION_URL, meta.getIp(), meta.getPort(), + username, password); + try { + connection = DriverManager.getConnection(connUrl); + } catch (SQLException e) { + throw new StorageInitializationException("cannot connect to " + meta); + } + } + + private boolean testConnection() { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); + String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String connUrl = String + .format(r.CONNECTION_URL, meta.getIp(), meta.getPort(), + username, password); + try { + Class.forName(r.DRIVER_URL); + DriverManager.getConnection(connUrl); + return true; + } catch (SQLException | ClassNotFoundException e) { + return false; + } + } + + private String getUrl(String databaseName) { + Map extraParams = meta.getExtraParams(); + String username = extraParams.getOrDefault(r.USERNAME, r.DEFAULT_USERNAME); + String password = extraParams.getOrDefault(r.PASSWORD, r.DEFAULT_PASSWORD); + String connUrl = String + .format(r.CONNECTION_URL, meta.getIp(), meta.getPort(), databaseName, + username, password); + return connUrl; + } + + private Connection getConnection(String databaseName) { + if (!r.NEED_CHANGE_DB) { //有的数据库不需要更换连接 + return connection; + } + if (databaseName.startsWith("dummy")) { + return null; + } + String ignoredDB = r.IGNORED_DATABASE; //需要忽略的数据库（系统自动维护，用户没有修改权限的数据库等） + String[] ignoredDBs = ignoredDB.split(","); + for(String db:ignoredDBs) { + if (databaseName.equalsIgnoreCase(db)) { + return null; + } + } + + try { + Statement stmt = connection.createStatement(); + stmt.execute(String.format(r.CREATE_DATABASE_STATEMENT, databaseName)); + stmt.close(); + } catch (SQLException e) { +// logger.info("database {} exists!", databaseName); + } + + try { + if (connectionPoolMap.containsKey(databaseName)) { + return connectionPoolMap.get(databaseName).getConnection(); + } + Class ConnectionPoolDataSource = Class.forName(r.CONNECTION_POOL_CLASS_NAME); + Object connectionPool = ConnectionPoolDataSource.getConstructor().newInstance(); + connectionPool.setUrl(getUrl(databaseName)); + connectionPoolMap.put(databaseName, connectionPool); + return connectionPool.getConnection(); + } catch (SQLException e) { + logger.error("cannot get connection for database {}: {}", databaseName, e.getMessage()); + return null; + } + } + + @Override + public TaskExecuteResult execute(StoragePhysicalTask task) { + List operators = task.getOperators(); + if (operators.size() != 1) { + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task")); + } + FragmentMeta fragment = task.getTargetFragment(); + Operator op = operators.get(0); + String storageUnit = task.getStorageUnit(); + boolean isDummyStorageUnit = task.isDummyStorageUnit(); + Connection conn = getConnection(storageUnit); + if (!isDummyStorageUnit && conn == null) { + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException(String.format("cannot connect to storage unit %s", storageUnit))); + } + + if (op.getType() == OperatorType.Project) { + Project project = (Project) op; + Filter filter; + if (operators.size() == 2) { + filter = ((Select) operators.get(1)).getFilter(); + } else { + filter = new AndFilter(Arrays + .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), + new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); + } + return isDummyStorageUnit ? executeHistoryProjectTask(project, filter) : executeProjectTask(conn, storageUnit, project, filter); + } else if (op.getType() == OperatorType.Insert) { + Insert insert = (Insert) op; + return executeInsertTask(conn, storageUnit, insert); + } else if (op.getType() == OperatorType.Delete) { + Delete delete = (Delete) op; + return executeDeleteTask(conn, storageUnit, delete); + } + return new TaskExecuteResult( + new NonExecutablePhysicalTaskException("unsupported physical task in relationdb")); + } + + @Override + public List getTimeSeries() { + List timeseries = new ArrayList<>(); + Map extraParams = meta.getExtraParams(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + while (databaseSet.next()) { + try { + String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 + if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { + continue; + } + Connection conn = getConnection(databaseName); + if (conn == null) { + continue; + } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 + String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 + if (columnName.equals("time")) { // time 列不显示 + continue; + } + Pair> nameAndTags = splitFullName(columnName, r.RELATIONDB_SEPARATOR); + if (databaseName.startsWith(DATABASE_PREFIX)) { + timeseries.add(new Timeseries( + tableName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + nameAndTags.k, + DataTypeTransformer.fromRelationDB(typeName), + nameAndTags.v) + ); + } else { + timeseries.add(new Timeseries( + databaseName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + tableName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + nameAndTags.k, + DataTypeTransformer.fromRelationDB(typeName), + nameAndTags.v) + ); + } + } + columnSet.close(); + } + tableSet.close(); + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + } + } + databaseSet.close(); + stmt.close(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + return timeseries; + } + + @Override + public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { + long minTime = Long.MAX_VALUE, maxTime = 0; + List paths = new ArrayList<>(); + try { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + while (databaseSet.next()) { + String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 + Connection conn = getConnection(databaseName); + if (conn == null) { + continue; + } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); + while (tableSet.next()) { + String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 + ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + StringBuilder columnNames = new StringBuilder(); + while (columnSet.next()) { + String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 + paths.add(databaseName + IGINX_SEPARATOR + tableName + IGINX_SEPARATOR + columnName); + columnNames.append(columnName); + columnNames.append(", "); // c1, c2, c3, + } + columnNames = new StringBuilder(columnNames.substring(0, columnNames.length() - 2)); // c1, c2, c3 + + // 获取 key 的范围 + String statement = String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); + Statement concatStmt = conn.createStatement(); + ResultSet concatSet = concatStmt.executeQuery(statement); + while (concatSet.next()) { + String concatValue = concatSet.getString("concat"); + long time = toHash(concatValue); + minTime = Math.min(time, minTime); + maxTime = Math.max(time, maxTime); + } + } + } + stmt.close(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + paths.sort(String::compareTo); + return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), + new TimeInterval(minTime, maxTime + 1)); + } + + private Map splitAndMergeQueryPatterns(String databaseName, Connection conn, List patterns) throws SQLException { + // table name -> column names + // 1 -> n + Map tableNameToColumnNames = new HashMap<>(); + String tableName; + String columnNames; + + for (String pattern : patterns) { + if (pattern.equals("*") || pattern.equals("*.*")) { + tableName = "%"; + columnNames = "%"; + } else { + if (pattern.split("\\" + IGINX_SEPARATOR).length == 1) { // REST 查询的路径中可能不含 . + tableName = pattern; + columnNames = "%"; + } else { + tableName = pattern.substring(0, pattern.lastIndexOf(".")).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); + columnNames = pattern.substring(pattern.lastIndexOf(".") + 1); + boolean columnEqualsStar = columnNames.equals("*"); + boolean tableContainsStar = tableName.contains("*"); + if (columnEqualsStar || tableContainsStar) { + tableName = tableName.replace('*', '%'); + if (columnEqualsStar) { + columnNames = "%"; + if (!tableName.endsWith("%")) { + tableName += "%"; + } + } + } + } + } + + if (!columnNames.endsWith("%")) { + columnNames += "%"; // 匹配 tagKV + } + ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); + while (rs.next()) { + tableName = rs.getString("TABLE_NAME"); + columnNames = rs.getString("COLUMN_NAME"); + if (columnNames.equals("time")) { + continue; + } + if (tableNameToColumnNames.containsKey(tableName)) { + columnNames = tableNameToColumnNames.get(tableName) + ", " + columnNames; + } + tableNameToColumnNames.put(tableName, columnNames); + } + rs.close(); + } + + return tableNameToColumnNames; + } + + private TaskExecuteResult executeProjectTask(Connection conn, String databaseName, Project project, Filter filter) { + try { + List resultSets = new ArrayList<>(); + ResultSet rs; + Statement stmt; + + Map tableNameToColumnNames = splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); + for (Map.Entry entry : tableNameToColumnNames.entrySet()) { + String tableName = entry.getKey(); + String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); + String statement = String.format(QUERY_STATEMENT, columnNames, getCompleteName(tableName), FilterTransformer.toString(filter)); + try { + stmt = conn.createStatement(); + rs = stmt.executeQuery(statement); + logger.info("[Query] execute query: {}", statement); + } catch (SQLException e) { + logger.error("meet error when executing query {}: {}", statement, e.getMessage()); + continue; + } + resultSets.add(rs); + } + + RowStream rowStream = new ClearEmptyRowStreamWrapper( + new RelationDBQueryRowStream(resultSets, false, filter, project.getTagFilter(),r.RELATIONDB_SEPARATOR)); + conn.close(); + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.error(e.getMessage()); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute project task in relationdb failure", e)); + } + } + + private Map> splitAndMergeHistoryQueryPatterns(List patterns) throws SQLException { + // > + Map> splitResults = new HashMap<>(); + String databaseName; + String tableName; + String columnNames; + + for (String pattern : patterns) { + if (pattern.equals("*") || pattern.equals("*.*")) { + databaseName = "%"; + tableName = "%"; + columnNames = "%"; + } else { + String[] parts = pattern.split("\\" + IGINX_SEPARATOR); + if (parts.length > 3) { // 大于三级，不合法 + logger.error("wrong pattern: {}", pattern); + continue; + } + if (parts.length == 2 && !parts[1].equals("*")) { // 只有两级且第二级不为 *，则此 pattern 不合法，无法拆分出三级 + continue; + } + + databaseName = parts[0]; + tableName = parts[1].equals("*") ? "%" : parts[1]; + columnNames = parts[2].equals("*") ? "%" : parts[2]; + } + + if (databaseName.equals("%")) { + Statement stmt = connection.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + while (databaseSet.next()) { + String tempDatabaseName = databaseSet.getString("DATNAME"); + if (tempDatabaseName.startsWith(DATABASE_PREFIX)) { + continue; + } + Connection conn = getConnection(tempDatabaseName); + if (conn == null) { + continue; + } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(tempDatabaseName, "public", tableName, new String[]{"TABLE"}); + while (tableSet.next()) { + String tempTableName = tableSet.getString("TABLE_NAME"); + ResultSet columnSet = databaseMetaData.getColumns(tempDatabaseName, "public", tempTableName, columnNames); + while (columnSet.next()) { + String tempColumnNames = columnSet.getString("COLUMN_NAME"); + Map tableNameToColumnNames = new HashMap<>(); + if (splitResults.containsKey(tempDatabaseName)) { + tableNameToColumnNames = splitResults.get(tempDatabaseName); + tempColumnNames = tableNameToColumnNames.get(tempTableName) + ", " + tempColumnNames; + } + tableNameToColumnNames.put(tempTableName, tempColumnNames); + splitResults.put(tempDatabaseName, tableNameToColumnNames); + } + } + } + } else { + Connection conn = getConnection(databaseName); + if (conn == null) { + continue; + } + ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); + while (rs.next()) { + tableName = rs.getString("TABLE_NAME"); + columnNames = rs.getString("COLUMN_NAME"); + Map tableNameToColumnNames = new HashMap<>(); + if (splitResults.containsKey(databaseName)) { + tableNameToColumnNames = splitResults.get(databaseName); + columnNames = tableNameToColumnNames.get(tableName) + ", " + columnNames; + } + tableNameToColumnNames.put(tableName, columnNames); + splitResults.put(databaseName, tableNameToColumnNames); + } + } + } + + return splitResults; + } + + private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { + try { + List resultSets = new ArrayList<>(); + ResultSet rs; + Connection conn = null; + Statement stmt; + + Map> splitResults = splitAndMergeHistoryQueryPatterns(project.getPatterns()); + for (Map.Entry> splitEntry : splitResults.entrySet()) { + String databaseName = splitEntry.getKey(); + conn = getConnection(databaseName); + if (conn == null) { + continue; + } + for (Map.Entry entry : splitEntry.getValue().entrySet()) { + String tableName = entry.getKey(); + String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); + String statement = String.format(QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, columnNames, columnNames, getCompleteName(tableName)); + try { + stmt = conn.createStatement(); + rs = stmt.executeQuery(statement); + logger.info("[Query] execute query: {}", statement); + } catch (SQLException e) { + logger.error("meet error when executing query {}: {}", statement, e.getMessage()); + continue; + } + resultSets.add(rs); + } + } + + RowStream rowStream = new ClearEmptyRowStreamWrapper( + new RelationDBQueryRowStream(resultSets, true, filter, project.getTagFilter(), r.RELATIONDB_SEPARATOR)); + if (conn != null) { + conn.close(); + } + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.error(e.getMessage()); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute project task in relationDB failure", e)); + } + } + + private TaskExecuteResult executeInsertTask(Connection conn, String storageUnit, Insert insert) { + DataView dataView = insert.getData(); + Exception e = null; + switch (dataView.getRawDataType()) { + case Row: + case NonAlignedRow: + e = insertNonAlignedRowRecords(conn, storageUnit, (RowDataView) dataView); + break; + case Column: + case NonAlignedColumn: + e = insertNonAlignedColumnRecords(conn, storageUnit, (ColumnDataView) dataView); + break; + } + if (e != null) { + return new TaskExecuteResult(null, + new PhysicalException("execute insert task in relationdb failure", e)); + } + return new TaskExecuteResult(null, null); + } + + private void createOrAlterTables(Connection conn, String storageUnit, List paths, List> tagsList, List dataTypeList) { + for (int i = 0; i < paths.size(); i++) { + String path = paths.get(i); + Map tags = new HashMap<>(); + if (tagsList != null && !tagsList.isEmpty()) { + tags = tagsList.get(i); + } + DataType dataType = dataTypeList.get(i); + String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); + String columnName = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); + + try { + Statement stmt = conn.createStatement(); + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = databaseMetaData.getTables(storageUnit, "public", tableName, new String[]{"TABLE"}); + columnName = toFullName(columnName, tags, r.RELATIONDB_SEPARATOR); + if (!tableSet.next()) { + String statement = String.format(CREATE_TABLE_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), DataTypeTransformer.toRelationDB(dataType)); + logger.info("[Create] execute create: {}", statement); + stmt.execute(statement); + } else { + ResultSet columnSet = databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); + if (!columnSet.next()) { + String statement = String.format(ADD_COLUMN_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), DataTypeTransformer.toRelationDB(dataType)); + logger.info("[Create] execute create: {}", statement); + stmt.execute(statement); + } + columnSet.close(); + } + tableSet.close(); + stmt.close(); + } catch (SQLException e) { + logger.error("create or alter table {} field {} error: {}", tableName, columnName, e.getMessage()); + } + } + } + + private Exception insertNonAlignedRowRecords(Connection conn, String storageUnit, RowDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = conn.createStatement(); + + // 创建表 + createOrAlterTables(conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + + // 插入数据 + Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> + int cnt = 0; + boolean firstRound = true; + while (cnt < data.getTimeSize()) { + int size = Math.min(data.getTimeSize() - cnt, batchSize); + Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 + for (int i = cnt; i < cnt + size; i++) { + BitmapView bitmapView = data.getBitmapView(i); + int index = 0; + for (int j = 0; j < data.getPathNum(); j++) { + String path = data.getPath(j); + DataType dataType = data.getDataType(j); + String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); + String columnName = path.substring(path.lastIndexOf('.') + 1); + Map tags = new HashMap<>(); + if (data.hasTagsList()) { + tags = data.getTags(j); + } + + StringBuilder columnKeys = new StringBuilder(); + List columnValues = new ArrayList<>(); + if (tableToColumnEntries.containsKey(tableName)) { + columnKeys = new StringBuilder(tableToColumnEntries.get(tableName).k); + columnValues = tableToColumnEntries.get(tableName).v; + } + + String value = "null"; + if (bitmapView.get(j)) { + if (dataType == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + } else { + value = data.getValue(i, index).toString(); + } + index++; + if (tableHasData.containsKey(tableName)) { + tableHasData.get(tableName)[i - cnt] = true; + } else { + boolean[] hasData = new boolean[size]; + hasData[i - cnt] = true; + tableHasData.put(tableName, hasData); + } + } + + if (firstRound) { + columnKeys.append(toFullName(columnName, tags, r.RELATIONDB_SEPARATOR)).append(", "); + } + + if (i - cnt < columnValues.size()) { + columnValues.set(i - cnt, columnValues.get(i - cnt) + value + ", "); + } else { + columnValues.add(data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 + } + + tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); + } + + firstRound = false; + } + + for (Map.Entry entry : tableHasData.entrySet()) { + String tableName = entry.getKey(); + boolean[] hasData = entry.getValue(); + String columnKeys = tableToColumnEntries.get(tableName).k; + List columnValues = tableToColumnEntries.get(tableName).v; + boolean needToInsert = false; + for (int i = hasData.length - 1; i >= 0; i--) { + if (!hasData[i]) { + columnValues.remove(i); + } else { + needToInsert = true; + } + } + if (needToInsert) { + tableToColumnEntries.put(tableName, new Pair<>(columnKeys, columnValues)); + } + } + + executeBatchInsert(stmt, tableToColumnEntries); + for (Pair> columnEntries : tableToColumnEntries.values()) { + columnEntries.v.clear(); + } + + cnt += size; + } + stmt.close(); + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + return e; + } + + return null; + } + + private Exception insertNonAlignedColumnRecords(Connection conn, String storageUnit, ColumnDataView data) { + int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + try { + Statement stmt = conn.createStatement(); + + // 创建表 + createOrAlterTables(conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + + // 插入数据 + Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> + Map pathIndexToBitmapIndex = new HashMap<>(); + int cnt = 0; + boolean firstRound = true; + while (cnt < data.getTimeSize()) { + int size = Math.min(data.getTimeSize() - cnt, batchSize); + Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 + for (int i = 0; i < data.getPathNum(); i++) { + String path = data.getPath(i); + DataType dataType = data.getDataType(i); + String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); + String columnName = path.substring(path.lastIndexOf('.') + 1); + Map tags = new HashMap<>(); + if (data.hasTagsList()) { + tags = data.getTags(i); + } + + BitmapView bitmapView = data.getBitmapView(i); + + StringBuilder columnKeys = new StringBuilder(); + List columnValues = new ArrayList<>(); + if (tableToColumnEntries.containsKey(tableName)) { + columnKeys = new StringBuilder(tableToColumnEntries.get(tableName).k); + columnValues = tableToColumnEntries.get(tableName).v; + } + + int index = 0; + if (pathIndexToBitmapIndex.containsKey(i)) { + index = pathIndexToBitmapIndex.get(i); + } + for (int j = cnt; j < cnt + size; j++) { + String value = "null"; + if (bitmapView.get(j)) { + if (dataType == DataType.BINARY) { + value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + } else { + value = data.getValue(i, index).toString(); + } + index++; + if (tableHasData.containsKey(tableName)) { + tableHasData.get(tableName)[j - cnt] = true; + } else { + boolean[] hasData = new boolean[size]; + hasData[j - cnt] = true; + tableHasData.put(tableName, hasData); + } + } + + if (j - cnt < columnValues.size()) { + columnValues.set(j - cnt, columnValues.get(j - cnt) + value + ", "); + } else { + columnValues.add(data.getKey(j) + ", " + value + ", "); // 添加 key(time) 列 + } + } + pathIndexToBitmapIndex.put(i, index); + + if (firstRound) { + columnKeys.append(toFullName(columnName, tags, r.RELATIONDB_SEPARATOR)).append(", "); + } + + tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); + } + + for (Map.Entry entry : tableHasData.entrySet()) { + String tableName = entry.getKey(); + boolean[] hasData = entry.getValue(); + String columnKeys = tableToColumnEntries.get(tableName).k; + List columnValues = tableToColumnEntries.get(tableName).v; + boolean needToInsert = false; + for (int i = hasData.length - 1; i >= 0; i--) { + if (!hasData[i]) { + columnValues.remove(i); + } else { + needToInsert = true; + } + } + if (needToInsert) { + tableToColumnEntries.put(tableName, new Pair<>(columnKeys, columnValues)); + } + } + + executeBatchInsert(stmt, tableToColumnEntries); + for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + entry.getValue().v.clear(); + } + + firstRound = false; + cnt += size; + } + stmt.close(); + conn.close(); + } catch (SQLException e) { + logger.error(e.getMessage()); + return e; + } + + return null; + } + + private void executeBatchInsert(Statement stmt, Map>> tableToColumnEntries) throws SQLException { + for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + String tableName = entry.getKey(); + String columnNames = entry.getValue().k.substring(0, entry.getValue().k.length() - 2); + List values = entry.getValue().v; + String[] parts = columnNames.split(", "); + boolean hasMultipleRows = parts.length != 1; + + StringBuilder statement = new StringBuilder(); + statement.append("INSERT INTO "); + statement.append(getCompleteName(tableName)); + statement.append(" (time, "); + statement.append(Arrays.stream(parts).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("")); + statement.append(") VALUES"); + statement.append(values.stream().map(x -> " (" + x.substring(0, x.length() - 2) + ")").reduce((a, b) -> a + ", " + b).orElse("")); + statement.append(" ON CONFLICT (time) DO UPDATE SET "); + if (hasMultipleRows) { + statement.append("("); // 只有一列不加括号 + } + statement.append(Arrays.stream(parts).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("")); + if (hasMultipleRows) { + statement.append(")"); // 只有一列不加括号 + } + statement.append(" = "); + if (hasMultipleRows) { + statement.append("("); // 只有一列不加括号 + } + statement.append(Arrays.stream(parts).map(x -> "excluded." + getCompleteName(x)).reduce((a, b) -> a + ", " + b).orElse("")); + if (hasMultipleRows) { + statement.append(")"); // 只有一列不加括号 + } + statement.append(";"); + +// logger.info("[Insert] execute insert: {}", statement); + stmt.addBatch(statement.toString()); + } + stmt.executeBatch(); + } + + private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { + try { + Statement stmt = conn.createStatement(); + String statement; + List paths = delete.getPatterns(); + List> deletedPaths; // table name -> column name + String tableName; + String columnName; + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = null; + ResultSet columnSet = null; + + if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { + if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { + conn.close(); + String defaultDB = r.DEFAULT_DATABASE; + Connection defaultConn = getConnection(defaultDB); // 正在使用的数据库无法被删除，因此需要切换到默认数据库 + if (defaultConn != null) { + stmt = defaultConn.createStatement(); + statement = String.format(DROP_DATABASE_STATEMENT, storageUnit); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 删除数据库 + stmt.close(); + defaultConn.close(); + return new TaskExecuteResult(null, null); + } else { + return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("cannot connect to database: relationdb", new SQLException())); + } + } else { + deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); + for (Pair pair : deletedPaths) { + tableName = pair.k; + columnName = pair.v; + tableSet = databaseMetaData.getTables(storageUnit, "public", tableName, new String[]{"TABLE"}); + if (tableSet.next()) { + statement = String.format(DROP_COLUMN_STATEMENT, getCompleteName(tableName), getCompleteName(columnName)); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 删除列 + } + } + } + } else { + deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); + for (Pair pair : deletedPaths) { + tableName = pair.k; + columnName = pair.v; + columnSet = databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); + if (columnSet.next()) { + for (TimeRange timeRange : delete.getTimeRanges()) { + statement = String.format(UPDATE_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), + timeRange.getBeginTime(), timeRange.getEndTime()); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 将目标列的目标范围的值置为空 + } + } + } + } + if (tableSet != null) { + tableSet.close(); + } + if (columnSet != null) { + columnSet.close(); + } + stmt.close(); + conn.close(); + return new TaskExecuteResult(null, null); + } catch (SQLException e) { + logger.error(e.getMessage()); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException("execute delete task in relationdb failure", e)); + } + } + + private List> determineDeletedPaths(List paths, TagFilter tagFilter) { + List timeSeries = getTimeSeries(); + List> deletedPaths = new ArrayList<>(); + + for (Timeseries ts: timeSeries) { + for (String path : paths) { + if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath())) { + if (tagFilter != null && !TagKVUtils.match(ts.getTags(), tagFilter)) { + continue; + } + String fullPath = ts.getPath(); + String tableName = fullPath.substring(0, fullPath.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); + String columnName = toFullName(fullPath.substring(fullPath.lastIndexOf('.') + 1), ts.getTags(), r.RELATIONDB_SEPARATOR); + deletedPaths.add(new Pair<>(tableName, columnName)); + break; + } + } + } + + return deletedPaths; + } + + private String getCompleteName(String name) { + return "\"" + name + "\""; +// return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; + } + + @Override + public void release() throws PhysicalException { + try { + connection.close(); + } catch (SQLException e) { + throw new PhysicalException(e); + } + } +} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java new file mode 100644 index 0000000000..02023c0672 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java @@ -0,0 +1,17 @@ +package cn.edu.tsinghua.iginx.relationdb.conf; + +public class RelationDBConf { + public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; + public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; + public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; + + public static final String IGINX_SEPARATOR = "."; + public static final String DATABASE_PREFIX = "unit"; + public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; + public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; + public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; + public static final String UPDATE_STATEMENT = "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; + public static final String USERNAME = "username"; + public static final String PASSWORD = "password"; + public static final String CONNECTION_POOL_CLASS_NAME = null; +} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java new file mode 100644 index 0000000000..958fa98342 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java @@ -0,0 +1,19 @@ +package cn.edu.tsinghua.iginx.relationdb.conf; + +import cn.edu.tsinghua.iginx.relationdb.conf.RelationDBConf; +public class dmdbConf extends RelationDBConf { + public final boolean NEED_CHANGE_DB = false; + public static final String DB_NAME = "dmDB"; + public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; + public static final String DRIVER_URL = "dm.jdbc.driver.DmDriver"; + public static final String CONNECTION_URL = "jdbc:dm://%s:%s/?user=%s&password=%s"; + public static final String DEFAULT_USERNAME = "SYSDBA"; + public static final String DEFAULT_PASSWORD = "SYSDBA001"; + public static final String IGNORED_DATABASE = ""; //dmDB中的表空间 + public static final String CREATE_DATABASE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; + public static final String QUERY_DATABASES_STATEMENT = null; + public static final String RELATIONDB_SEPARATOR = "."; + public static final String DEFAULT_DATABASE = null; + public static final int BATCH_SIZE = 10000; + public static final String CONNECTION_POOL_CLASS_NAME = "com.huawei.dmdb.jdbc.DmdbConnectionPoolDataSource"; +} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java new file mode 100644 index 0000000000..ddae6e21fc --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java @@ -0,0 +1,20 @@ +package cn.edu.tsinghua.iginx.relationdb.conf; + +import cn.edu.tsinghua.iginx.relationdb.conf.RelationDBConf; +public class mysqlConf extends RelationDBConf { + + public final boolean NEED_CHANGE_DB = true; + public static final String DB_NAME = "mysql"; + public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; + public static final String DRIVER_NAME = "com.mysql.cj.jdbc.Driver"; + public static final String CONNECTION_URL = "jdbc:mysql://%s:%s/?user=%s&password=%s"; + public static final String DEFAULT_USERNAME = "root"; + public static final String DEFAULT_PASSWORD = ""; + public static final String IGNORED_DATABASE = "sys,performance_schema,information_schema"; + public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s"; + public static final String QUERY_DATABASES_STATEMENT = "SHOW DATABASES;"; + public static final String RELATIONDB_SEPARATOR = "\u2E82"; + public static final String DEFAULT_DATABASE = "mysql"; + public static final int BATCH_SIZE = 10000; + public static final String CONNECTION_POOL_CLASS_NAME = "com.mysql.cj.jdbc.MysqlConnectionPoolDataSource"; +} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java new file mode 100644 index 0000000000..9063ea84d4 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java @@ -0,0 +1,243 @@ +package cn.edu.tsinghua.iginx.relationdb.query.entity; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; +import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; +import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; +import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; +import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.relationdb.tools.DataTypeTransformer; +import cn.edu.tsinghua.iginx.thrift.DataType; +import cn.edu.tsinghua.iginx.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.*; + +import static cn.edu.tsinghua.iginx.relationdb.conf.RelationDBConf.IGINX_SEPARATOR; +import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; +//import static cn.edu.tsinghua.iginx.relationDB.tools.Constants.RELATIONDB_SEPARATOR; +import static cn.edu.tsinghua.iginx.relationdb.tools.HashUtils.toHash; +import static cn.edu.tsinghua.iginx.relationdb.tools.TagKVUtils.splitFullName; + +public class RelationDBQueryRowStream implements RowStream { + + private static final Logger logger = LoggerFactory.getLogger(RelationDBQueryRowStream.class); + + private final List resultSets; + + private final Header header; + + private final boolean isDummy; + + private final Filter filter; + + private boolean[] gotNext; // 标记每个结果集是否已经获取到下一行，如果是，则在下次调用 next() 时无需再调用该结果集的 next() + + private long[] cachedTimestamps; // 缓存每个结果集当前的 time 列的值 + + private Object[] cachedValues; // 缓存每列当前的值 + + private int[] resultSetSizes; // 记录每个结果集的列数 + + private Map fieldToColumnName; // 记录匹配 tagFilter 的列名 + + private Row cachedRow; + + private boolean hasCachedRow; + + public RelationDBQueryRowStream(List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter, String RELATIONDB_SEPARATOR) throws SQLException { + this.resultSets = resultSets; + this.isDummy = isDummy; + this.filter = filter; + + if (resultSets.isEmpty()) { + this.header = new Header(Field.KEY, Collections.emptyList()); + return; + } + + boolean filterByTags = tagFilter != null; + + Field time = null; + List fields = new ArrayList<>(); + this.resultSetSizes = new int[resultSets.size()]; + this.fieldToColumnName = new HashMap<>(); + + for (int i = 0; i < resultSets.size(); i++) { + ResultSetMetaData resultSetMetaData = resultSets.get(i).getMetaData(); + String tableName = resultSetMetaData.getTableName(1); + int cnt = 0; + for (int j = 1; j <= resultSetMetaData.getColumnCount(); j++) { + String columnName = resultSetMetaData.getColumnName(j); + String typeName = resultSetMetaData.getColumnTypeName(j); + if (j == 1 && columnName.equals("time")) { + time = Field.KEY; + continue; + } + + Pair> namesAndTags = splitFullName(columnName,RELATIONDB_SEPARATOR); + Field field = new Field( + tableName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + + namesAndTags.k.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromRelationDB(typeName), + namesAndTags.v + ); + + if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { + continue; + } + fieldToColumnName.put(field, columnName); + fields.add(field); + cnt++; + } + resultSetSizes[i] = cnt; + } + + this.header = new Header(time, fields); + + this.gotNext = new boolean[resultSets.size()]; + Arrays.fill(gotNext, false); + this.cachedTimestamps = new long[resultSets.size()]; + Arrays.fill(cachedTimestamps, Long.MAX_VALUE); + this.cachedValues = new Object[fields.size()]; + Arrays.fill(cachedValues, null); + this.cachedRow = null; + this.hasCachedRow = false; + } + + @Override + public Header getHeader() { + return header; + } + + @Override + public void close() { + try { + for (ResultSet resultSet : resultSets) { + resultSet.close(); + } + } catch (SQLException e) { + logger.error(e.getMessage()); + } + } + + @Override + public boolean hasNext() { + if (resultSets.isEmpty()) { + return false; + } + + try { + if (!hasCachedRow) { + cacheOneRow(); + } + } catch (SQLException | PhysicalException e) { + logger.error(e.getMessage()); + } + + return cachedRow != null; + } + + @Override + public Row next() throws PhysicalException { + try { + Row row; + if (!hasCachedRow) { + cacheOneRow(); + } + row = cachedRow; + hasCachedRow = false; + cachedRow = null; + return row; + } catch (SQLException | PhysicalException e) { + logger.error(e.getMessage()); + throw new RowFetchException(e); + } + } + + private void cacheOneRow() throws SQLException, PhysicalException { + boolean hasNext = false; + long timestamp; + Object[] values = new Object[header.getFieldSize()]; + + int startIndex = 0; + int endIndex = 0; + for (int i = 0; i < resultSets.size(); i++) { + ResultSet resultSet = resultSets.get(i); + if (resultSetSizes[i] == 0) { + continue; + } + endIndex += resultSetSizes[i]; + if (!gotNext[i]) { + boolean tempHasNext = resultSet.next(); + hasNext |= tempHasNext; + gotNext[i] = true; + + if (tempHasNext) { + long tempTimestamp; + Object tempValue; + + if (isDummy) { + tempTimestamp = toHash(resultSet.getString("time")); + } else { + tempTimestamp = resultSet.getLong("time"); + } + cachedTimestamps[i] = tempTimestamp; + + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int j = 0; j < resultSetSizes[i]; j++) { + Object value = resultSet.getObject(fieldToColumnName.get(header.getField(startIndex + j))); + if (header.getField(startIndex + j).getType() == DataType.BINARY && value != null) { + tempValue = value.toString().getBytes(); + } else { + tempValue = value; + } + cachedValues[startIndex + j] = tempValue; + } + } else { + cachedTimestamps[i] = Long.MAX_VALUE; + for (int j = startIndex; j < endIndex; j++) { + cachedValues[j] = null; + } + } + } else { + hasNext = true; + } + startIndex = endIndex; + } + + if (hasNext) { + timestamp = Arrays.stream(cachedTimestamps).min().getAsLong(); + startIndex = 0; + endIndex = 0; + for (int i = 0; i < resultSets.size(); i++) { + endIndex += resultSetSizes[i]; + if (cachedTimestamps[i] == timestamp) { + for (int j = 0; j < resultSetSizes[i]; j++) { + values[startIndex + j] = cachedValues[startIndex + j]; + } + gotNext[i] = false; + } else { + for (int j = 0; j < resultSetSizes[i]; j++) { + values[startIndex + j] = null; + } + gotNext[i] = true; + } + startIndex = endIndex; + } + cachedRow = new Row(header, timestamp, values); + if (isDummy && !validate(filter, cachedRow)) { + cacheOneRow(); + } + } else { + cachedRow = null; + } + hasCachedRow = true; + } +} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java new file mode 100644 index 0000000000..1555ec54ce --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java @@ -0,0 +1,39 @@ +//package cn.edu.tsinghua.iginx.relationdb.tools; +// +//import static cn.edu.tsinghua.iginx.relationdb.conf; +//public class Constants { +// +// public static final String IGINX_SEPARATOR = "."; +// public static final String POSTGRESQL_SEPARATOR = "\u2E82"; +// public static final int BATCH_SIZE = 10000; +// +// public static final String USERNAME = "username"; +// +// public static final String PASSWORD = "password"; +// +// public static final String DEFAULT_USERNAME = "postgres"; +// +// public static final String DEFAULT_PASSWORD = "postgres"; +// +// public static final String DATABASE_PREFIX = "unit"; +// +// public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; +// +// public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; +// +// public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; +// +// public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; +// +// public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; +// +// public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; +// +// public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; +// +// public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; +// +// public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; +// +// public static final String UPDATE_STATEMENT = "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; +//} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java new file mode 100644 index 0000000000..7a4f1ded05 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java @@ -0,0 +1,43 @@ +package cn.edu.tsinghua.iginx.relationdb.tools; + +import cn.edu.tsinghua.iginx.thrift.DataType; + +import static cn.edu.tsinghua.iginx.thrift.DataType.*; + +public class DataTypeTransformer { + + public static DataType fromRelationDB(String dataType) { + if (dataType.equalsIgnoreCase("bool")) { + return BOOLEAN; + } else if (dataType.equalsIgnoreCase("int") || dataType.equalsIgnoreCase("int2") || dataType.equalsIgnoreCase("int4") || + dataType.equalsIgnoreCase("serial2") || dataType.equalsIgnoreCase("serial4")) { + return INTEGER; + } else if (dataType.equalsIgnoreCase("int8") || dataType.equalsIgnoreCase("serial8")) { + return LONG; + } else if (dataType.equalsIgnoreCase(("float4"))) { + return FLOAT; + } else if (dataType.equalsIgnoreCase("decimal") || dataType.equalsIgnoreCase("float8")) { + return DOUBLE; + } else { + return BINARY; + } + } + + public static String toRelationDB(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return "BOOLEAN"; + case INTEGER: + return "INTEGER"; + case LONG: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE PRECISION"; + case BINARY: + default: + return "TEXT"; + } + } +} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java new file mode 100644 index 0000000000..3d298135c4 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package cn.edu.tsinghua.iginx.relationdb.tools; + +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; + +import java.util.stream.Collectors; + +public class FilterTransformer { + + public static final long MAX_TIMESTAMP = Long.MAX_VALUE; + + public static String toString(Filter filter) { + if (filter == null) { + return ""; + } + switch (filter.getType()) { + case And: + return toString((AndFilter) filter); + case Or: + return toString((OrFilter) filter); + case Not: + return toString((NotFilter) filter); + case Value: + return toString((ValueFilter) filter); + case Key: + return toString((KeyFilter) filter); + default: + return ""; + } + } + + private static String toString(AndFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); + } + + private static String toString(NotFilter filter) { + return "not " + filter.toString(); + } + + private static String toString(KeyFilter filter) { + return "time " + Op.op2Str(filter.getOp()) + " " + Math.min(filter.getValue(), MAX_TIMESTAMP); + } + + private static String toString(ValueFilter filter) { + return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); + } + + private static String toString(OrFilter filter) { + return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); + } + +} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java new file mode 100644 index 0000000000..adb8465f7d --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java @@ -0,0 +1,17 @@ +package cn.edu.tsinghua.iginx.relationdb.tools; + +public class HashUtils { + + public static long toHash(String s) { + char c[] = s.toCharArray(); + long hv = 0; + long base = 131; + for (int i = 0; i < c.length; i++) { + hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 + } + if (hv < 0) { + return -1 * hv; + } + return hv; + } +} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java new file mode 100644 index 0000000000..f2a18c9033 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java @@ -0,0 +1,54 @@ +package cn.edu.tsinghua.iginx.relationdb.tools; + +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; + +public class TagFilterUtils { + @SuppressWarnings("unused") + public static String transformToFilterStr(TagFilter filter) { + StringBuilder builder = new StringBuilder(); + transformToFilterStr(filter, builder); + return builder.toString(); + } + + private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { + switch (filter.getType()) { + case And: + AndTagFilter andFilter = (AndTagFilter) filter; + for (int i = 0; i < andFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(andFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" and "); + } + } + break; + case Or: + OrTagFilter orFilter = (OrTagFilter) filter; + for (int i = 0; i < orFilter.getChildren().size(); i++) { + builder.append('('); + transformToFilterStr(orFilter.getChildren().get(i), builder); + builder.append(')'); + if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 + builder.append(" or "); + } + } + break; + case Base: + BaseTagFilter baseFilter = (BaseTagFilter) filter; + builder.append(baseFilter.getTagKey()); + builder.append("="); + builder.append(baseFilter.getTagValue()); + break; + // TODO: case label + case BasePrecise: + break; + case Precise: + case WithoutTag: + break; + } + } +} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java new file mode 100644 index 0000000000..12676b4e66 --- /dev/null +++ b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java @@ -0,0 +1,42 @@ +package cn.edu.tsinghua.iginx.relationdb.tools; + +import cn.edu.tsinghua.iginx.utils.Pair; + +import java.util.*; + +//import static cn.edu.tsinghua.iginx.relationDB.tools.Constants.RELATIONDB_SEPARATOR; + +public class TagKVUtils { + + public static Pair> splitFullName(String fullName,String RELATIONDB_SEPARATOR) { + if (!fullName.contains(RELATIONDB_SEPARATOR)) { + return new Pair<>(fullName, null); + } + + String[] parts = fullName.split("\\" + RELATIONDB_SEPARATOR); + String name = parts[0]; + + Map tags = new HashMap<>(); + for (int i = 1; i < parts.length; i++) { + if (i % 2 != 0) { + continue; + } + String tagKey = parts[i - 1]; + String tagValue = parts[i]; + tags.put(tagKey, tagValue); + } + return new Pair<>(name, tags); + } + + public static String toFullName(String name, Map tags,String RELATIONDB_SEPARATOR) { + if (tags != null && !tags.isEmpty()) { + TreeMap sortedTags = new TreeMap<>(tags); + StringBuilder pathBuilder = new StringBuilder(); + sortedTags.forEach((tagKey, tagValue) -> { + pathBuilder.append(RELATIONDB_SEPARATOR).append(tagKey).append(RELATIONDB_SEPARATOR).append(tagValue); + }); + name += pathBuilder.toString(); + } + return name; + } +} diff --git a/pom.xml b/pom.xml index 4e8cdc8392..90d49d4ed2 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,7 @@ dataSources/opentsdb dataSources/postgresql + dataSources/relationdb dataSources/parquet example test From 9b875fa468884857e94bc57a584c4a2ceb7125d6 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Mon, 17 Apr 2023 09:59:40 +0800 Subject: [PATCH 50/94] update --- dataSources/relationdb/pom.xml | 114 --- .../iginx/relationdb/RelationDBStorage.java | 949 ------------------ .../iginx/relationdb/conf/RelationDBConf.java | 17 - .../iginx/relationdb/conf/dmdbConf.java | 19 - .../iginx/relationdb/conf/mysqlConf.java | 20 - .../entity/RelationDBQueryRowStream.java | 243 ----- .../iginx/relationdb/tools/Constants.java | 39 - .../relationdb/tools/DataTypeTransformer.java | 43 - .../relationdb/tools/FilterTransformer.java | 69 -- .../iginx/relationdb/tools/HashUtils.java | 17 - .../relationdb/tools/TagFilterUtils.java | 54 - .../iginx/relationdb/tools/TagKVUtils.java | 42 - 12 files changed, 1626 deletions(-) delete mode 100644 dataSources/relationdb/pom.xml delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java delete mode 100644 dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java diff --git a/dataSources/relationdb/pom.xml b/dataSources/relationdb/pom.xml deleted file mode 100644 index f586cf2aaf..0000000000 --- a/dataSources/relationdb/pom.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - 4.0.0 - - - iginx - cn.edu.tsinghua - ${revision} - ../../pom.xml - - - relationdb - IGinX RelationDB - - - - - - - - - - - - - - - - - - - - - - - - 8 - 8 - - - - - cn.edu.tsinghua - iginx-core - ${project.version} - provided - - - org.postgresql - postgresql - 42.5.4 - - - com.mysql - mysql-connector-j - 8.0.32 - - - com.huawei.dmdb - dmdb-jdbc - 1.7.1.1 - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - package - - copy-dependencies - - - ../../core/target/iginx-core-${project.version}/driver/relationdb/ - - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - package - - run - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java deleted file mode 100644 index 8eca8616bd..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/RelationDBStorage.java +++ /dev/null @@ -1,949 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.relationdb; - -import cn.edu.tsinghua.iginx.engine.physical.exception.NonExecutablePhysicalTaskException; -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; -import cn.edu.tsinghua.iginx.engine.physical.exception.StorageInitializationException; -import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; -import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Timeseries; -import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; -import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; -import cn.edu.tsinghua.iginx.engine.shared.TimeRange; -import cn.edu.tsinghua.iginx.engine.shared.data.read.ClearEmptyRowStreamWrapper; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; -import cn.edu.tsinghua.iginx.engine.shared.operator.*; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; -import cn.edu.tsinghua.iginx.metadata.entity.*; -import cn.edu.tsinghua.iginx.relationdb.query.entity.RelationDBQueryRowStream; -import cn.edu.tsinghua.iginx.relationdb.tools.DataTypeTransformer; -import cn.edu.tsinghua.iginx.relationdb.tools.FilterTransformer; -import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.utils.Pair; -import cn.edu.tsinghua.iginx.utils.StringUtils; -import cn.edu.tsinghua.iginx.relationdb.conf.mysqlConf; -import cn.edu.tsinghua.iginx.relationdb.conf.dmdbConf; -//import org.postgresql.ds.PGConnectionPoolDataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.charset.StandardCharsets; -import java.sql.*; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; -import com.huawei.dmdb.jdbc.DmdbConnectionPoolDataSource; -import com.mysql.cj.jdbc.MysqlConnectionPoolDataSource; - -//import static cn.edu.tsinghua.iginx.relationDB.tools.Constants.*; -import static cn.edu.tsinghua.iginx.relationdb.tools.HashUtils.toHash; -import static cn.edu.tsinghua.iginx.relationdb.tools.TagKVUtils.splitFullName; -import static cn.edu.tsinghua.iginx.relationdb.tools.TagKVUtils.toFullName; - -public class RelationDBStorage implements IStorage { - - private static final Logger logger = LoggerFactory.getLogger(RelationDBStorage.class); - - private final StorageEngineMeta meta; - - private final Map connectionPoolMap = new ConcurrentHashMap<>(); - - private final Connection connection; - - public final RelationDBConf r; //关系数据库配置 - - public final String IGINX_SEPARATOR; - public final String RELATIONDB_SEPARATOR; - - public RelationDBStorage(StorageEngineMeta meta) throws StorageInitializationException { - this.meta = meta; - if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - - //找到具体的关系字类并创建实例 - String engineName = meta.getStorageEngine(); - String className = engineName + "Conf"; //关系子类的配置 类名 - Class clr = (Class)Class.forName("cn.edu.tsinghua.iginx.relationDB.conf."+className); - r = clr.getConstructor().newInstance(); //子类实例 - - this.IGINX_SEPARATOR = r.IGINX_SEPARATOR; - this.RELATIONDB_SEPARATOR = r.RELATIONDB_SEPARATOR; - - String connUrl = String - .format(r.CONNECTION_URL, meta.getIp(), meta.getPort(), - username, password); - try { - connection = DriverManager.getConnection(connUrl); - } catch (SQLException e) { - throw new StorageInitializationException("cannot connect to " + meta); - } - } - - private boolean testConnection() { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format(r.CONNECTION_URL, meta.getIp(), meta.getPort(), - username, password); - try { - Class.forName(r.DRIVER_URL); - DriverManager.getConnection(connUrl); - return true; - } catch (SQLException | ClassNotFoundException e) { - return false; - } - } - - private String getUrl(String databaseName) { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(r.USERNAME, r.DEFAULT_USERNAME); - String password = extraParams.getOrDefault(r.PASSWORD, r.DEFAULT_PASSWORD); - String connUrl = String - .format(r.CONNECTION_URL, meta.getIp(), meta.getPort(), databaseName, - username, password); - return connUrl; - } - - private Connection getConnection(String databaseName) { - if (!r.NEED_CHANGE_DB) { //有的数据库不需要更换连接 - return connection; - } - if (databaseName.startsWith("dummy")) { - return null; - } - String ignoredDB = r.IGNORED_DATABASE; //需要忽略的数据库（系统自动维护，用户没有修改权限的数据库等） - String[] ignoredDBs = ignoredDB.split(","); - for(String db:ignoredDBs) { - if (databaseName.equalsIgnoreCase(db)) { - return null; - } - } - - try { - Statement stmt = connection.createStatement(); - stmt.execute(String.format(r.CREATE_DATABASE_STATEMENT, databaseName)); - stmt.close(); - } catch (SQLException e) { -// logger.info("database {} exists!", databaseName); - } - - try { - if (connectionPoolMap.containsKey(databaseName)) { - return connectionPoolMap.get(databaseName).getConnection(); - } - Class ConnectionPoolDataSource = Class.forName(r.CONNECTION_POOL_CLASS_NAME); - Object connectionPool = ConnectionPoolDataSource.getConstructor().newInstance(); - connectionPool.setUrl(getUrl(databaseName)); - connectionPoolMap.put(databaseName, connectionPool); - return connectionPool.getConnection(); - } catch (SQLException e) { - logger.error("cannot get connection for database {}: {}", databaseName, e.getMessage()); - return null; - } - } - - @Override - public TaskExecuteResult execute(StoragePhysicalTask task) { - List operators = task.getOperators(); - if (operators.size() != 1) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - FragmentMeta fragment = task.getTargetFragment(); - Operator op = operators.get(0); - String storageUnit = task.getStorageUnit(); - boolean isDummyStorageUnit = task.isDummyStorageUnit(); - Connection conn = getConnection(storageUnit); - if (!isDummyStorageUnit && conn == null) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException(String.format("cannot connect to storage unit %s", storageUnit))); - } - - if (op.getType() == OperatorType.Project) { - Project project = (Project) op; - Filter filter; - if (operators.size() == 2) { - filter = ((Select) operators.get(1)).getFilter(); - } else { - filter = new AndFilter(Arrays - .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), - new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); - } - return isDummyStorageUnit ? executeHistoryProjectTask(project, filter) : executeProjectTask(conn, storageUnit, project, filter); - } else if (op.getType() == OperatorType.Insert) { - Insert insert = (Insert) op; - return executeInsertTask(conn, storageUnit, insert); - } else if (op.getType() == OperatorType.Delete) { - Delete delete = (Delete) op; - return executeDeleteTask(conn, storageUnit, delete); - } - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task in relationdb")); - } - - @Override - public List getTimeSeries() { - List timeseries = new ArrayList<>(); - Map extraParams = meta.getExtraParams(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); - while (databaseSet.next()) { - try { - String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 - if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { - continue; - } - Connection conn = getConnection(databaseName); - if (conn == null) { - continue; - } - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 - String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 - if (columnName.equals("time")) { // time 列不显示 - continue; - } - Pair> nameAndTags = splitFullName(columnName, r.RELATIONDB_SEPARATOR); - if (databaseName.startsWith(DATABASE_PREFIX)) { - timeseries.add(new Timeseries( - tableName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR + nameAndTags.k, - DataTypeTransformer.fromRelationDB(typeName), - nameAndTags.v) - ); - } else { - timeseries.add(new Timeseries( - databaseName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR + tableName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR + nameAndTags.k, - DataTypeTransformer.fromRelationDB(typeName), - nameAndTags.v) - ); - } - } - columnSet.close(); - } - tableSet.close(); - conn.close(); - } catch (SQLException e) { - logger.error(e.getMessage()); - } - } - databaseSet.close(); - stmt.close(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - return timeseries; - } - - @Override - public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { - long minTime = Long.MAX_VALUE, maxTime = 0; - List paths = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); - while (databaseSet.next()) { - String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 - Connection conn = getConnection(databaseName); - if (conn == null) { - continue; - } - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); - StringBuilder columnNames = new StringBuilder(); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 - paths.add(databaseName + IGINX_SEPARATOR + tableName + IGINX_SEPARATOR + columnName); - columnNames.append(columnName); - columnNames.append(", "); // c1, c2, c3, - } - columnNames = new StringBuilder(columnNames.substring(0, columnNames.length() - 2)); // c1, c2, c3 - - // 获取 key 的范围 - String statement = String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); - Statement concatStmt = conn.createStatement(); - ResultSet concatSet = concatStmt.executeQuery(statement); - while (concatSet.next()) { - String concatValue = concatSet.getString("concat"); - long time = toHash(concatValue); - minTime = Math.min(time, minTime); - maxTime = Math.max(time, maxTime); - } - } - } - stmt.close(); - } catch (SQLException e) { - throw new RuntimeException(e); - } - paths.sort(String::compareTo); - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), - new TimeInterval(minTime, maxTime + 1)); - } - - private Map splitAndMergeQueryPatterns(String databaseName, Connection conn, List patterns) throws SQLException { - // table name -> column names - // 1 -> n - Map tableNameToColumnNames = new HashMap<>(); - String tableName; - String columnNames; - - for (String pattern : patterns) { - if (pattern.equals("*") || pattern.equals("*.*")) { - tableName = "%"; - columnNames = "%"; - } else { - if (pattern.split("\\" + IGINX_SEPARATOR).length == 1) { // REST 查询的路径中可能不含 . - tableName = pattern; - columnNames = "%"; - } else { - tableName = pattern.substring(0, pattern.lastIndexOf(".")).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); - columnNames = pattern.substring(pattern.lastIndexOf(".") + 1); - boolean columnEqualsStar = columnNames.equals("*"); - boolean tableContainsStar = tableName.contains("*"); - if (columnEqualsStar || tableContainsStar) { - tableName = tableName.replace('*', '%'); - if (columnEqualsStar) { - columnNames = "%"; - if (!tableName.endsWith("%")) { - tableName += "%"; - } - } - } - } - } - - if (!columnNames.endsWith("%")) { - columnNames += "%"; // 匹配 tagKV - } - ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); - while (rs.next()) { - tableName = rs.getString("TABLE_NAME"); - columnNames = rs.getString("COLUMN_NAME"); - if (columnNames.equals("time")) { - continue; - } - if (tableNameToColumnNames.containsKey(tableName)) { - columnNames = tableNameToColumnNames.get(tableName) + ", " + columnNames; - } - tableNameToColumnNames.put(tableName, columnNames); - } - rs.close(); - } - - return tableNameToColumnNames; - } - - private TaskExecuteResult executeProjectTask(Connection conn, String databaseName, Project project, Filter filter) { - try { - List resultSets = new ArrayList<>(); - ResultSet rs; - Statement stmt; - - Map tableNameToColumnNames = splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); - for (Map.Entry entry : tableNameToColumnNames.entrySet()) { - String tableName = entry.getKey(); - String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); - String statement = String.format(QUERY_STATEMENT, columnNames, getCompleteName(tableName), FilterTransformer.toString(filter)); - try { - stmt = conn.createStatement(); - rs = stmt.executeQuery(statement); - logger.info("[Query] execute query: {}", statement); - } catch (SQLException e) { - logger.error("meet error when executing query {}: {}", statement, e.getMessage()); - continue; - } - resultSets.add(rs); - } - - RowStream rowStream = new ClearEmptyRowStreamWrapper( - new RelationDBQueryRowStream(resultSets, false, filter, project.getTagFilter(),r.RELATIONDB_SEPARATOR)); - conn.close(); - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - logger.error(e.getMessage()); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in relationdb failure", e)); - } - } - - private Map> splitAndMergeHistoryQueryPatterns(List patterns) throws SQLException { - // > - Map> splitResults = new HashMap<>(); - String databaseName; - String tableName; - String columnNames; - - for (String pattern : patterns) { - if (pattern.equals("*") || pattern.equals("*.*")) { - databaseName = "%"; - tableName = "%"; - columnNames = "%"; - } else { - String[] parts = pattern.split("\\" + IGINX_SEPARATOR); - if (parts.length > 3) { // 大于三级，不合法 - logger.error("wrong pattern: {}", pattern); - continue; - } - if (parts.length == 2 && !parts[1].equals("*")) { // 只有两级且第二级不为 *，则此 pattern 不合法，无法拆分出三级 - continue; - } - - databaseName = parts[0]; - tableName = parts[1].equals("*") ? "%" : parts[1]; - columnNames = parts[2].equals("*") ? "%" : parts[2]; - } - - if (databaseName.equals("%")) { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); - while (databaseSet.next()) { - String tempDatabaseName = databaseSet.getString("DATNAME"); - if (tempDatabaseName.startsWith(DATABASE_PREFIX)) { - continue; - } - Connection conn = getConnection(tempDatabaseName); - if (conn == null) { - continue; - } - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(tempDatabaseName, "public", tableName, new String[]{"TABLE"}); - while (tableSet.next()) { - String tempTableName = tableSet.getString("TABLE_NAME"); - ResultSet columnSet = databaseMetaData.getColumns(tempDatabaseName, "public", tempTableName, columnNames); - while (columnSet.next()) { - String tempColumnNames = columnSet.getString("COLUMN_NAME"); - Map tableNameToColumnNames = new HashMap<>(); - if (splitResults.containsKey(tempDatabaseName)) { - tableNameToColumnNames = splitResults.get(tempDatabaseName); - tempColumnNames = tableNameToColumnNames.get(tempTableName) + ", " + tempColumnNames; - } - tableNameToColumnNames.put(tempTableName, tempColumnNames); - splitResults.put(tempDatabaseName, tableNameToColumnNames); - } - } - } - } else { - Connection conn = getConnection(databaseName); - if (conn == null) { - continue; - } - ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); - while (rs.next()) { - tableName = rs.getString("TABLE_NAME"); - columnNames = rs.getString("COLUMN_NAME"); - Map tableNameToColumnNames = new HashMap<>(); - if (splitResults.containsKey(databaseName)) { - tableNameToColumnNames = splitResults.get(databaseName); - columnNames = tableNameToColumnNames.get(tableName) + ", " + columnNames; - } - tableNameToColumnNames.put(tableName, columnNames); - splitResults.put(databaseName, tableNameToColumnNames); - } - } - } - - return splitResults; - } - - private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { - try { - List resultSets = new ArrayList<>(); - ResultSet rs; - Connection conn = null; - Statement stmt; - - Map> splitResults = splitAndMergeHistoryQueryPatterns(project.getPatterns()); - for (Map.Entry> splitEntry : splitResults.entrySet()) { - String databaseName = splitEntry.getKey(); - conn = getConnection(databaseName); - if (conn == null) { - continue; - } - for (Map.Entry entry : splitEntry.getValue().entrySet()) { - String tableName = entry.getKey(); - String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); - String statement = String.format(QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, columnNames, columnNames, getCompleteName(tableName)); - try { - stmt = conn.createStatement(); - rs = stmt.executeQuery(statement); - logger.info("[Query] execute query: {}", statement); - } catch (SQLException e) { - logger.error("meet error when executing query {}: {}", statement, e.getMessage()); - continue; - } - resultSets.add(rs); - } - } - - RowStream rowStream = new ClearEmptyRowStreamWrapper( - new RelationDBQueryRowStream(resultSets, true, filter, project.getTagFilter(), r.RELATIONDB_SEPARATOR)); - if (conn != null) { - conn.close(); - } - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - logger.error(e.getMessage()); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in relationDB failure", e)); - } - } - - private TaskExecuteResult executeInsertTask(Connection conn, String storageUnit, Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertNonAlignedRowRecords(conn, storageUnit, (RowDataView) dataView); - break; - case Column: - case NonAlignedColumn: - e = insertNonAlignedColumnRecords(conn, storageUnit, (ColumnDataView) dataView); - break; - } - if (e != null) { - return new TaskExecuteResult(null, - new PhysicalException("execute insert task in relationdb failure", e)); - } - return new TaskExecuteResult(null, null); - } - - private void createOrAlterTables(Connection conn, String storageUnit, List paths, List> tagsList, List dataTypeList) { - for (int i = 0; i < paths.size(); i++) { - String path = paths.get(i); - Map tags = new HashMap<>(); - if (tagsList != null && !tagsList.isEmpty()) { - tags = tagsList.get(i); - } - DataType dataType = dataTypeList.get(i); - String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); - String columnName = path.substring(path.lastIndexOf('.') + 1).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); - - try { - Statement stmt = conn.createStatement(); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(storageUnit, "public", tableName, new String[]{"TABLE"}); - columnName = toFullName(columnName, tags, r.RELATIONDB_SEPARATOR); - if (!tableSet.next()) { - String statement = String.format(CREATE_TABLE_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), DataTypeTransformer.toRelationDB(dataType)); - logger.info("[Create] execute create: {}", statement); - stmt.execute(statement); - } else { - ResultSet columnSet = databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); - if (!columnSet.next()) { - String statement = String.format(ADD_COLUMN_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), DataTypeTransformer.toRelationDB(dataType)); - logger.info("[Create] execute create: {}", statement); - stmt.execute(statement); - } - columnSet.close(); - } - tableSet.close(); - stmt.close(); - } catch (SQLException e) { - logger.error("create or alter table {} field {} error: {}", tableName, columnName, e.getMessage()); - } - } - } - - private Exception insertNonAlignedRowRecords(Connection conn, String storageUnit, RowDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = conn.createStatement(); - - // 创建表 - createOrAlterTables(conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); - - // 插入数据 - Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> - int cnt = 0; - boolean firstRound = true; - while (cnt < data.getTimeSize()) { - int size = Math.min(data.getTimeSize() - cnt, batchSize); - Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 - for (int i = cnt; i < cnt + size; i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - String path = data.getPath(j); - DataType dataType = data.getDataType(j); - String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); - String columnName = path.substring(path.lastIndexOf('.') + 1); - Map tags = new HashMap<>(); - if (data.hasTagsList()) { - tags = data.getTags(j); - } - - StringBuilder columnKeys = new StringBuilder(); - List columnValues = new ArrayList<>(); - if (tableToColumnEntries.containsKey(tableName)) { - columnKeys = new StringBuilder(tableToColumnEntries.get(tableName).k); - columnValues = tableToColumnEntries.get(tableName).v; - } - - String value = "null"; - if (bitmapView.get(j)) { - if (dataType == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; - } else { - value = data.getValue(i, index).toString(); - } - index++; - if (tableHasData.containsKey(tableName)) { - tableHasData.get(tableName)[i - cnt] = true; - } else { - boolean[] hasData = new boolean[size]; - hasData[i - cnt] = true; - tableHasData.put(tableName, hasData); - } - } - - if (firstRound) { - columnKeys.append(toFullName(columnName, tags, r.RELATIONDB_SEPARATOR)).append(", "); - } - - if (i - cnt < columnValues.size()) { - columnValues.set(i - cnt, columnValues.get(i - cnt) + value + ", "); - } else { - columnValues.add(data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 - } - - tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); - } - - firstRound = false; - } - - for (Map.Entry entry : tableHasData.entrySet()) { - String tableName = entry.getKey(); - boolean[] hasData = entry.getValue(); - String columnKeys = tableToColumnEntries.get(tableName).k; - List columnValues = tableToColumnEntries.get(tableName).v; - boolean needToInsert = false; - for (int i = hasData.length - 1; i >= 0; i--) { - if (!hasData[i]) { - columnValues.remove(i); - } else { - needToInsert = true; - } - } - if (needToInsert) { - tableToColumnEntries.put(tableName, new Pair<>(columnKeys, columnValues)); - } - } - - executeBatchInsert(stmt, tableToColumnEntries); - for (Pair> columnEntries : tableToColumnEntries.values()) { - columnEntries.v.clear(); - } - - cnt += size; - } - stmt.close(); - conn.close(); - } catch (SQLException e) { - logger.error(e.getMessage()); - return e; - } - - return null; - } - - private Exception insertNonAlignedColumnRecords(Connection conn, String storageUnit, ColumnDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = conn.createStatement(); - - // 创建表 - createOrAlterTables(conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); - - // 插入数据 - Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> - Map pathIndexToBitmapIndex = new HashMap<>(); - int cnt = 0; - boolean firstRound = true; - while (cnt < data.getTimeSize()) { - int size = Math.min(data.getTimeSize() - cnt, batchSize); - Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 - for (int i = 0; i < data.getPathNum(); i++) { - String path = data.getPath(i); - DataType dataType = data.getDataType(i); - String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); - String columnName = path.substring(path.lastIndexOf('.') + 1); - Map tags = new HashMap<>(); - if (data.hasTagsList()) { - tags = data.getTags(i); - } - - BitmapView bitmapView = data.getBitmapView(i); - - StringBuilder columnKeys = new StringBuilder(); - List columnValues = new ArrayList<>(); - if (tableToColumnEntries.containsKey(tableName)) { - columnKeys = new StringBuilder(tableToColumnEntries.get(tableName).k); - columnValues = tableToColumnEntries.get(tableName).v; - } - - int index = 0; - if (pathIndexToBitmapIndex.containsKey(i)) { - index = pathIndexToBitmapIndex.get(i); - } - for (int j = cnt; j < cnt + size; j++) { - String value = "null"; - if (bitmapView.get(j)) { - if (dataType == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; - } else { - value = data.getValue(i, index).toString(); - } - index++; - if (tableHasData.containsKey(tableName)) { - tableHasData.get(tableName)[j - cnt] = true; - } else { - boolean[] hasData = new boolean[size]; - hasData[j - cnt] = true; - tableHasData.put(tableName, hasData); - } - } - - if (j - cnt < columnValues.size()) { - columnValues.set(j - cnt, columnValues.get(j - cnt) + value + ", "); - } else { - columnValues.add(data.getKey(j) + ", " + value + ", "); // 添加 key(time) 列 - } - } - pathIndexToBitmapIndex.put(i, index); - - if (firstRound) { - columnKeys.append(toFullName(columnName, tags, r.RELATIONDB_SEPARATOR)).append(", "); - } - - tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); - } - - for (Map.Entry entry : tableHasData.entrySet()) { - String tableName = entry.getKey(); - boolean[] hasData = entry.getValue(); - String columnKeys = tableToColumnEntries.get(tableName).k; - List columnValues = tableToColumnEntries.get(tableName).v; - boolean needToInsert = false; - for (int i = hasData.length - 1; i >= 0; i--) { - if (!hasData[i]) { - columnValues.remove(i); - } else { - needToInsert = true; - } - } - if (needToInsert) { - tableToColumnEntries.put(tableName, new Pair<>(columnKeys, columnValues)); - } - } - - executeBatchInsert(stmt, tableToColumnEntries); - for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { - entry.getValue().v.clear(); - } - - firstRound = false; - cnt += size; - } - stmt.close(); - conn.close(); - } catch (SQLException e) { - logger.error(e.getMessage()); - return e; - } - - return null; - } - - private void executeBatchInsert(Statement stmt, Map>> tableToColumnEntries) throws SQLException { - for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { - String tableName = entry.getKey(); - String columnNames = entry.getValue().k.substring(0, entry.getValue().k.length() - 2); - List values = entry.getValue().v; - String[] parts = columnNames.split(", "); - boolean hasMultipleRows = parts.length != 1; - - StringBuilder statement = new StringBuilder(); - statement.append("INSERT INTO "); - statement.append(getCompleteName(tableName)); - statement.append(" (time, "); - statement.append(Arrays.stream(parts).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("")); - statement.append(") VALUES"); - statement.append(values.stream().map(x -> " (" + x.substring(0, x.length() - 2) + ")").reduce((a, b) -> a + ", " + b).orElse("")); - statement.append(" ON CONFLICT (time) DO UPDATE SET "); - if (hasMultipleRows) { - statement.append("("); // 只有一列不加括号 - } - statement.append(Arrays.stream(parts).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("")); - if (hasMultipleRows) { - statement.append(")"); // 只有一列不加括号 - } - statement.append(" = "); - if (hasMultipleRows) { - statement.append("("); // 只有一列不加括号 - } - statement.append(Arrays.stream(parts).map(x -> "excluded." + getCompleteName(x)).reduce((a, b) -> a + ", " + b).orElse("")); - if (hasMultipleRows) { - statement.append(")"); // 只有一列不加括号 - } - statement.append(";"); - -// logger.info("[Insert] execute insert: {}", statement); - stmt.addBatch(statement.toString()); - } - stmt.executeBatch(); - } - - private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { - try { - Statement stmt = conn.createStatement(); - String statement; - List paths = delete.getPatterns(); - List> deletedPaths; // table name -> column name - String tableName; - String columnName; - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - ResultSet columnSet = null; - - if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { - if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { - conn.close(); - String defaultDB = r.DEFAULT_DATABASE; - Connection defaultConn = getConnection(defaultDB); // 正在使用的数据库无法被删除，因此需要切换到默认数据库 - if (defaultConn != null) { - stmt = defaultConn.createStatement(); - statement = String.format(DROP_DATABASE_STATEMENT, storageUnit); - logger.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 删除数据库 - stmt.close(); - defaultConn.close(); - return new TaskExecuteResult(null, null); - } else { - return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("cannot connect to database: relationdb", new SQLException())); - } - } else { - deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); - for (Pair pair : deletedPaths) { - tableName = pair.k; - columnName = pair.v; - tableSet = databaseMetaData.getTables(storageUnit, "public", tableName, new String[]{"TABLE"}); - if (tableSet.next()) { - statement = String.format(DROP_COLUMN_STATEMENT, getCompleteName(tableName), getCompleteName(columnName)); - logger.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 删除列 - } - } - } - } else { - deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); - for (Pair pair : deletedPaths) { - tableName = pair.k; - columnName = pair.v; - columnSet = databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); - if (columnSet.next()) { - for (TimeRange timeRange : delete.getTimeRanges()) { - statement = String.format(UPDATE_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), - timeRange.getBeginTime(), timeRange.getEndTime()); - logger.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 将目标列的目标范围的值置为空 - } - } - } - } - if (tableSet != null) { - tableSet.close(); - } - if (columnSet != null) { - columnSet.close(); - } - stmt.close(); - conn.close(); - return new TaskExecuteResult(null, null); - } catch (SQLException e) { - logger.error(e.getMessage()); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in relationdb failure", e)); - } - } - - private List> determineDeletedPaths(List paths, TagFilter tagFilter) { - List timeSeries = getTimeSeries(); - List> deletedPaths = new ArrayList<>(); - - for (Timeseries ts: timeSeries) { - for (String path : paths) { - if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath())) { - if (tagFilter != null && !TagKVUtils.match(ts.getTags(), tagFilter)) { - continue; - } - String fullPath = ts.getPath(); - String tableName = fullPath.substring(0, fullPath.lastIndexOf('.')).replace(IGINX_SEPARATOR, RELATIONDB_SEPARATOR); - String columnName = toFullName(fullPath.substring(fullPath.lastIndexOf('.') + 1), ts.getTags(), r.RELATIONDB_SEPARATOR); - deletedPaths.add(new Pair<>(tableName, columnName)); - break; - } - } - } - - return deletedPaths; - } - - private String getCompleteName(String name) { - return "\"" + name + "\""; -// return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; - } - - @Override - public void release() throws PhysicalException { - try { - connection.close(); - } catch (SQLException e) { - throw new PhysicalException(e); - } - } -} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java deleted file mode 100644 index 02023c0672..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/RelationDBConf.java +++ /dev/null @@ -1,17 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.conf; - -public class RelationDBConf { - public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; - public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; - public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; - - public static final String IGINX_SEPARATOR = "."; - public static final String DATABASE_PREFIX = "unit"; - public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; - public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; - public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; - public static final String UPDATE_STATEMENT = "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - public static final String CONNECTION_POOL_CLASS_NAME = null; -} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java deleted file mode 100644 index 958fa98342..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/dmdbConf.java +++ /dev/null @@ -1,19 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.conf; - -import cn.edu.tsinghua.iginx.relationdb.conf.RelationDBConf; -public class dmdbConf extends RelationDBConf { - public final boolean NEED_CHANGE_DB = false; - public static final String DB_NAME = "dmDB"; - public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; - public static final String DRIVER_URL = "dm.jdbc.driver.DmDriver"; - public static final String CONNECTION_URL = "jdbc:dm://%s:%s/?user=%s&password=%s"; - public static final String DEFAULT_USERNAME = "SYSDBA"; - public static final String DEFAULT_PASSWORD = "SYSDBA001"; - public static final String IGNORED_DATABASE = ""; //dmDB中的表空间 - public static final String CREATE_DATABASE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; - public static final String QUERY_DATABASES_STATEMENT = null; - public static final String RELATIONDB_SEPARATOR = "."; - public static final String DEFAULT_DATABASE = null; - public static final int BATCH_SIZE = 10000; - public static final String CONNECTION_POOL_CLASS_NAME = "com.huawei.dmdb.jdbc.DmdbConnectionPoolDataSource"; -} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java deleted file mode 100644 index ddae6e21fc..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/conf/mysqlConf.java +++ /dev/null @@ -1,20 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.conf; - -import cn.edu.tsinghua.iginx.relationdb.conf.RelationDBConf; -public class mysqlConf extends RelationDBConf { - - public final boolean NEED_CHANGE_DB = true; - public static final String DB_NAME = "mysql"; - public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; - public static final String DRIVER_NAME = "com.mysql.cj.jdbc.Driver"; - public static final String CONNECTION_URL = "jdbc:mysql://%s:%s/?user=%s&password=%s"; - public static final String DEFAULT_USERNAME = "root"; - public static final String DEFAULT_PASSWORD = ""; - public static final String IGNORED_DATABASE = "sys,performance_schema,information_schema"; - public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s"; - public static final String QUERY_DATABASES_STATEMENT = "SHOW DATABASES;"; - public static final String RELATIONDB_SEPARATOR = "\u2E82"; - public static final String DEFAULT_DATABASE = "mysql"; - public static final int BATCH_SIZE = 10000; - public static final String CONNECTION_POOL_CLASS_NAME = "com.mysql.cj.jdbc.MysqlConnectionPoolDataSource"; -} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java deleted file mode 100644 index 9063ea84d4..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/query/entity/RelationDBQueryRowStream.java +++ /dev/null @@ -1,243 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.query.entity; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; -import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.relationdb.tools.DataTypeTransformer; -import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.*; - -import static cn.edu.tsinghua.iginx.relationdb.conf.RelationDBConf.IGINX_SEPARATOR; -import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; -//import static cn.edu.tsinghua.iginx.relationDB.tools.Constants.RELATIONDB_SEPARATOR; -import static cn.edu.tsinghua.iginx.relationdb.tools.HashUtils.toHash; -import static cn.edu.tsinghua.iginx.relationdb.tools.TagKVUtils.splitFullName; - -public class RelationDBQueryRowStream implements RowStream { - - private static final Logger logger = LoggerFactory.getLogger(RelationDBQueryRowStream.class); - - private final List resultSets; - - private final Header header; - - private final boolean isDummy; - - private final Filter filter; - - private boolean[] gotNext; // 标记每个结果集是否已经获取到下一行，如果是，则在下次调用 next() 时无需再调用该结果集的 next() - - private long[] cachedTimestamps; // 缓存每个结果集当前的 time 列的值 - - private Object[] cachedValues; // 缓存每列当前的值 - - private int[] resultSetSizes; // 记录每个结果集的列数 - - private Map fieldToColumnName; // 记录匹配 tagFilter 的列名 - - private Row cachedRow; - - private boolean hasCachedRow; - - public RelationDBQueryRowStream(List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter, String RELATIONDB_SEPARATOR) throws SQLException { - this.resultSets = resultSets; - this.isDummy = isDummy; - this.filter = filter; - - if (resultSets.isEmpty()) { - this.header = new Header(Field.KEY, Collections.emptyList()); - return; - } - - boolean filterByTags = tagFilter != null; - - Field time = null; - List fields = new ArrayList<>(); - this.resultSetSizes = new int[resultSets.size()]; - this.fieldToColumnName = new HashMap<>(); - - for (int i = 0; i < resultSets.size(); i++) { - ResultSetMetaData resultSetMetaData = resultSets.get(i).getMetaData(); - String tableName = resultSetMetaData.getTableName(1); - int cnt = 0; - for (int j = 1; j <= resultSetMetaData.getColumnCount(); j++) { - String columnName = resultSetMetaData.getColumnName(j); - String typeName = resultSetMetaData.getColumnTypeName(j); - if (j == 1 && columnName.equals("time")) { - time = Field.KEY; - continue; - } - - Pair> namesAndTags = splitFullName(columnName,RELATIONDB_SEPARATOR); - Field field = new Field( - tableName.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + namesAndTags.k.replace(RELATIONDB_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromRelationDB(typeName), - namesAndTags.v - ); - - if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { - continue; - } - fieldToColumnName.put(field, columnName); - fields.add(field); - cnt++; - } - resultSetSizes[i] = cnt; - } - - this.header = new Header(time, fields); - - this.gotNext = new boolean[resultSets.size()]; - Arrays.fill(gotNext, false); - this.cachedTimestamps = new long[resultSets.size()]; - Arrays.fill(cachedTimestamps, Long.MAX_VALUE); - this.cachedValues = new Object[fields.size()]; - Arrays.fill(cachedValues, null); - this.cachedRow = null; - this.hasCachedRow = false; - } - - @Override - public Header getHeader() { - return header; - } - - @Override - public void close() { - try { - for (ResultSet resultSet : resultSets) { - resultSet.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage()); - } - } - - @Override - public boolean hasNext() { - if (resultSets.isEmpty()) { - return false; - } - - try { - if (!hasCachedRow) { - cacheOneRow(); - } - } catch (SQLException | PhysicalException e) { - logger.error(e.getMessage()); - } - - return cachedRow != null; - } - - @Override - public Row next() throws PhysicalException { - try { - Row row; - if (!hasCachedRow) { - cacheOneRow(); - } - row = cachedRow; - hasCachedRow = false; - cachedRow = null; - return row; - } catch (SQLException | PhysicalException e) { - logger.error(e.getMessage()); - throw new RowFetchException(e); - } - } - - private void cacheOneRow() throws SQLException, PhysicalException { - boolean hasNext = false; - long timestamp; - Object[] values = new Object[header.getFieldSize()]; - - int startIndex = 0; - int endIndex = 0; - for (int i = 0; i < resultSets.size(); i++) { - ResultSet resultSet = resultSets.get(i); - if (resultSetSizes[i] == 0) { - continue; - } - endIndex += resultSetSizes[i]; - if (!gotNext[i]) { - boolean tempHasNext = resultSet.next(); - hasNext |= tempHasNext; - gotNext[i] = true; - - if (tempHasNext) { - long tempTimestamp; - Object tempValue; - - if (isDummy) { - tempTimestamp = toHash(resultSet.getString("time")); - } else { - tempTimestamp = resultSet.getLong("time"); - } - cachedTimestamps[i] = tempTimestamp; - - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - for (int j = 0; j < resultSetSizes[i]; j++) { - Object value = resultSet.getObject(fieldToColumnName.get(header.getField(startIndex + j))); - if (header.getField(startIndex + j).getType() == DataType.BINARY && value != null) { - tempValue = value.toString().getBytes(); - } else { - tempValue = value; - } - cachedValues[startIndex + j] = tempValue; - } - } else { - cachedTimestamps[i] = Long.MAX_VALUE; - for (int j = startIndex; j < endIndex; j++) { - cachedValues[j] = null; - } - } - } else { - hasNext = true; - } - startIndex = endIndex; - } - - if (hasNext) { - timestamp = Arrays.stream(cachedTimestamps).min().getAsLong(); - startIndex = 0; - endIndex = 0; - for (int i = 0; i < resultSets.size(); i++) { - endIndex += resultSetSizes[i]; - if (cachedTimestamps[i] == timestamp) { - for (int j = 0; j < resultSetSizes[i]; j++) { - values[startIndex + j] = cachedValues[startIndex + j]; - } - gotNext[i] = false; - } else { - for (int j = 0; j < resultSetSizes[i]; j++) { - values[startIndex + j] = null; - } - gotNext[i] = true; - } - startIndex = endIndex; - } - cachedRow = new Row(header, timestamp, values); - if (isDummy && !validate(filter, cachedRow)) { - cacheOneRow(); - } - } else { - cachedRow = null; - } - hasCachedRow = true; - } -} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java deleted file mode 100644 index 1555ec54ce..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/Constants.java +++ /dev/null @@ -1,39 +0,0 @@ -//package cn.edu.tsinghua.iginx.relationdb.tools; -// -//import static cn.edu.tsinghua.iginx.relationdb.conf; -//public class Constants { -// -// public static final String IGINX_SEPARATOR = "."; -// public static final String POSTGRESQL_SEPARATOR = "\u2E82"; -// public static final int BATCH_SIZE = 10000; -// -// public static final String USERNAME = "username"; -// -// public static final String PASSWORD = "password"; -// -// public static final String DEFAULT_USERNAME = "postgres"; -// -// public static final String DEFAULT_PASSWORD = "postgres"; -// -// public static final String DATABASE_PREFIX = "unit"; -// -// public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; -// -// public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; -// -// public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; -// -// public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; -// -// public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; -// -// public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; -// -// public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; -// -// public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; -// -// public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; -// -// public static final String UPDATE_STATEMENT = "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; -//} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java deleted file mode 100644 index 7a4f1ded05..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/DataTypeTransformer.java +++ /dev/null @@ -1,43 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.tools; - -import cn.edu.tsinghua.iginx.thrift.DataType; - -import static cn.edu.tsinghua.iginx.thrift.DataType.*; - -public class DataTypeTransformer { - - public static DataType fromRelationDB(String dataType) { - if (dataType.equalsIgnoreCase("bool")) { - return BOOLEAN; - } else if (dataType.equalsIgnoreCase("int") || dataType.equalsIgnoreCase("int2") || dataType.equalsIgnoreCase("int4") || - dataType.equalsIgnoreCase("serial2") || dataType.equalsIgnoreCase("serial4")) { - return INTEGER; - } else if (dataType.equalsIgnoreCase("int8") || dataType.equalsIgnoreCase("serial8")) { - return LONG; - } else if (dataType.equalsIgnoreCase(("float4"))) { - return FLOAT; - } else if (dataType.equalsIgnoreCase("decimal") || dataType.equalsIgnoreCase("float8")) { - return DOUBLE; - } else { - return BINARY; - } - } - - public static String toRelationDB(DataType dataType) { - switch (dataType) { - case BOOLEAN: - return "BOOLEAN"; - case INTEGER: - return "INTEGER"; - case LONG: - return "BIGINT"; - case FLOAT: - return "REAL"; - case DOUBLE: - return "DOUBLE PRECISION"; - case BINARY: - default: - return "TEXT"; - } - } -} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java deleted file mode 100644 index 3d298135c4..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/FilterTransformer.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.relationdb.tools; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; - -import java.util.stream.Collectors; - -public class FilterTransformer { - - public static final long MAX_TIMESTAMP = Long.MAX_VALUE; - - public static String toString(Filter filter) { - if (filter == null) { - return ""; - } - switch (filter.getType()) { - case And: - return toString((AndFilter) filter); - case Or: - return toString((OrFilter) filter); - case Not: - return toString((NotFilter) filter); - case Value: - return toString((ValueFilter) filter); - case Key: - return toString((KeyFilter) filter); - default: - return ""; - } - } - - private static String toString(AndFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); - } - - private static String toString(NotFilter filter) { - return "not " + filter.toString(); - } - - private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " " + Math.min(filter.getValue(), MAX_TIMESTAMP); - } - - private static String toString(ValueFilter filter) { - return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); - } - - private static String toString(OrFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); - } - -} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java deleted file mode 100644 index adb8465f7d..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/HashUtils.java +++ /dev/null @@ -1,17 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.tools; - -public class HashUtils { - - public static long toHash(String s) { - char c[] = s.toCharArray(); - long hv = 0; - long base = 131; - for (int i = 0; i < c.length; i++) { - hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 - } - if (hv < 0) { - return -1 * hv; - } - return hv; - } -} diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java deleted file mode 100644 index f2a18c9033..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagFilterUtils.java +++ /dev/null @@ -1,54 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.tools; - -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; - -public class TagFilterUtils { - @SuppressWarnings("unused") - public static String transformToFilterStr(TagFilter filter) { - StringBuilder builder = new StringBuilder(); - transformToFilterStr(filter, builder); - return builder.toString(); - } - - private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { - switch (filter.getType()) { - case And: - AndTagFilter andFilter = (AndTagFilter) filter; - for (int i = 0; i < andFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(andFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" and "); - } - } - break; - case Or: - OrTagFilter orFilter = (OrTagFilter) filter; - for (int i = 0; i < orFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(orFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" or "); - } - } - break; - case Base: - BaseTagFilter baseFilter = (BaseTagFilter) filter; - builder.append(baseFilter.getTagKey()); - builder.append("="); - builder.append(baseFilter.getTagValue()); - break; - // TODO: case label - case BasePrecise: - break; - case Precise: - case WithoutTag: - break; - } - } -} \ No newline at end of file diff --git a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java b/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java deleted file mode 100644 index 12676b4e66..0000000000 --- a/dataSources/relationdb/src/main/java/cn/edu/tsinghua/iginx/relationdb/tools/TagKVUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -package cn.edu.tsinghua.iginx.relationdb.tools; - -import cn.edu.tsinghua.iginx.utils.Pair; - -import java.util.*; - -//import static cn.edu.tsinghua.iginx.relationDB.tools.Constants.RELATIONDB_SEPARATOR; - -public class TagKVUtils { - - public static Pair> splitFullName(String fullName,String RELATIONDB_SEPARATOR) { - if (!fullName.contains(RELATIONDB_SEPARATOR)) { - return new Pair<>(fullName, null); - } - - String[] parts = fullName.split("\\" + RELATIONDB_SEPARATOR); - String name = parts[0]; - - Map tags = new HashMap<>(); - for (int i = 1; i < parts.length; i++) { - if (i % 2 != 0) { - continue; - } - String tagKey = parts[i - 1]; - String tagValue = parts[i]; - tags.put(tagKey, tagValue); - } - return new Pair<>(name, tags); - } - - public static String toFullName(String name, Map tags,String RELATIONDB_SEPARATOR) { - if (tags != null && !tags.isEmpty()) { - TreeMap sortedTags = new TreeMap<>(tags); - StringBuilder pathBuilder = new StringBuilder(); - sortedTags.forEach((tagKey, tagValue) -> { - pathBuilder.append(RELATIONDB_SEPARATOR).append(tagKey).append(RELATIONDB_SEPARATOR).append(tagValue); - }); - name += pathBuilder.toString(); - } - return name; - } -} From 5f86c22f02de38a3f44786ee90b4b62f200a8765 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Mon, 17 Apr 2023 10:02:23 +0800 Subject: [PATCH 51/94] update --- conf/config.properties | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 8133d8d135..22c7783ce7 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,14 +14,14 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres #storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData #storageEngineList=127.0.0.1#5236#dmdb#username=SYSDBA#password=SYSDBA001#sessionPoolSize=20#has_data=false#is_read_only=false -storageEngineList=127.0.0.1#3306#mysql#username=root#sessionPoolSize=20#has_data=false#is_read_only=false +#storageEngineList=127.0.0.1#3306#mysql#username=root#sessionPoolSize=20#has_data=false#is_read_only=false # 写入的副本个数 replicaNum=0 diff --git a/pom.xml b/pom.xml index 90d49d4ed2..926ff7f0f2 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ dataSources/opentsdb dataSources/postgresql - dataSources/relationdb + dataSources/parquet example test From fcb29b072fca3f72880740876d378dfb0f85235a Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Thu, 20 Apr 2023 20:14:32 +0800 Subject: [PATCH 52/94] t --- conf/config.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 22c7783ce7..718bbc79bc 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,11 +14,11 @@ username=root password=root # 时序数据库列表，使用','分隔不同实例 -storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres +storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData #storageEngineList=127.0.0.1#5236#dmdb#username=SYSDBA#password=SYSDBA001#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#3306#mysql#username=root#sessionPoolSize=20#has_data=false#is_read_only=false From c4016c07f9b551ed204f679a6950ea23d6a65c68 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 21 Apr 2023 09:14:48 +0800 Subject: [PATCH 53/94] fix --- .github/workflows/DB-CE.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 976fec2c8c..761823e4fc 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -33,26 +33,26 @@ jobs: - name: Run ZooKeeper uses: ./.github/actions/zookeeperRunner - - name: Run DB - uses: ./.github/actions/dbRunner - with: - version: ${{matrix.DB-name}} + - name: Run DB + uses: ./.github/actions/dbRunner + with: + version: ${{matrix.DB-name}} - - name: Set The DB Name - uses: ./.github/actions/confWriter - with: - confFile: DBName - DB-name: ${{matrix.DB-name}} + - name: Set The DB Name + uses: ./.github/actions/confWriter + with: + confFile: DBName + DB-name: ${{matrix.DB-name}} - - name: Install IginX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P passFormat + - name: Install IginX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P passFormat - - name: Set The UDF Conf - uses: ./.github/actions/confWriter - with: - confFile: UDFConf + - name: Set The UDF Conf + uses: ./.github/actions/confWriter + with: + confFile: UDFConf #第 1 阶段测试开始========================================== - name: Prepare CapExp environment oriHasDataExpHasData From d617c8a2aaa8126f7ddbbd2c88d04fec3d17bd65 Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 21 Apr 2023 10:10:21 +0800 Subject: [PATCH 54/94] fix format --- .github/actions/dbRunner/action.yml | 20 +- .github/actions/dbWriter/action.yml | 20 +- .github/workflows/DB-CE.yml | 230 +++++++++--------- .github/workflows/standalone-test.yml | 3 +- .../engine/shared/data/write/DataView.java | 1 - 5 files changed, 137 insertions(+), 137 deletions(-) diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml index b2bd53a6a5..145fd38186 100644 --- a/.github/actions/dbRunner/action.yml +++ b/.github/actions/dbRunner/action.yml @@ -88,13 +88,13 @@ runs: name: Run DB shell: bash run: | - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/postgresql.sh" - "${GITHUB_WORKSPACE}/.github/postgresql.sh" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh" - "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh" - else - echo "$RUNNER_OS is not supported" - exit 1 - fi + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/postgresql.sh" + "${GITHUB_WORKSPACE}/.github/postgresql.sh" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh" + "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh" + else + echo "$RUNNER_OS is not supported" + exit 1 + fi diff --git a/.github/actions/dbWriter/action.yml b/.github/actions/dbWriter/action.yml index 3d680ce3f7..93eb26c16b 100644 --- a/.github/actions/dbWriter/action.yml +++ b/.github/actions/dbWriter/action.yml @@ -18,14 +18,14 @@ runs: run: | mvn test -q -Dtest=IoTDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false -P passFormat - - if: inputs.version=='InfluxDB' - name: Write history Data - shell: bash - run: | - mvn test -q -Dtest=InfluxDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false + - if: inputs.version=='InfluxDB' + name: Write history Data + shell: bash + run: | + mvn test -q -Dtest=InfluxDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false - - if: inputs.version=='PostgreSQL' - name: Write history Data - shell: bash - run: | - mvn test -q -Dtest=PostgreSQLHistoryDataGeneratorTest#${{inputs.Test-Way}} -DfailIfNoTests=false + - if: inputs.version=='PostgreSQL' + name: Write history Data + shell: bash + run: | + mvn test -q -Dtest=PostgreSQLHistoryDataGeneratorTest#${{inputs.Test-Way}} -DfailIfNoTests=false diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 761823e4fc..eb87912f17 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -13,123 +13,123 @@ concurrency: cancel-in-progress: true jobs: - DB-CE: - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - java: [ 8 ] - python-version: [ "3.7" ] - os: [ ubuntu-latest, macos-latest ] - DB-name: ["IoTDB12", "InfluxDB", "PostgreSQL"] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environmet Dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner + DB-CE: + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.7"] + os: [ubuntu-latest, macos-latest] + DB-name: ["IoTDB12", "InfluxDB", "PostgreSQL"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Environmet Dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner - - name: Run DB - uses: ./.github/actions/dbRunner - with: - version: ${{matrix.DB-name}} + - name: Run DB + uses: ./.github/actions/dbRunner + with: + version: ${{matrix.DB-name}} - - name: Set The DB Name - uses: ./.github/actions/confWriter - with: - confFile: DBName - DB-name: ${{matrix.DB-name}} + - name: Set The DB Name + uses: ./.github/actions/confWriter + with: + confFile: DBName + DB-name: ${{matrix.DB-name}} - - name: Install IginX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P passFormat + - name: Install IginX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P passFormat - - name: Set The UDF Conf - uses: ./.github/actions/confWriter - with: - confFile: UDFConf + - name: Set The UDF Conf + uses: ./.github/actions/confWriter + with: + confFile: UDFConf - #第 1 阶段测试开始========================================== - - name: Prepare CapExp environment oriHasDataExpHasData -# if: always() - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{matrix.DB-name}} - Test-Way: oriHasDataExpHasData - - name: oriHasDataExpHasData IT -# if: always() - run: | - mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpHasData -DfailIfNoTests=false - if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false - else - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - fi - #第 2 阶段测试开始========================================== - - name: Prepare CapExp environment oriNoDataExpNoData -# if: always() - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{matrix.DB-name}} - Test-Way: oriNoDataExpNoData - - name: oriNoDataExpNoData IT -# if: always() - run: | - mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriNoDataExpNoData -DfailIfNoTests=false - if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false - else - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - fi - #第 3 阶段测试开始========================================== - - name: Prepare CapExp environment oriHasDataExpNoData -# if: always() - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{matrix.DB-name}} - Test-Way: oriHasDataExpNoData - - name: oriHasDataExpNoData IT -# if: always() - run: | - mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpNoData -DfailIfNoTests=false - if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false - else - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - fi - #第 4 阶段测试开始========================================== - - name: Prepare CapExp environment oriNoDataExpHasData -# if: always() - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{matrix.DB-name}} - Test-Way: oriNoDataExpHasData - - name: oriNoDataExpHasData IT -# if: always() - run: | - if [ "${{matrix.DB-name}}" == "IoTDB12" ]; then - mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false - elif [ "${{matrix.DB-name}}" == "PostgreSQL" ]; then - mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false - elif [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=InfluxDBHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false - else - echo "${{matrix.DB-name}} is not supported" - fi - - uses: codecov/codecov-action@v1 - with: - file: ./**/target/site/jacoco/jacoco.xml - name: codecov \ No newline at end of file + #第 1 阶段测试开始========================================== + - name: Prepare CapExp environment oriHasDataExpHasData + # if: always() + uses: ./.github/actions/capacityExpansionUnionTest + with: + version: ${VERSION} + DB-name: ${{matrix.DB-name}} + Test-Way: oriHasDataExpHasData + - name: oriHasDataExpHasData IT + # if: always() + run: | + mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpHasData -DfailIfNoTests=false + if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then + mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false + else + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + fi + #第 2 阶段测试开始========================================== + - name: Prepare CapExp environment oriNoDataExpNoData + # if: always() + uses: ./.github/actions/capacityExpansionUnionTest + with: + version: ${VERSION} + DB-name: ${{matrix.DB-name}} + Test-Way: oriNoDataExpNoData + - name: oriNoDataExpNoData IT + # if: always() + run: | + mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriNoDataExpNoData -DfailIfNoTests=false + if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then + mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false + else + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + fi + #第 3 阶段测试开始========================================== + - name: Prepare CapExp environment oriHasDataExpNoData + # if: always() + uses: ./.github/actions/capacityExpansionUnionTest + with: + version: ${VERSION} + DB-name: ${{matrix.DB-name}} + Test-Way: oriHasDataExpNoData + - name: oriHasDataExpNoData IT + # if: always() + run: | + mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpNoData -DfailIfNoTests=false + if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then + mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false + else + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + fi + #第 4 阶段测试开始========================================== + - name: Prepare CapExp environment oriNoDataExpHasData + # if: always() + uses: ./.github/actions/capacityExpansionUnionTest + with: + version: ${VERSION} + DB-name: ${{matrix.DB-name}} + Test-Way: oriNoDataExpHasData + - name: oriNoDataExpHasData IT + # if: always() + run: | + if [ "${{matrix.DB-name}}" == "IoTDB12" ]; then + mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false + elif [ "${{matrix.DB-name}}" == "PostgreSQL" ]; then + mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false + elif [ "${{matrix.DB-name}}" == "InfluxDB" ]; then + mvn test -q -Dtest=InfluxDBHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false + else + echo "${{matrix.DB-name}} is not supported" + fi + - uses: codecov/codecov-action@v1 + with: + file: ./**/target/site/jacoco/jacoco.xml + name: codecov diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index 1b55e3789a..f631a78b51 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -19,7 +19,8 @@ jobs: java: [8] python-version: ["3.7"] os: [ubuntu-latest, macos-latest] - DB-name: ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "MongoDB"] + DB-name: + ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "MongoDB"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java index ae2266bdd6..53b1bf0a75 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java @@ -144,5 +144,4 @@ public Map getTags(int index) { public boolean hasTagsList() { return getTagsList() != null && getTagsList().size() != 0; } - } From d9960ea3aec8436ae45430f290e8442d0e294a9c Mon Sep 17 00:00:00 2001 From: wenyu <2025567822@qq.com> Date: Fri, 21 Apr 2023 11:52:10 +0800 Subject: [PATCH 55/94] fix --- dataSources/postgresql/pom.xml | 7 +- .../iginx/postgresql/PostgreSQLStorage.java | 435 +++++++++++++----- .../entity/PostgreSQLQueryRowStream.java | 42 +- .../iginx/postgresql/tools/Constants.java | 9 +- .../postgresql/tools/DataTypeTransformer.java | 11 +- .../postgresql/tools/FilterTransformer.java | 6 +- .../iginx/postgresql/tools/HashUtils.java | 2 +- .../postgresql/tools/TagFilterUtils.java | 4 +- .../iginx/postgresql/tools/TagKVUtils.java | 16 +- .../session/PostgreSQLSessionExample.java | 6 +- ...tgreSQLHistoryDataCapacityExpansionIT.java | 158 ++++--- .../PostgreSQLHistoryDataGeneratorTest.java | 27 +- .../integration/func/sql/SQLSessionIT.java | 9 +- 13 files changed, 470 insertions(+), 262 deletions(-) diff --git a/dataSources/postgresql/pom.xml b/dataSources/postgresql/pom.xml index 8f1a1b92f9..e28f513d5b 100644 --- a/dataSources/postgresql/pom.xml +++ b/dataSources/postgresql/pom.xml @@ -41,13 +41,12 @@ copy-dependencies - package copy-dependencies + package - ../../core/target/iginx-core-${project.version}/driver/postgresql/ - + ../../core/target/iginx-core-${project.version}/driver/postgresql/ provided @@ -60,10 +59,10 @@ copy-native-libraries - package run + package diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 51ec5c1546..233201fb91 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -18,6 +18,11 @@ */ package cn.edu.tsinghua.iginx.postgresql; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; +import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; +import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; +import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; + import cn.edu.tsinghua.iginx.engine.physical.exception.NonExecutablePhysicalTaskException; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; @@ -48,20 +53,14 @@ import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; -import org.postgresql.ds.PGConnectionPoolDataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; - -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; -import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; -import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; -import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; +import org.postgresql.ds.PGConnectionPoolDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PostgreSQLStorage implements IStorage { @@ -69,7 +68,8 @@ public class PostgreSQLStorage implements IStorage { private final StorageEngineMeta meta; - private final Map connectionPoolMap = new ConcurrentHashMap<>(); + private final Map connectionPoolMap = + new ConcurrentHashMap<>(); private final Connection connection; @@ -81,9 +81,10 @@ public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationExc Map extraParams = meta.getExtraParams(); String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); + String connUrl = + String.format( + "jdbc:postgresql://%s:%s/?user=%s&password=%s", + meta.getIp(), meta.getPort(), username, password); try { connection = DriverManager.getConnection(connUrl); } catch (SQLException e) { @@ -95,9 +96,10 @@ private boolean testConnection() { Map extraParams = meta.getExtraParams(); String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); + String connUrl = + String.format( + "jdbc:postgresql://%s:%s/?user=%s&password=%s", + meta.getIp(), meta.getPort(), username, password); try { Class.forName("org.postgresql.Driver"); DriverManager.getConnection(connUrl); @@ -111,9 +113,10 @@ private String getUrl(String databaseName) { Map extraParams = meta.getExtraParams(); String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), databaseName, - username, password); + String connUrl = + String.format( + "jdbc:postgresql://%s:%s/%s?user=%s&password=%s", + meta.getIp(), meta.getPort(), databaseName, username, password); return connUrl; } @@ -121,7 +124,8 @@ private Connection getConnection(String databaseName) { if (databaseName.startsWith("dummy")) { return null; } - if (databaseName.equalsIgnoreCase("template0") || databaseName.equalsIgnoreCase("template1")) { + if (databaseName.equalsIgnoreCase("template0") + || databaseName.equalsIgnoreCase("template1")) { return null; } @@ -130,7 +134,7 @@ private Connection getConnection(String databaseName) { stmt.execute(String.format(CREATE_DATABASE_STATEMENT, databaseName)); stmt.close(); } catch (SQLException e) { -// logger.info("database {} exists!", databaseName); + // logger.info("database {} exists!", databaseName); } try { @@ -152,7 +156,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { List operators = task.getOperators(); if (operators.size() != 1) { return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); + new NonExecutablePhysicalTaskException("unsupported physical task")); } FragmentMeta fragment = task.getTargetFragment(); Operator op = operators.get(0); @@ -161,7 +165,8 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { Connection conn = getConnection(storageUnit); if (!isDummyStorageUnit && conn == null) { return new TaskExecuteResult( - new NonExecutablePhysicalTaskException(String.format("cannot connect to storage unit %s", storageUnit))); + new NonExecutablePhysicalTaskException( + String.format("cannot connect to storage unit %s", storageUnit))); } if (op.getType() == OperatorType.Project) { @@ -170,11 +175,17 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { if (operators.size() == 2) { filter = ((Select) operators.get(1)).getFilter(); } else { - filter = new AndFilter(Arrays - .asList(new KeyFilter(Op.GE, fragment.getTimeInterval().getStartTime()), - new KeyFilter(Op.L, fragment.getTimeInterval().getEndTime()))); + filter = + new AndFilter( + Arrays.asList( + new KeyFilter( + Op.GE, fragment.getTimeInterval().getStartTime()), + new KeyFilter( + Op.L, fragment.getTimeInterval().getEndTime()))); } - return isDummyStorageUnit ? executeHistoryProjectTask(project, filter) : executeProjectTask(conn, storageUnit, project, filter); + return isDummyStorageUnit + ? executeHistoryProjectTask(project, filter) + : executeProjectTask(conn, storageUnit, project, filter); } else if (op.getType() == OperatorType.Insert) { Insert insert = (Insert) op; return executeInsertTask(conn, storageUnit, insert); @@ -183,7 +194,7 @@ public TaskExecuteResult execute(StoragePhysicalTask task) { return executeDeleteTask(conn, storageUnit, delete); } return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task in postgresql")); + new NonExecutablePhysicalTaskException("unsupported physical task in postgresql")); } @Override @@ -196,7 +207,9 @@ public List getTimeSeries() { while (databaseSet.next()) { try { String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 - if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { + if ((extraParams.get("has_data") == null + || extraParams.get("has_data").equals("false")) + && !databaseName.startsWith(DATABASE_PREFIX)) { continue; } Connection conn = getConnection(databaseName); @@ -204,32 +217,45 @@ public List getTimeSeries() { continue; } DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); + ResultSet tableSet = + databaseMetaData.getTables( + databaseName, "public", "%", new String[] {"TABLE"}); while (tableSet.next()) { String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + ResultSet columnSet = + databaseMetaData.getColumns(databaseName, "public", tableName, "%"); while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 if (columnName.equals("time")) { // time 列不显示 continue; } - Pair> nameAndTags = splitFullName(columnName); + Pair> nameAndTags = + splitFullName(columnName); if (databaseName.startsWith(DATABASE_PREFIX)) { - timeseries.add(new Timeseries( - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR + nameAndTags.k, - DataTypeTransformer.fromPostgreSQL(typeName), - nameAndTags.v) - ); + timeseries.add( + new Timeseries( + tableName.replace( + POSTGRESQL_SEPARATOR, + IGINX_SEPARATOR) + + IGINX_SEPARATOR + + nameAndTags.k, + DataTypeTransformer.fromPostgreSQL(typeName), + nameAndTags.v)); } else { - timeseries.add(new Timeseries( - databaseName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR + nameAndTags.k, - DataTypeTransformer.fromPostgreSQL(typeName), - nameAndTags.v) - ); + timeseries.add( + new Timeseries( + databaseName.replace( + POSTGRESQL_SEPARATOR, + IGINX_SEPARATOR) + + IGINX_SEPARATOR + + tableName.replace( + POSTGRESQL_SEPARATOR, + IGINX_SEPARATOR) + + IGINX_SEPARATOR + + nameAndTags.k, + DataTypeTransformer.fromPostgreSQL(typeName), + nameAndTags.v)); } } columnSet.close(); @@ -249,7 +275,8 @@ public List getTimeSeries() { } @Override - public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { + public Pair getBoundaryOfStorage(String prefix) + throws PhysicalException { long minTime = Long.MAX_VALUE, maxTime = 0; List paths = new ArrayList<>(); try { @@ -262,21 +289,33 @@ public Pair getBoundaryOfStorage(String prefix) t continue; } DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(databaseName, "public", "%", new String[]{"TABLE"}); + ResultSet tableSet = + databaseMetaData.getTables( + databaseName, "public", "%", new String[] {"TABLE"}); while (tableSet.next()) { String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + ResultSet columnSet = + databaseMetaData.getColumns(databaseName, "public", tableName, "%"); StringBuilder columnNames = new StringBuilder(); while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 - paths.add(databaseName + IGINX_SEPARATOR + tableName + IGINX_SEPARATOR + columnName); + paths.add( + databaseName + + IGINX_SEPARATOR + + tableName + + IGINX_SEPARATOR + + columnName); columnNames.append(columnName); columnNames.append(", "); // c1, c2, c3, } - columnNames = new StringBuilder(columnNames.substring(0, columnNames.length() - 2)); // c1, c2, c3 + columnNames = + new StringBuilder( + columnNames.substring( + 0, columnNames.length() - 2)); // c1, c2, c3 // 获取 key 的范围 - String statement = String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); + String statement = + String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); Statement concatStmt = conn.createStatement(); ResultSet concatSet = concatStmt.executeQuery(statement); while (concatSet.next()) { @@ -292,11 +331,13 @@ public Pair getBoundaryOfStorage(String prefix) t throw new RuntimeException(e); } paths.sort(String::compareTo); - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), - new TimeInterval(minTime, maxTime + 1)); + return new Pair<>( + new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), + new TimeInterval(minTime, maxTime + 1)); } - private Map splitAndMergeQueryPatterns(String databaseName, Connection conn, List patterns) throws SQLException { + private Map splitAndMergeQueryPatterns( + String databaseName, Connection conn, List patterns) throws SQLException { // table name -> column names // 1 -> n Map tableNameToColumnNames = new HashMap<>(); @@ -312,7 +353,9 @@ private Map splitAndMergeQueryPatterns(String databaseName, Conn tableName = pattern; columnNames = "%"; } else { - tableName = pattern.substring(0, pattern.lastIndexOf(".")).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + tableName = + pattern.substring(0, pattern.lastIndexOf(".")) + .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); columnNames = pattern.substring(pattern.lastIndexOf(".") + 1); boolean columnEqualsStar = columnNames.equals("*"); boolean tableContainsStar = tableName.contains("*"); @@ -331,7 +374,8 @@ private Map splitAndMergeQueryPatterns(String databaseName, Conn if (!columnNames.endsWith("%")) { columnNames += "%"; // 匹配 tagKV } - ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); + ResultSet rs = + conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); while (rs.next()) { tableName = rs.getString("TABLE_NAME"); columnNames = rs.getString("COLUMN_NAME"); @@ -349,40 +393,56 @@ private Map splitAndMergeQueryPatterns(String databaseName, Conn return tableNameToColumnNames; } - private TaskExecuteResult executeProjectTask(Connection conn, String databaseName, Project project, Filter filter) { + private TaskExecuteResult executeProjectTask( + Connection conn, String databaseName, Project project, Filter filter) { try { List resultSets = new ArrayList<>(); ResultSet rs; Statement stmt; - Map tableNameToColumnNames = splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); + Map tableNameToColumnNames = + splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); - String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); - String statement = String.format(QUERY_STATEMENT, columnNames, getCompleteName(tableName), FilterTransformer.toString(filter)); + String columnNames = + Arrays.stream(entry.getValue().split(", ")) + .map(this::getCompleteName) + .reduce((a, b) -> a + ", " + b) + .orElse("%"); + String statement = + String.format( + QUERY_STATEMENT, + columnNames, + getCompleteName(tableName), + FilterTransformer.toString(filter)); try { stmt = conn.createStatement(); rs = stmt.executeQuery(statement); logger.info("[Query] execute query: {}", statement); } catch (SQLException e) { - logger.error("meet error when executing query {}: {}", statement, e.getMessage()); + logger.error( + "meet error when executing query {}: {}", statement, e.getMessage()); continue; } resultSets.add(rs); } - RowStream rowStream = new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream(resultSets, false, filter, project.getTagFilter())); + RowStream rowStream = + new ClearEmptyRowStreamWrapper( + new PostgreSQLQueryRowStream( + resultSets, false, filter, project.getTagFilter())); conn.close(); return new TaskExecuteResult(rowStream); } catch (SQLException e) { logger.error(e.getMessage()); return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); + new PhysicalTaskExecuteFailureException( + "execute project task in postgresql failure", e)); } } - private Map> splitAndMergeHistoryQueryPatterns(List patterns) throws SQLException { + private Map> splitAndMergeHistoryQueryPatterns( + List patterns) throws SQLException { // > Map> splitResults = new HashMap<>(); String databaseName; @@ -400,7 +460,8 @@ private Map> splitAndMergeHistoryQueryPatterns(List< logger.error("wrong pattern: {}", pattern); continue; } - if (parts.length == 2 && !parts[1].equals("*")) { // 只有两级且第二级不为 *，则此 pattern 不合法，无法拆分出三级 + if (parts.length == 2 + && !parts[1].equals("*")) { // 只有两级且第二级不为 *，则此 pattern 不合法，无法拆分出三级 continue; } @@ -422,16 +483,23 @@ private Map> splitAndMergeHistoryQueryPatterns(List< continue; } DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(tempDatabaseName, "public", tableName, new String[]{"TABLE"}); + ResultSet tableSet = + databaseMetaData.getTables( + tempDatabaseName, "public", tableName, new String[] {"TABLE"}); while (tableSet.next()) { String tempTableName = tableSet.getString("TABLE_NAME"); - ResultSet columnSet = databaseMetaData.getColumns(tempDatabaseName, "public", tempTableName, columnNames); + ResultSet columnSet = + databaseMetaData.getColumns( + tempDatabaseName, "public", tempTableName, columnNames); while (columnSet.next()) { String tempColumnNames = columnSet.getString("COLUMN_NAME"); Map tableNameToColumnNames = new HashMap<>(); if (splitResults.containsKey(tempDatabaseName)) { tableNameToColumnNames = splitResults.get(tempDatabaseName); - tempColumnNames = tableNameToColumnNames.get(tempTableName) + ", " + tempColumnNames; + tempColumnNames = + tableNameToColumnNames.get(tempTableName) + + ", " + + tempColumnNames; } tableNameToColumnNames.put(tempTableName, tempColumnNames); splitResults.put(tempDatabaseName, tableNameToColumnNames); @@ -443,7 +511,9 @@ private Map> splitAndMergeHistoryQueryPatterns(List< if (conn == null) { continue; } - ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); + ResultSet rs = + conn.getMetaData() + .getColumns(databaseName, "public", tableName, columnNames); while (rs.next()) { tableName = rs.getString("TABLE_NAME"); columnNames = rs.getString("COLUMN_NAME"); @@ -468,7 +538,8 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt Connection conn = null; Statement stmt; - Map> splitResults = splitAndMergeHistoryQueryPatterns(project.getPatterns()); + Map> splitResults = + splitAndMergeHistoryQueryPatterns(project.getPatterns()); for (Map.Entry> splitEntry : splitResults.entrySet()) { String databaseName = splitEntry.getKey(); conn = getConnection(databaseName); @@ -477,22 +548,36 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt } for (Map.Entry entry : splitEntry.getValue().entrySet()) { String tableName = entry.getKey(); - String columnNames = Arrays.stream(entry.getValue().split(", ")).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("%"); - String statement = String.format(QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, columnNames, columnNames, getCompleteName(tableName)); + String columnNames = + Arrays.stream(entry.getValue().split(", ")) + .map(this::getCompleteName) + .reduce((a, b) -> a + ", " + b) + .orElse("%"); + String statement = + String.format( + QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, + columnNames, + columnNames, + getCompleteName(tableName)); try { stmt = conn.createStatement(); rs = stmt.executeQuery(statement); logger.info("[Query] execute query: {}", statement); } catch (SQLException e) { - logger.error("meet error when executing query {}: {}", statement, e.getMessage()); + logger.error( + "meet error when executing query {}: {}", + statement, + e.getMessage()); continue; } resultSets.add(rs); } } - RowStream rowStream = new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream(resultSets, true, filter, project.getTagFilter())); + RowStream rowStream = + new ClearEmptyRowStreamWrapper( + new PostgreSQLQueryRowStream( + resultSets, true, filter, project.getTagFilter())); if (conn != null) { conn.close(); } @@ -500,11 +585,13 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt } catch (SQLException e) { logger.error(e.getMessage()); return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in postgresql failure", e)); + new PhysicalTaskExecuteFailureException( + "execute project task in postgresql failure", e)); } } - private TaskExecuteResult executeInsertTask(Connection conn, String storageUnit, Insert insert) { + private TaskExecuteResult executeInsertTask( + Connection conn, String storageUnit, Insert insert) { DataView dataView = insert.getData(); Exception e = null; switch (dataView.getRawDataType()) { @@ -518,13 +605,18 @@ private TaskExecuteResult executeInsertTask(Connection conn, String storageUnit, break; } if (e != null) { - return new TaskExecuteResult(null, - new PhysicalException("execute insert task in postgresql failure", e)); + return new TaskExecuteResult( + null, new PhysicalException("execute insert task in postgresql failure", e)); } return new TaskExecuteResult(null, null); } - private void createOrAlterTables(Connection conn, String storageUnit, List paths, List> tagsList, List dataTypeList) { + private void createOrAlterTables( + Connection conn, + String storageUnit, + List paths, + List> tagsList, + List dataTypeList) { for (int i = 0; i < paths.size(); i++) { String path = paths.get(i); Map tags = new HashMap<>(); @@ -532,22 +624,40 @@ private void createOrAlterTables(Connection conn, String storageUnit, List>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> + Map>> tableToColumnEntries = + new HashMap<>(); // <表名, <列名，值列表>> int cnt = 0; boolean firstRound = true; while (cnt < data.getTimeSize()) { @@ -582,7 +699,9 @@ private Exception insertNonAlignedRowRecords(Connection conn, String storageUnit for (int j = 0; j < data.getPathNum(); j++) { String path = data.getPath(j); DataType dataType = data.getDataType(j); - String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String tableName = + path.substring(0, path.lastIndexOf('.')) + .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = path.substring(path.lastIndexOf('.') + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { @@ -599,7 +718,12 @@ private Exception insertNonAlignedRowRecords(Connection conn, String storageUnit String value = "null"; if (bitmapView.get(j)) { if (dataType == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + value = + "'" + + new String( + (byte[]) data.getValue(i, index), + StandardCharsets.UTF_8) + + "'"; } else { value = data.getValue(i, index).toString(); } @@ -620,10 +744,12 @@ private Exception insertNonAlignedRowRecords(Connection conn, String storageUnit if (i - cnt < columnValues.size()) { columnValues.set(i - cnt, columnValues.get(i - cnt) + value + ", "); } else { - columnValues.add(data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 + columnValues.add( + data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 } - tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); + tableToColumnEntries.put( + tableName, new Pair<>(columnKeys.toString(), columnValues)); } firstRound = false; @@ -664,16 +790,19 @@ private Exception insertNonAlignedRowRecords(Connection conn, String storageUnit return null; } - private Exception insertNonAlignedColumnRecords(Connection conn, String storageUnit, ColumnDataView data) { + private Exception insertNonAlignedColumnRecords( + Connection conn, String storageUnit, ColumnDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); // 创建表 - createOrAlterTables(conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + createOrAlterTables( + conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); // 插入数据 - Map>> tableToColumnEntries = new HashMap<>(); // <表名, <列名，值列表>> + Map>> tableToColumnEntries = + new HashMap<>(); // <表名, <列名，值列表>> Map pathIndexToBitmapIndex = new HashMap<>(); int cnt = 0; boolean firstRound = true; @@ -683,7 +812,9 @@ private Exception insertNonAlignedColumnRecords(Connection conn, String storageU for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); DataType dataType = data.getDataType(i); - String tableName = path.substring(0, path.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String tableName = + path.substring(0, path.lastIndexOf('.')) + .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = path.substring(path.lastIndexOf('.') + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { @@ -707,7 +838,12 @@ private Exception insertNonAlignedColumnRecords(Connection conn, String storageU String value = "null"; if (bitmapView.get(j)) { if (dataType == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) + "'"; + value = + "'" + + new String( + (byte[]) data.getValue(i, index), + StandardCharsets.UTF_8) + + "'"; } else { value = data.getValue(i, index).toString(); } @@ -724,7 +860,8 @@ private Exception insertNonAlignedColumnRecords(Connection conn, String storageU if (j - cnt < columnValues.size()) { columnValues.set(j - cnt, columnValues.get(j - cnt) + value + ", "); } else { - columnValues.add(data.getKey(j) + ", " + value + ", "); // 添加 key(time) 列 + columnValues.add( + data.getKey(j) + ", " + value + ", "); // 添加 key(time) 列 } } pathIndexToBitmapIndex.put(i, index); @@ -733,7 +870,8 @@ private Exception insertNonAlignedColumnRecords(Connection conn, String storageU columnKeys.append(toFullName(columnName, tags)).append(", "); } - tableToColumnEntries.put(tableName, new Pair<>(columnKeys.toString(), columnValues)); + tableToColumnEntries.put( + tableName, new Pair<>(columnKeys.toString(), columnValues)); } for (Map.Entry entry : tableHasData.entrySet()) { @@ -755,7 +893,8 @@ private Exception insertNonAlignedColumnRecords(Connection conn, String storageU } executeBatchInsert(stmt, tableToColumnEntries); - for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + for (Map.Entry>> entry : + tableToColumnEntries.entrySet()) { entry.getValue().v.clear(); } @@ -772,8 +911,11 @@ private Exception insertNonAlignedColumnRecords(Connection conn, String storageU return null; } - private void executeBatchInsert(Statement stmt, Map>> tableToColumnEntries) throws SQLException { - for (Map.Entry>> entry : tableToColumnEntries.entrySet()) { + private void executeBatchInsert( + Statement stmt, Map>> tableToColumnEntries) + throws SQLException { + for (Map.Entry>> entry : + tableToColumnEntries.entrySet()) { String tableName = entry.getKey(); String columnNames = entry.getValue().k.substring(0, entry.getValue().k.length() - 2); List values = entry.getValue().v; @@ -784,14 +926,26 @@ private void executeBatchInsert(Statement stmt, Map a + ", " + b).orElse("")); + statement.append( + Arrays.stream(parts) + .map(this::getCompleteName) + .reduce((a, b) -> a + ", " + b) + .orElse("")); statement.append(") VALUES"); - statement.append(values.stream().map(x -> " (" + x.substring(0, x.length() - 2) + ")").reduce((a, b) -> a + ", " + b).orElse("")); + statement.append( + values.stream() + .map(x -> " (" + x.substring(0, x.length() - 2) + ")") + .reduce((a, b) -> a + ", " + b) + .orElse("")); statement.append(" ON CONFLICT (time) DO UPDATE SET "); if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } - statement.append(Arrays.stream(parts).map(this::getCompleteName).reduce((a, b) -> a + ", " + b).orElse("")); + statement.append( + Arrays.stream(parts) + .map(this::getCompleteName) + .reduce((a, b) -> a + ", " + b) + .orElse("")); if (hasMultipleRows) { statement.append(")"); // 只有一列不加括号 } @@ -799,19 +953,24 @@ private void executeBatchInsert(Statement stmt, Map "excluded." + getCompleteName(x)).reduce((a, b) -> a + ", " + b).orElse("")); + statement.append( + Arrays.stream(parts) + .map(x -> "excluded." + getCompleteName(x)) + .reduce((a, b) -> a + ", " + b) + .orElse("")); if (hasMultipleRows) { statement.append(")"); // 只有一列不加括号 } statement.append(";"); -// logger.info("[Insert] execute insert: {}", statement); + // logger.info("[Insert] execute insert: {}", statement); stmt.addBatch(statement.toString()); } stmt.executeBatch(); } - private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, Delete delete) { + private TaskExecuteResult executeDeleteTask( + Connection conn, String storageUnit, Delete delete) { try { Statement stmt = conn.createStatement(); String statement; @@ -824,9 +983,12 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, ResultSet columnSet = null; if (delete.getTimeRanges() == null || delete.getTimeRanges().size() == 0) { - if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { + if (paths.size() == 1 + && paths.get(0).equals("*") + && delete.getTagFilter() == null) { conn.close(); - Connection postgresConn = getConnection("postgres"); // 正在使用的数据库无法被删除，因此需要切换到名为 postgres 的默认数据库 + Connection postgresConn = + getConnection("postgres"); // 正在使用的数据库无法被删除，因此需要切换到名为 postgres 的默认数据库 if (postgresConn != null) { stmt = postgresConn.createStatement(); statement = String.format(DROP_DATABASE_STATEMENT, storageUnit); @@ -836,16 +998,25 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, postgresConn.close(); return new TaskExecuteResult(null, null); } else { - return new TaskExecuteResult(new PhysicalTaskExecuteFailureException("cannot connect to database: postgres", new SQLException())); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + "cannot connect to database: postgres", + new SQLException())); } } else { deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); for (Pair pair : deletedPaths) { tableName = pair.k; columnName = pair.v; - tableSet = databaseMetaData.getTables(storageUnit, "public", tableName, new String[]{"TABLE"}); + tableSet = + databaseMetaData.getTables( + storageUnit, "public", tableName, new String[] {"TABLE"}); if (tableSet.next()) { - statement = String.format(DROP_COLUMN_STATEMENT, getCompleteName(tableName), getCompleteName(columnName)); + statement = + String.format( + DROP_COLUMN_STATEMENT, + getCompleteName(tableName), + getCompleteName(columnName)); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 删除列 } @@ -856,11 +1027,18 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, for (Pair pair : deletedPaths) { tableName = pair.k; columnName = pair.v; - columnSet = databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); + columnSet = + databaseMetaData.getColumns( + storageUnit, "public", tableName, columnName); if (columnSet.next()) { for (TimeRange timeRange : delete.getTimeRanges()) { - statement = String.format(UPDATE_STATEMENT, getCompleteName(tableName), getCompleteName(columnName), - timeRange.getBeginTime(), timeRange.getEndTime()); + statement = + String.format( + UPDATE_STATEMENT, + getCompleteName(tableName), + getCompleteName(columnName), + timeRange.getBeginTime(), + timeRange.getEndTime()); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 将目标列的目标范围的值置为空 } @@ -879,23 +1057,30 @@ private TaskExecuteResult executeDeleteTask(Connection conn, String storageUnit, } catch (SQLException e) { logger.error(e.getMessage()); return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in postgresql failure", e)); + new PhysicalTaskExecuteFailureException( + "execute delete task in postgresql failure", e)); } } - private List> determineDeletedPaths(List paths, TagFilter tagFilter) { + private List> determineDeletedPaths( + List paths, TagFilter tagFilter) { List timeSeries = getTimeSeries(); List> deletedPaths = new ArrayList<>(); - for (Timeseries ts: timeSeries) { + for (Timeseries ts : timeSeries) { for (String path : paths) { if (Pattern.matches(StringUtils.reformatPath(path), ts.getPath())) { if (tagFilter != null && !TagKVUtils.match(ts.getTags(), tagFilter)) { continue; } String fullPath = ts.getPath(); - String tableName = fullPath.substring(0, fullPath.lastIndexOf('.')).replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String columnName = toFullName(fullPath.substring(fullPath.lastIndexOf('.') + 1), ts.getTags()); + String tableName = + fullPath.substring(0, fullPath.lastIndexOf('.')) + .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + String columnName = + toFullName( + fullPath.substring(fullPath.lastIndexOf('.') + 1), + ts.getTags()); deletedPaths.add(new Pair<>(tableName, columnName)); break; } @@ -907,7 +1092,7 @@ private List> determineDeletedPaths(List paths, Tag private String getCompleteName(String name) { return "\"" + name + "\""; -// return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; + // return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; } @Override diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index 83ad3f74de..fff0ee007d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -1,5 +1,11 @@ package cn.edu.tsinghua.iginx.postgresql.query.entity; +import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; +import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; + import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; @@ -12,19 +18,12 @@ import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.*; - -import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; -import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; -import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PostgreSQLQueryRowStream implements RowStream { @@ -52,7 +51,9 @@ public class PostgreSQLQueryRowStream implements RowStream { private boolean hasCachedRow; - public PostgreSQLQueryRowStream(List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter) throws SQLException { + public PostgreSQLQueryRowStream( + List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter) + throws SQLException { this.resultSets = resultSets; this.isDummy = isDummy; this.filter = filter; @@ -82,12 +83,14 @@ public PostgreSQLQueryRowStream(List resultSets, boolean isDummy, Fil } Pair> namesAndTags = splitFullName(columnName); - Field field = new Field( - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + namesAndTags.k.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName), - namesAndTags.v - ); + Field field = + new Field( + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + + namesAndTags.k.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v); if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { continue; @@ -192,8 +195,11 @@ private void cacheOneRow() throws SQLException, PhysicalException { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); for (int j = 0; j < resultSetSizes[i]; j++) { - Object value = resultSet.getObject(fieldToColumnName.get(header.getField(startIndex + j))); - if (header.getField(startIndex + j).getType() == DataType.BINARY && value != null) { + Object value = + resultSet.getObject( + fieldToColumnName.get(header.getField(startIndex + j))); + if (header.getField(startIndex + j).getType() == DataType.BINARY + && value != null) { tempValue = value.toString().getBytes(); } else { tempValue = value; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 5fda6ff469..0e26b52100 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -26,9 +26,11 @@ public class Constants { public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; - public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; + public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = + "SELECT concat(%s) AS time, %s FROM %s;"; - public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; + public static final String CREATE_TABLE_STATEMENT = + "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; @@ -36,5 +38,6 @@ public class Constants { public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; - public static final String UPDATE_STATEMENT = "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; + public static final String UPDATE_STATEMENT = + "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java index 1e439beecc..b80e9abc95 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java @@ -1,16 +1,19 @@ package cn.edu.tsinghua.iginx.postgresql.tools; -import cn.edu.tsinghua.iginx.thrift.DataType; - import static cn.edu.tsinghua.iginx.thrift.DataType.*; +import cn.edu.tsinghua.iginx.thrift.DataType; + public class DataTypeTransformer { public static DataType fromPostgreSQL(String dataType) { if (dataType.equalsIgnoreCase("bool")) { return BOOLEAN; - } else if (dataType.equalsIgnoreCase("int") || dataType.equalsIgnoreCase("int2") || dataType.equalsIgnoreCase("int4") || - dataType.equalsIgnoreCase("serial2") || dataType.equalsIgnoreCase("serial4")) { + } else if (dataType.equalsIgnoreCase("int") + || dataType.equalsIgnoreCase("int2") + || dataType.equalsIgnoreCase("int4") + || dataType.equalsIgnoreCase("serial2") + || dataType.equalsIgnoreCase("serial4")) { return INTEGER; } else if (dataType.equalsIgnoreCase("int8") || dataType.equalsIgnoreCase("serial8")) { return LONG; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index f4ae6a3431..2212db5e7a 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -19,7 +19,6 @@ package cn.edu.tsinghua.iginx.postgresql.tools; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; - import java.util.stream.Collectors; public class FilterTransformer { @@ -58,7 +57,10 @@ private static String toString(NotFilter filter) { } private static String toString(KeyFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " " + Math.min(filter.getValue(), MAX_TIMESTAMP); + return "time " + + Op.op2Str(filter.getOp()) + + " " + + Math.min(filter.getValue(), MAX_TIMESTAMP); } private static String toString(ValueFilter filter) { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java index 8428ebd1c2..14091e9f1f 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java @@ -7,7 +7,7 @@ public static long toHash(String s) { long hv = 0; long base = 131; for (int i = 0; i < c.length; i++) { - hv = hv * base + (long) c[i]; //利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 + hv = hv * base + (long) c[i]; // 利用自然数溢出，即超过 LONG_MAX 自动溢出，节省时间 } if (hv < 0) { return -1 * hv; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index 62db8882e0..6943945b91 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -12,7 +12,7 @@ public static String transformToFilterStr(TagFilter filter) { transformToFilterStr(filter, builder); return builder.toString(); } - + private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { switch (filter.getType()) { case And: @@ -43,7 +43,7 @@ private static void transformToFilterStr(TagFilter filter, StringBuilder builder builder.append("="); builder.append(baseFilter.getTagValue()); break; - // TODO: case label + // TODO: case label case BasePrecise: break; case Precise: diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java index e2af27de7b..2440975717 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -1,11 +1,10 @@ package cn.edu.tsinghua.iginx.postgresql.tools; -import cn.edu.tsinghua.iginx.utils.Pair; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import cn.edu.tsinghua.iginx.utils.Pair; import java.util.*; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; - public class TagKVUtils { public static Pair> splitFullName(String fullName) { @@ -32,9 +31,14 @@ public static String toFullName(String name, Map tags) { if (tags != null && !tags.isEmpty()) { TreeMap sortedTags = new TreeMap<>(tags); StringBuilder pathBuilder = new StringBuilder(); - sortedTags.forEach((tagKey, tagValue) -> { - pathBuilder.append(POSTGRESQL_SEPARATOR).append(tagKey).append(POSTGRESQL_SEPARATOR).append(tagValue); - }); + sortedTags.forEach( + (tagKey, tagValue) -> { + pathBuilder + .append(POSTGRESQL_SEPARATOR) + .append(tagKey) + .append(POSTGRESQL_SEPARATOR) + .append(tagValue); + }); name += pathBuilder.toString(); } return name; diff --git a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java index 5fd4d58833..adf6bf27d6 100644 --- a/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java +++ b/example/src/main/java/cn/edu/tsinghua/iginx/session/PostgreSQLSessionExample.java @@ -22,10 +22,9 @@ import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.thrift.TimePrecision; -import org.apache.commons.lang3.RandomStringUtils; - import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.RandomStringUtils; public class PostgreSQLSessionExample { @@ -95,7 +94,8 @@ private static void insertColumnRecords() throws SessionException, ExecutionExce dataTypeList.add(DataType.BINARY); } - session.insertColumnRecords(paths, timestamps, valuesList, dataTypeList, null, TimePrecision.NS); + session.insertColumnRecords( + paths, timestamps, valuesList, dataTypeList, null, TimePrecision.NS); } private static void insertRowRecords() throws SessionException, ExecutionException { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java index 7da4092211..fd36bb9e27 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java @@ -3,32 +3,37 @@ import cn.edu.tsinghua.iginx.integration.expansion.CapacityExpansionIT; import cn.edu.tsinghua.iginx.integration.expansion.unit.SQLTestTools; import cn.edu.tsinghua.iginx.thrift.RemovedStorageEngineInfo; -import org.junit.Test; - import java.util.ArrayList; import java.util.List; +import org.junit.Test; public class PostgreSQLHistoryDataCapacityExpansionIT extends CapacityExpansionIT { public PostgreSQLHistoryDataCapacityExpansionIT() { super("postgresql"); } + @Test public void schemaPrefix() throws Exception { testSchemaPrefix(); } + @Test public void testSchemaPrefix() throws Exception { - session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, schema_prefix:expansion, is_read_only:true\");"); + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + + ENGINE_TYPE + + "\", \"username:postgres, password:postgres, has_data:true, schema_prefix:expansion, is_read_only:true\");"); String statement = "select * from expansion.ln.wf03"; - String expect = "ResultSets:\n" + - "+---+-----------------------------+----------------------------------+\n" + - "|key|expansion.ln.wf03.wt01.status|expansion.ln.wf03.wt01.temperature|\n" + - "+---+-----------------------------+----------------------------------+\n" + - "| 77| true| null|\n" + - "|200| false| 77.71|\n" + - "+---+-----------------------------+----------------------------------+\n" + - "Total line number = 2\n"; + String expect = + "ResultSets:\n" + + "+---+-----------------------------+----------------------------------+\n" + + "|key|expansion.ln.wf03.wt01.status|expansion.ln.wf03.wt01.temperature|\n" + + "+---+-----------------------------+----------------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+-----------------------------+----------------------------------+\n" + + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expect); statement = "count points"; @@ -38,109 +43,100 @@ public void testSchemaPrefix() throws Exception { @Test public void testDataPrefix() throws Exception { - session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:true\");"); + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + + ENGINE_TYPE + + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:true\");"); String statement = "select * from test"; - String expect = "ResultSets:\n" + - "+---+---------------------+--------------------------+\n" + - "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" + - "+---+---------------------+--------------------------+\n" + - "| 77| true| null|\n" + - "|200| false| 77.71|\n" + - "+---+---------------------+--------------------------+\n" + - "Total line number = 2\n"; + String expect = + "ResultSets:\n" + + "+---+---------------------+--------------------------+\n" + + "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" + + "+---+---------------------+--------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+---------------------+--------------------------+\n" + + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expect); statement = "select * from ln"; - expect = "ResultSets:\n" + - "+---+\n" + - "|key|\n" + - "+---+\n" + - "+---+\n" + - "Empty set.\n"; + expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; SQLTestTools.executeAndCompare(session, statement, expect); } @Test - public void testAddSameDataPrefixWithDiffSchemaPrefix_AND_testRemoveHistoryDataSource() throws Exception { - session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p1, is_read_only:true\");"); - session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p2, is_read_only:true\");"); + public void testAddSameDataPrefixWithDiffSchemaPrefix_AND_testRemoveHistoryDataSource() + throws Exception { + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + + ENGINE_TYPE + + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p1, is_read_only:true\");"); + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + + ENGINE_TYPE + + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p2, is_read_only:true\");"); String statement = "select * from p1.test"; - String expect = "ResultSets:\n" + - "+---+------------------------+-----------------------------+\n" + - "|key|p1.test.wf03.wt01.status|p1.test.wf03.wt01.temperature|\n" + - "+---+------------------------+-----------------------------+\n" + - "| 77| true| null|\n" + - "|200| false| 77.71|\n" + - "+---+------------------------+-----------------------------+\n" + - "Total line number = 2\n"; + String expect = + "ResultSets:\n" + + "+---+------------------------+-----------------------------+\n" + + "|key|p1.test.wf03.wt01.status|p1.test.wf03.wt01.temperature|\n" + + "+---+------------------------+-----------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+------------------------+-----------------------------+\n" + + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expect); statement = "select * from p2.test"; - expect = "ResultSets:\n" + - "+---+------------------------+-----------------------------+\n" + - "|key|p2.test.wf03.wt01.status|p2.test.wf03.wt01.temperature|\n" + - "+---+------------------------+-----------------------------+\n" + - "| 77| true| null|\n" + - "|200| false| 77.71|\n" + - "+---+------------------------+-----------------------------+\n" + - "Total line number = 2\n"; + expect = + "ResultSets:\n" + + "+---+------------------------+-----------------------------+\n" + + "|key|p2.test.wf03.wt01.status|p2.test.wf03.wt01.temperature|\n" + + "+---+------------------------+-----------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+------------------------+-----------------------------+\n" + + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expect); statement = "select * from test"; - expect = "ResultSets:\n" + - "+---+\n" + - "|key|\n" + - "+---+\n" + - "+---+\n" + - "Empty set.\n"; + expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; SQLTestTools.executeAndCompare(session, statement, expect); - session.executeSql("ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:false\");"); + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + + ENGINE_TYPE + + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:false\");"); statement = "select * from test"; - expect = "ResultSets:\n" + - "+---+---------------------+--------------------------+\n" + - "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" + - "+---+---------------------+--------------------------+\n" + - "| 77| true| null|\n" + - "|200| false| 77.71|\n" + - "+---+---------------------+--------------------------+\n" + - "Total line number = 2\n"; + expect = + "ResultSets:\n" + + "+---+---------------------+--------------------------+\n" + + "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" + + "+---+---------------------+--------------------------+\n" + + "| 77| true| null|\n" + + "|200| false| 77.71|\n" + + "+---+---------------------+--------------------------+\n" + + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expect); List removedStorageEngineList = new ArrayList<>(); removedStorageEngineList.add(new RemovedStorageEngineInfo("127.0.0.1", 5433, "", "test")); session.removeHistoryDataSource(removedStorageEngineList); statement = "select * from test"; - expect = "ResultSets:\n" + - "+---+\n" + - "|key|\n" + - "+---+\n" + - "+---+\n" + - "Empty set.\n"; + expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; SQLTestTools.executeAndCompare(session, statement, expect); - removedStorageEngineList.set(0, new RemovedStorageEngineInfo("127.0.0.1", 5433, "p2", "test")); + removedStorageEngineList.set( + 0, new RemovedStorageEngineInfo("127.0.0.1", 5433, "p2", "test")); sessionPool.removeHistoryDataSource(removedStorageEngineList); statement = "select * from p2.test"; - expect = "ResultSets:\n" + - "+---+\n" + - "|key|\n" + - "+---+\n" + - "+---+\n" + - "Empty set.\n"; + expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; SQLTestTools.executeAndCompare(session, statement, expect); session.executeSql("remove historydataresource (\"127.0.0.1\", 5433, \"p1\", \"test\")"); statement = "select * from p1.test"; - expect = "ResultSets:\n" + - "+---+\n" + - "|key|\n" + - "+---+\n" + - "+---+\n" + - "Empty set.\n"; + expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; SQLTestTools.executeAndCompare(session, statement, expect); } - - } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index 51b23c36aa..4c3fa4f318 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -1,22 +1,23 @@ package cn.edu.tsinghua.iginx.integration.expansion.postgresql; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator { - private static final Logger logger = LoggerFactory.getLogger(PostgreSQLHistoryDataGeneratorTest.class); + private static final Logger logger = + LoggerFactory.getLogger(PostgreSQLHistoryDataGeneratorTest.class); private static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; - private static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s, PRIMARY KEY(time));"; + private static final String CREATE_TABLE_STATEMENT = + "CREATE TABLE %s (time BIGINT NOT NULL, %s, PRIMARY KEY(time));"; private static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; @@ -77,9 +78,12 @@ public void writeHistoryDataToA() throws Exception { logger.info("database ln exists!"); } - stmt.execute(String.format(CREATE_TABLE_STATEMENT, "wf01", "status boolean, temperature float8")); + stmt.execute( + String.format( + CREATE_TABLE_STATEMENT, "wf01", "status boolean, temperature float8")); - stmt.execute(String.format(INSERT_STATEMENT, "wf01", "(100, true, null), (200, false, 20.71)")); + stmt.execute( + String.format(INSERT_STATEMENT, "wf01", "(100, true, null), (200, false, 20.71)")); stmt.close(); conn.close(); @@ -101,9 +105,12 @@ public void writeHistoryDataToB() throws Exception { logger.info("database ln exists!"); } - stmt.execute(String.format(CREATE_TABLE_STATEMENT, "wf03", "status boolean, temperature float8")); + stmt.execute( + String.format( + CREATE_TABLE_STATEMENT, "wf03", "status boolean, temperature float8")); - stmt.execute(String.format(INSERT_STATEMENT, "wf03", "(77, true, null), (200, false, 77.71)")); + stmt.execute( + String.format(INSERT_STATEMENT, "wf03", "(77, true, null), (200, false, 77.71)")); logger.info("write data to 127.0.0.1:5433 success!"); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index e97a8b3def..ce21c3860c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -3629,7 +3629,8 @@ public void testDateFormat() { + "Total line number = 12\n"; executeAndCompare(query, expected); - query = "SELECT date FROM us.d2 WHERE key >= 2021-08-26 16:15:27 AND key <= 2021.08.26T16:15:32.001 ORDER BY key;"; + query = + "SELECT date FROM us.d2 WHERE key >= 2021-08-26 16:15:27 AND key <= 2021.08.26T16:15:32.001 ORDER BY key;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3651,7 +3652,8 @@ public void testDateFormat() { + "Total line number = 12\n"; executeAndCompare(query, expected); - query = "SELECT date FROM us.d2 WHERE key >= 2021.08.26 16:15:29 AND key <= 2021-08-26T16:15:30.001 ORDER BY key;"; + query = + "SELECT date FROM us.d2 WHERE key >= 2021.08.26 16:15:29 AND key <= 2021-08-26T16:15:30.001 ORDER BY key;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3665,7 +3667,8 @@ public void testDateFormat() { + "Total line number = 4\n"; executeAndCompare(query, expected); - query = "SELECT date FROM us.d2 WHERE key >= 2021/08/26 16:15:28 AND key <= 2021/08/26T16:15:31.001 ORDER BY key;"; + query = + "SELECT date FROM us.d2 WHERE key >= 2021/08/26 16:15:28 AND key <= 2021/08/26T16:15:31.001 ORDER BY key;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" From 25c1f664f04f2d83c449b7f1c0878a0c5ff23328 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Mon, 8 May 2023 17:30:42 +0800 Subject: [PATCH 56/94] Add passFormat --- .github/workflows/DB-CE.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index c3f0c4c812..e2a0c3b276 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -65,11 +65,11 @@ jobs: - name: oriHasDataExpHasData IT # if: always() run: | - mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpHasData -DfailIfNoTests=false + mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpHasData -DfailIfNoTests=false -P passFormat if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false -P passFormat else - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P passFormat fi #第 2 阶段测试开始========================================== - name: Prepare CapExp environment oriNoDataExpNoData @@ -82,11 +82,11 @@ jobs: - name: oriNoDataExpNoData IT # if: always() run: | - mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriNoDataExpNoData -DfailIfNoTests=false + mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriNoDataExpNoData -DfailIfNoTests=false -P passFormat if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false -P passFormat else - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P passFormat fi #第 3 阶段测试开始========================================== - name: Prepare CapExp environment oriHasDataExpNoData @@ -99,11 +99,11 @@ jobs: - name: oriHasDataExpNoData IT # if: always() run: | - mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpNoData -DfailIfNoTests=false + mvn test -q -Dtest=${{matrix.DB-name}}HistoryDataCapacityExpansionIT#oriHasDataExpNoData -DfailIfNoTests=false -P passFormat if [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=${INFLUXDB-FUNCTEST} -DfailIfNoTests=false -P passFormat else - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P passFormat fi #第 4 阶段测试开始========================================== - name: Prepare CapExp environment oriNoDataExpHasData @@ -117,15 +117,15 @@ jobs: # if: always() run: | if [ "${{matrix.DB-name}}" == "IoTDB12" ]; then - mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false + mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false -P passFormat + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P passFormat + mvn test -q -Dtest=IoTDB12HistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false -P passFormat elif [ "${{matrix.DB-name}}" == "PostgreSQL" ]; then - mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false - mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false + mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false -P passFormat + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P passFormat + mvn test -q -Dtest=PostgreSQLHistoryDataCapacityExpansionIT#schemaPrefix -DfailIfNoTests=false -P passFormat elif [ "${{matrix.DB-name}}" == "InfluxDB" ]; then - mvn test -q -Dtest=InfluxDBHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false + mvn test -q -Dtest=InfluxDBHistoryDataCapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false -P passFormat else echo "${{matrix.DB-name}} is not supported" fi From 5bba69e32bff7d5d0d1825f8282d014b3b6e7b92 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Mon, 8 May 2023 17:40:19 +0800 Subject: [PATCH 57/94] Add passFormat --- .github/actions/dbWriter/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dbWriter/action.yml b/.github/actions/dbWriter/action.yml index 93eb26c16b..7a443bdccd 100644 --- a/.github/actions/dbWriter/action.yml +++ b/.github/actions/dbWriter/action.yml @@ -22,10 +22,10 @@ runs: name: Write history Data shell: bash run: | - mvn test -q -Dtest=InfluxDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false + mvn test -q -Dtest=InfluxDBHistoryDataGenerator#${{inputs.Test-Way}} -DfailIfNoTests=false -P passFormat - if: inputs.version=='PostgreSQL' name: Write history Data shell: bash run: | - mvn test -q -Dtest=PostgreSQLHistoryDataGeneratorTest#${{inputs.Test-Way}} -DfailIfNoTests=false + mvn test -q -Dtest=PostgreSQLHistoryDataGeneratorTest#${{inputs.Test-Way}} -DfailIfNoTests=false -P passFormat From c2846c66fd3636840f5d4ce33510eca885d0eee6 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 9 May 2023 09:18:12 +0800 Subject: [PATCH 58/94] Create a new branch --- .../tsinghua/iginx/postgresql/tools/TagFilterUtils.java | 1 - .../iginx/integration/controller/testConfig.properties | 5 +---- .../cn/edu/tsinghua/iginx/integration/tool/DBConf.java | 8 ++++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java index 6943945b91..02a778980b 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java @@ -45,7 +45,6 @@ private static void transformToFilterStr(TagFilter filter, StringBuilder builder break; // TODO: case label case BasePrecise: - break; case Precise: case WithoutTag: break; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties index 38f269b2b0..4c2d46aec4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/testConfig.properties @@ -18,8 +18,5 @@ influxdb-test-list={}SQLSessionIT,{}SQLSessionPoolIT,RestAnnotationIT,TransformI IoTDB12-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=true,isSupportTagKV=true InfluxDB-config=isAbleToClearData=false,isAbleToDelete=false,isAbleToShowTimeSeries=false,isSupportSpecialPath=false,isSupportTagKV=false Parquet-config=isAbleToClearData=false,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=false,isSupportTagKV=true -PostgreSQL-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=true,isSupportTagKV=true MongoDB-config=isAbleToClearData=false,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=false,isSupportTagKV=true - - - +PostgreSQL-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportSpecialPath=true,isSupportTagKV=true diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java index 539f2c7841..32a2efc4d6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java @@ -5,8 +5,8 @@ public enum DBType { parquet, iotdb12, influxdb, - postgresql, - mongodb + mongodb, + postgresql } public static DBType getDBType(String dbName) { @@ -17,10 +17,10 @@ public static DBType getDBType(String dbName) { return DBType.influxdb; case "parquet": return DBType.parquet; - case "postgresql": - return DBType.postgresql; case "mongodb": return DBType.mongodb; + case "postgresql": + return DBType.postgresql; default: throw new IllegalArgumentException("Invalid DBName value provided"); } From e50ada8927a0d24cd3d4bfff3984682983e00392 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Wed, 17 May 2023 15:23:19 +0800 Subject: [PATCH 59/94] Fix pg DB-CE --- .github/workflows/standalone-test.yml | 3 +- .../expansion/CapacityExpansionIT.java | 7 + ...tgreSQLHistoryDataCapacityExpansionIT.java | 138 ++---------------- .../func/session/BaseSessionIT.java | 15 +- .../func/session/SessionConcurrencyIT.java | 3 +- .../integration/func/session/SessionIT.java | 21 ++- 6 files changed, 38 insertions(+), 149 deletions(-) diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index cab71cd7b0..daf8cff793 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -19,7 +19,8 @@ jobs: java: [8] python-version: ["3.7"] os: [ubuntu-latest, macos-latest] - DB-name: ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"] + DB-name: + ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java index 19d3875a80..68c59e4887 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java @@ -155,6 +155,13 @@ else if (ENGINE_TYPE.toLowerCase().contains("parquet")) + "\", \"username:root, password:root, sessionPoolSize:20, has_data:" + hasData + ", is_read_only:true\");"); + else if (ENGINE_TYPE.toLowerCase().contains("postgresql")) + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5431, \"" + + ENGINE_TYPE + + "\", \"username:postgres, password:postgres, has_data:" + + hasData + + ", is_read_only:true\");"); else { logger.error("not support the DB: {}", ENGINE_TYPE); fail(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java index fd36bb9e27..4c1544c6e4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java @@ -1,142 +1,26 @@ package cn.edu.tsinghua.iginx.integration.expansion.postgresql; import cn.edu.tsinghua.iginx.integration.expansion.CapacityExpansionIT; -import cn.edu.tsinghua.iginx.integration.expansion.unit.SQLTestTools; -import cn.edu.tsinghua.iginx.thrift.RemovedStorageEngineInfo; -import java.util.ArrayList; -import java.util.List; -import org.junit.Test; public class PostgreSQLHistoryDataCapacityExpansionIT extends CapacityExpansionIT { public PostgreSQLHistoryDataCapacityExpansionIT() { super("postgresql"); } - @Test - public void schemaPrefix() throws Exception { - testSchemaPrefix(); - } - - @Test - public void testSchemaPrefix() throws Exception { + @Override + protected void addStorageWithPrefix(String dataPrefix, String schemaPrefix) throws Exception { session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + "ADD STORAGEENGINE (\"127.0.0.1\", 5431, \"" + ENGINE_TYPE - + "\", \"username:postgres, password:postgres, has_data:true, schema_prefix:expansion, is_read_only:true\");"); - - String statement = "select * from expansion.ln.wf03"; - String expect = - "ResultSets:\n" - + "+---+-----------------------------+----------------------------------+\n" - + "|key|expansion.ln.wf03.wt01.status|expansion.ln.wf03.wt01.temperature|\n" - + "+---+-----------------------------+----------------------------------+\n" - + "| 77| true| null|\n" - + "|200| false| 77.71|\n" - + "+---+-----------------------------+----------------------------------+\n" - + "Total line number = 2\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - statement = "count points"; - expect = "Points num: 3\n"; - SQLTestTools.executeAndCompare(session, statement, expect); + + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:" + + dataPrefix + + ", schema_prefix:" + + schemaPrefix + + ", is_read_only:true\");"); } - @Test - public void testDataPrefix() throws Exception { - session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" - + ENGINE_TYPE - + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:true\");"); - - String statement = "select * from test"; - String expect = - "ResultSets:\n" - + "+---+---------------------+--------------------------+\n" - + "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" - + "+---+---------------------+--------------------------+\n" - + "| 77| true| null|\n" - + "|200| false| 77.71|\n" - + "+---+---------------------+--------------------------+\n" - + "Total line number = 2\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - statement = "select * from ln"; - expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - } - - @Test - public void testAddSameDataPrefixWithDiffSchemaPrefix_AND_testRemoveHistoryDataSource() - throws Exception { - session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" - + ENGINE_TYPE - + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p1, is_read_only:true\");"); - session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" - + ENGINE_TYPE - + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, schema_prefix:p2, is_read_only:true\");"); - - String statement = "select * from p1.test"; - String expect = - "ResultSets:\n" - + "+---+------------------------+-----------------------------+\n" - + "|key|p1.test.wf03.wt01.status|p1.test.wf03.wt01.temperature|\n" - + "+---+------------------------+-----------------------------+\n" - + "| 77| true| null|\n" - + "|200| false| 77.71|\n" - + "+---+------------------------+-----------------------------+\n" - + "Total line number = 2\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - statement = "select * from p2.test"; - expect = - "ResultSets:\n" - + "+---+------------------------+-----------------------------+\n" - + "|key|p2.test.wf03.wt01.status|p2.test.wf03.wt01.temperature|\n" - + "+---+------------------------+-----------------------------+\n" - + "| 77| true| null|\n" - + "|200| false| 77.71|\n" - + "+---+------------------------+-----------------------------+\n" - + "Total line number = 2\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - statement = "select * from test"; - expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" - + ENGINE_TYPE - + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:test, is_read_only:false\");"); - statement = "select * from test"; - expect = - "ResultSets:\n" - + "+---+---------------------+--------------------------+\n" - + "|key|test.wf03.wt01.status|test.wf03.wt01.temperature|\n" - + "+---+---------------------+--------------------------+\n" - + "| 77| true| null|\n" - + "|200| false| 77.71|\n" - + "+---+---------------------+--------------------------+\n" - + "Total line number = 2\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - List removedStorageEngineList = new ArrayList<>(); - removedStorageEngineList.add(new RemovedStorageEngineInfo("127.0.0.1", 5433, "", "test")); - session.removeHistoryDataSource(removedStorageEngineList); - statement = "select * from test"; - expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - removedStorageEngineList.set( - 0, new RemovedStorageEngineInfo("127.0.0.1", 5433, "p2", "test")); - sessionPool.removeHistoryDataSource(removedStorageEngineList); - statement = "select * from p2.test"; - expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - SQLTestTools.executeAndCompare(session, statement, expect); - - session.executeSql("remove historydataresource (\"127.0.0.1\", 5433, \"p1\", \"test\")"); - statement = "select * from p1.test"; - expect = "ResultSets:\n" + "+---+\n" + "|key|\n" + "+---+\n" + "+---+\n" + "Empty set.\n"; - SQLTestTools.executeAndCompare(session, statement, expect); + @Override + protected int getPort() throws Exception { + return 5431; } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java index 025984db22..63e371430c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java @@ -1,5 +1,7 @@ package cn.edu.tsinghua.iginx.integration.func.session; +import static org.junit.Assert.fail; + import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -9,17 +11,14 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.DataType; -import org.junit.After; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - -import static org.junit.Assert.fail; +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BaseSessionIT { @@ -46,7 +45,7 @@ public abstract class BaseSessionIT { protected long delTimePeriod = delEndTime - delStartTime; protected double deleteAvg = ((START_TIME + END_TIME) * TIME_PERIOD / 2.0 - - (delStartTime + delEndTime - 1) * delTimePeriod / 2.0) + - (delStartTime + delEndTime - 1) * delTimePeriod / 2.0) / (TIME_PERIOD - delTimePeriod); protected int currPath = 0; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java index f1fd3424fe..3761f6fb65 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java @@ -12,7 +12,6 @@ import cn.edu.tsinghua.iginx.thrift.DataType; import java.util.ArrayList; import java.util.List; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +21,7 @@ public class SessionConcurrencyIT extends BaseSessionIT { // TODO: a very suspicious test; somebody should do something // TODO: The following test must be added after bug fix -// @Test + // @Test public void multiThreadTestBad() throws SessionException, InterruptedException, ExecutionException { // query test, multithread insert for storage; multithread query diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java index 0e6692b726..735aa83fdd 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java @@ -18,6 +18,13 @@ */ package cn.edu.tsinghua.iginx.integration.func.session; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.tool.CombinedInsertTests; @@ -25,21 +32,13 @@ import cn.edu.tsinghua.iginx.session.SessionQueryDataSet; import cn.edu.tsinghua.iginx.thrift.AggregateType; import cn.edu.tsinghua.iginx.thrift.DataType; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Random; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SessionIT extends BaseSessionIT { From aa941c6de3c02c56afe083bc30848514d4451004 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Wed, 17 May 2023 16:51:37 +0800 Subject: [PATCH 60/94] Add order by time to query statement --- .../java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 0e26b52100..5f5118ea72 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -24,7 +24,7 @@ public class Constants { public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; - public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s;"; + public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s ORDER BY time;"; public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; From 8c7a3651f3c3e11c6c18f50badd4ad658be79484 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Wed, 17 May 2023 17:23:43 +0800 Subject: [PATCH 61/94] Modify port number --- .github/workflows/standalone-test.yml | 3 +-- .../expansion/CapacityExpansionIT.java | 2 +- ...tgreSQLHistoryDataCapacityExpansionIT.java | 4 ++-- .../PostgreSQLHistoryDataGeneratorTest.java | 2 +- .../func/session/BaseSessionIT.java | 15 ++++++------- .../func/session/SessionConcurrencyIT.java | 3 ++- .../integration/func/session/SessionIT.java | 21 ++++++++++--------- .../iginx/integration/tool/DBConf.java | 4 ++-- test/src/test/resources/DBConf.txt | 1 + 9 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 test/src/test/resources/DBConf.txt diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index daf8cff793..cab71cd7b0 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -19,8 +19,7 @@ jobs: java: [8] python-version: ["3.7"] os: [ubuntu-latest, macos-latest] - DB-name: - ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"] + DB-name: ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java index 68c59e4887..288984c8e7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java @@ -157,7 +157,7 @@ else if (ENGINE_TYPE.toLowerCase().contains("parquet")) + ", is_read_only:true\");"); else if (ENGINE_TYPE.toLowerCase().contains("postgresql")) session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5431, \"" + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:" + hasData diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java index 4c1544c6e4..5285567b60 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java @@ -10,7 +10,7 @@ public PostgreSQLHistoryDataCapacityExpansionIT() { @Override protected void addStorageWithPrefix(String dataPrefix, String schemaPrefix) throws Exception { session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5431, \"" + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + ENGINE_TYPE + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:" + dataPrefix @@ -21,6 +21,6 @@ protected void addStorageWithPrefix(String dataPrefix, String schemaPrefix) thro @Override protected int getPort() throws Exception { - return 5431; + return 5433; } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index 4c3fa4f318..e4edce16f9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -56,7 +56,7 @@ public void clearData() { Statement stmtB = connB.createStatement(); stmtB.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); stmtB.close(); - stmtB.close(); + connB.close(); } catch (Exception e) { logger.error(e.getMessage()); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java index 63e371430c..025984db22 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java @@ -1,7 +1,5 @@ package cn.edu.tsinghua.iginx.integration.func.session; -import static org.junit.Assert.fail; - import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -11,15 +9,18 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.DataType; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.junit.After; import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.fail; + public abstract class BaseSessionIT { private static final Logger logger = LoggerFactory.getLogger(BaseSessionIT.class); @@ -45,7 +46,7 @@ public abstract class BaseSessionIT { protected long delTimePeriod = delEndTime - delStartTime; protected double deleteAvg = ((START_TIME + END_TIME) * TIME_PERIOD / 2.0 - - (delStartTime + delEndTime - 1) * delTimePeriod / 2.0) + - (delStartTime + delEndTime - 1) * delTimePeriod / 2.0) / (TIME_PERIOD - delTimePeriod); protected int currPath = 0; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java index 3761f6fb65..f1fd3424fe 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java @@ -12,6 +12,7 @@ import cn.edu.tsinghua.iginx.thrift.DataType; import java.util.ArrayList; import java.util.List; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +22,7 @@ public class SessionConcurrencyIT extends BaseSessionIT { // TODO: a very suspicious test; somebody should do something // TODO: The following test must be added after bug fix - // @Test +// @Test public void multiThreadTestBad() throws SessionException, InterruptedException, ExecutionException { // query test, multithread insert for storage; multithread query diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java index 735aa83fdd..0e6692b726 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java @@ -18,13 +18,6 @@ */ package cn.edu.tsinghua.iginx.integration.func.session; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.tool.CombinedInsertTests; @@ -32,13 +25,21 @@ import cn.edu.tsinghua.iginx.session.SessionQueryDataSet; import cn.edu.tsinghua.iginx.thrift.AggregateType; import cn.edu.tsinghua.iginx.thrift.DataType; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Random; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class SessionIT extends BaseSessionIT { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java index b3434a74f6..d75569c25b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java @@ -18,10 +18,10 @@ public static DBType getDBType(String dbName) { return DBType.influxdb; case "parquet": return DBType.parquet; - case "mongodb": - return DBType.mongodb; case "postgresql": return DBType.postgresql; + case "mongodb": + return DBType.mongodb; case "redis": return DBType.redis; default: diff --git a/test/src/test/resources/DBConf.txt b/test/src/test/resources/DBConf.txt new file mode 100644 index 0000000000..bd03896306 --- /dev/null +++ b/test/src/test/resources/DBConf.txt @@ -0,0 +1 @@ +PostgreSQL \ No newline at end of file From 254d1a6f6a8faf0fff6eb9e0aba9f4056a6adfde Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Wed, 17 May 2023 17:25:31 +0800 Subject: [PATCH 62/94] Remove DBConf.txt --- test/src/test/resources/DBConf.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test/src/test/resources/DBConf.txt diff --git a/test/src/test/resources/DBConf.txt b/test/src/test/resources/DBConf.txt deleted file mode 100644 index bd03896306..0000000000 --- a/test/src/test/resources/DBConf.txt +++ /dev/null @@ -1 +0,0 @@ -PostgreSQL \ No newline at end of file From e7417408a27f7b78fac1d1f6ea7263c01149c80c Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Wed, 17 May 2023 17:46:46 +0800 Subject: [PATCH 63/94] Change conf --- conf/config.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/config.properties b/conf/config.properties index 9ef154948b..32a19a6ed9 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -18,7 +18,7 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres +#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData #storageEngineList=127.0.0.1#5236#dmdb#username=SYSDBA#password=SYSDBA001#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#3306#mysql#username=root#sessionPoolSize=20#has_data=false#is_read_only=false From 4b6a0c96cd4ca83c65ccb60ffdeabbabd64c23f7 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 18 May 2023 15:56:59 +0800 Subject: [PATCH 64/94] Remove unused lines --- conf/config.properties | 2 -- .../tsinghua/iginx/integration/func/sql/SQLSessionIT.java | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 32a19a6ed9..4548bbc556 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -20,8 +20,6 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres #storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData -#storageEngineList=127.0.0.1#5236#dmdb#username=SYSDBA#password=SYSDBA001#sessionPoolSize=20#has_data=false#is_read_only=false -#storageEngineList=127.0.0.1#3306#mysql#username=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#27017#mongodb #storageEngineList=127.0.0.1#6379#redis diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index 722a73141a..4e83d3e5ca 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -3553,7 +3553,7 @@ public void testDateFormat() { executor.execute(String.format(insert, dateFormats.get(i), i)); } - String query = "SELECT date FROM us.d2 ORDER BY key;"; + String query = "SELECT date FROM us.d2;"; String expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3576,7 +3576,7 @@ public void testDateFormat() { executor.executeAndCompare(query, expected); query = - "SELECT date FROM us.d2 WHERE key >= 2021-08-26 16:15:27 AND key <= 2021.08.26T16:15:32.001 ORDER BY key;"; + "SELECT date FROM us.d2 WHERE key >= 2021-08-26 16:15:27 AND key <= 2021.08.26T16:15:32.001;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3599,7 +3599,7 @@ public void testDateFormat() { executor.executeAndCompare(query, expected); query = - "SELECT date FROM us.d2 WHERE key >= 2021.08.26 16:15:29 AND key <= 2021-08-26T16:15:30.001 ORDER BY key;"; + "SELECT date FROM us.d2 WHERE key >= 2021.08.26 16:15:29 AND key <= 2021-08-26T16:15:30.001;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" @@ -3614,7 +3614,7 @@ public void testDateFormat() { executor.executeAndCompare(query, expected); query = - "SELECT date FROM us.d2 WHERE key >= 2021/08/26 16:15:28 AND key <= 2021/08/26T16:15:31.001 ORDER BY key;"; + "SELECT date FROM us.d2 WHERE key >= 2021/08/26 16:15:28 AND key <= 2021/08/26T16:15:31.001;"; expected = "ResultSets:\n" + "+-------------------+----------+\n" From d679510e1a78f126e0987e285539e2e93a384974 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 18 May 2023 16:44:45 +0800 Subject: [PATCH 65/94] Specify database when generating history data --- .github/workflows/standalone-test.yml | 3 ++- .../PostgreSQLHistoryDataGeneratorTest.java | 2 +- .../func/session/BaseSessionIT.java | 15 +++++++------ .../func/session/SessionConcurrencyIT.java | 3 +-- .../integration/func/session/SessionIT.java | 21 +++++++++---------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index cab71cd7b0..daf8cff793 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -19,7 +19,8 @@ jobs: java: [8] python-version: ["3.7"] os: [ubuntu-latest, macos-latest] - DB-name: ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"] + DB-name: + ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index e4edce16f9..6c8759a902 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -35,7 +35,7 @@ public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator private Connection connect(int port) { try { - String url = String.format("jdbc:postgresql://127.0.0.1:%d/", port); + String url = String.format("jdbc:postgresql://127.0.0.1:%d/%s", port, DATABASE_NAME); Class.forName("org.postgresql.Driver"); return DriverManager.getConnection(url, USERNAME, PASSWORD); } catch (SQLException | ClassNotFoundException e) { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java index 025984db22..63e371430c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/BaseSessionIT.java @@ -1,5 +1,7 @@ package cn.edu.tsinghua.iginx.integration.func.session; +import static org.junit.Assert.fail; + import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -9,17 +11,14 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.DataType; -import org.junit.After; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; - -import static org.junit.Assert.fail; +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BaseSessionIT { @@ -46,7 +45,7 @@ public abstract class BaseSessionIT { protected long delTimePeriod = delEndTime - delStartTime; protected double deleteAvg = ((START_TIME + END_TIME) * TIME_PERIOD / 2.0 - - (delStartTime + delEndTime - 1) * delTimePeriod / 2.0) + - (delStartTime + delEndTime - 1) * delTimePeriod / 2.0) / (TIME_PERIOD - delTimePeriod); protected int currPath = 0; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java index f1fd3424fe..3761f6fb65 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionConcurrencyIT.java @@ -12,7 +12,6 @@ import cn.edu.tsinghua.iginx.thrift.DataType; import java.util.ArrayList; import java.util.List; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +21,7 @@ public class SessionConcurrencyIT extends BaseSessionIT { // TODO: a very suspicious test; somebody should do something // TODO: The following test must be added after bug fix -// @Test + // @Test public void multiThreadTestBad() throws SessionException, InterruptedException, ExecutionException { // query test, multithread insert for storage; multithread query diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java index 0e6692b726..735aa83fdd 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionIT.java @@ -18,6 +18,13 @@ */ package cn.edu.tsinghua.iginx.integration.func.session; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import cn.edu.tsinghua.iginx.exceptions.ExecutionException; import cn.edu.tsinghua.iginx.exceptions.SessionException; import cn.edu.tsinghua.iginx.integration.tool.CombinedInsertTests; @@ -25,21 +32,13 @@ import cn.edu.tsinghua.iginx.session.SessionQueryDataSet; import cn.edu.tsinghua.iginx.thrift.AggregateType; import cn.edu.tsinghua.iginx.thrift.DataType; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Random; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SessionIT extends BaseSessionIT { From f3fbed6e392897797faf94013443be40bbe45d4f Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 18 May 2023 16:57:44 +0800 Subject: [PATCH 66/94] Switch databases properly --- .../PostgreSQLHistoryDataGeneratorTest.java | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index 6c8759a902..f3c296bb58 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -33,9 +33,14 @@ public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator private static final String DATABASE_NAME = "ln"; - private Connection connect(int port) { + private Connection connect(int port, boolean useSystemDatabase) { try { - String url = String.format("jdbc:postgresql://127.0.0.1:%d/%s", port, DATABASE_NAME); + String url; + if (useSystemDatabase) { + url = String.format("jdbc:postgresql://127.0.0.1:%d/", port); + } else { + url = String.format("jdbc:postgresql://127.0.0.1:%d/%s", port, DATABASE_NAME); + } Class.forName("org.postgresql.Driver"); return DriverManager.getConnection(url, USERNAME, PASSWORD); } catch (SQLException | ClassNotFoundException e) { @@ -46,13 +51,13 @@ private Connection connect(int port) { @Test public void clearData() { try { - Connection connA = connect(PORT_A); + Connection connA = connect(PORT_A, true); Statement stmtA = connA.createStatement(); stmtA.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); stmtA.close(); connA.close(); - Connection connB = connect(PORT_B); + Connection connB = connect(PORT_B, true); Statement stmtB = connB.createStatement(); stmtB.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); stmtB.close(); @@ -65,7 +70,7 @@ public void clearData() { @Test public void writeHistoryDataToA() throws Exception { - Connection conn = connect(PORT_A); + Connection conn = connect(PORT_A, true); if (conn == null) { logger.error("cannot connect to 5432!"); return; @@ -78,10 +83,15 @@ public void writeHistoryDataToA() throws Exception { logger.info("database ln exists!"); } + stmt.close(); + conn.close(); + + conn = connect(PORT_A, false); + stmt = conn.createStatement(); + stmt.execute( String.format( CREATE_TABLE_STATEMENT, "wf01", "status boolean, temperature float8")); - stmt.execute( String.format(INSERT_STATEMENT, "wf01", "(100, true, null), (200, false, 20.71)")); @@ -92,7 +102,7 @@ public void writeHistoryDataToA() throws Exception { @Test public void writeHistoryDataToB() throws Exception { - Connection conn = connect(PORT_B); + Connection conn = connect(PORT_B, true); if (conn == null) { logger.error("cannot connect to 5433!"); return; @@ -105,6 +115,12 @@ public void writeHistoryDataToB() throws Exception { logger.info("database ln exists!"); } + stmt.close(); + conn.close(); + + conn = connect(PORT_B, false); + stmt = conn.createStatement(); + stmt.execute( String.format( CREATE_TABLE_STATEMENT, "wf03", "status boolean, temperature float8")); From 388b0c43001f855176064709e67941ff7d59b607 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 00:23:40 +0800 Subject: [PATCH 67/94] Enable dummy tables with time column --- .../iginx/postgresql/PostgreSQLStorage.java | 120 ++++++++++++++---- .../iginx/postgresql/entity/TableType.java | 7 + .../entity/PostgreSQLQueryRowStream.java | 46 +++++-- .../iginx/postgresql/tools/Constants.java | 7 +- 4 files changed, 140 insertions(+), 40 deletions(-) create mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 233201fb91..bbc24538ca 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -19,6 +19,7 @@ package cn.edu.tsinghua.iginx.postgresql; import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; +import static cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer.fromPostgreSQL; import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; @@ -47,6 +48,7 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.metadata.entity.*; +import cn.edu.tsinghua.iginx.postgresql.entity.TableType; import cn.edu.tsinghua.iginx.postgresql.query.entity.PostgreSQLQueryRowStream; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; @@ -240,7 +242,7 @@ public List getTimeSeries() { IGINX_SEPARATOR) + IGINX_SEPARATOR + nameAndTags.k, - DataTypeTransformer.fromPostgreSQL(typeName), + fromPostgreSQL(typeName), nameAndTags.v)); } else { timeseries.add( @@ -254,7 +256,7 @@ public List getTimeSeries() { IGINX_SEPARATOR) + IGINX_SEPARATOR + nameAndTags.k, - DataTypeTransformer.fromPostgreSQL(typeName), + fromPostgreSQL(typeName), nameAndTags.v)); } } @@ -297,8 +299,14 @@ public Pair getBoundaryOfStorage(String prefix) ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); StringBuilder columnNames = new StringBuilder(); + boolean hasTimeColumn = false; while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 + String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 + if (columnName.equalsIgnoreCase("time") + && fromPostgreSQL(typeName) == DataType.LONG) { + hasTimeColumn = true; + } paths.add( databaseName + IGINX_SEPARATOR @@ -308,21 +316,33 @@ public Pair getBoundaryOfStorage(String prefix) columnNames.append(columnName); columnNames.append(", "); // c1, c2, c3, } - columnNames = - new StringBuilder( - columnNames.substring( - 0, columnNames.length() - 2)); // c1, c2, c3 + if (hasTimeColumn) { // 本身就有 time 列 + // 获取 key 的范围 + String statement = String.format(QUERY_TIME_STATEMENT, tableName); + Statement timeStmt = conn.createStatement(); + ResultSet timeSet = timeStmt.executeQuery(statement); + while (timeSet.next()) { + long time = timeSet.getLong("time"); + minTime = Math.min(time, minTime); + maxTime = Math.max(time, maxTime); + } + } else { + columnNames = + new StringBuilder( + columnNames.substring( + 0, columnNames.length() - 2)); // c1, c2, c3 - // 获取 key 的范围 - String statement = - String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); - Statement concatStmt = conn.createStatement(); - ResultSet concatSet = concatStmt.executeQuery(statement); - while (concatSet.next()) { - String concatValue = concatSet.getString("concat"); - long time = toHash(concatValue); - minTime = Math.min(time, minTime); - maxTime = Math.max(time, maxTime); + // 获取 key 的范围 + String statement = + String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); + Statement concatStmt = conn.createStatement(); + ResultSet concatSet = concatStmt.executeQuery(statement); + while (concatSet.next()) { + String concatValue = concatSet.getString("concat"); + long time = toHash(concatValue); + minTime = Math.min(time, minTime); + maxTime = Math.max(time, maxTime); + } } } } @@ -396,7 +416,9 @@ private Map splitAndMergeQueryPatterns( private TaskExecuteResult executeProjectTask( Connection conn, String databaseName, Project project, Filter filter) { try { + List databaseNameList = new ArrayList<>(); List resultSets = new ArrayList<>(); + List isDummy = new ArrayList<>(); ResultSet rs; Statement stmt; @@ -424,13 +446,19 @@ private TaskExecuteResult executeProjectTask( "meet error when executing query {}: {}", statement, e.getMessage()); continue; } + databaseNameList.add(databaseName); resultSets.add(rs); + isDummy.add(TableType.nonDummy); } RowStream rowStream = new ClearEmptyRowStreamWrapper( new PostgreSQLQueryRowStream( - resultSets, false, filter, project.getTagFilter())); + databaseNameList, + resultSets, + isDummy, + filter, + project.getTagFilter())); conn.close(); return new TaskExecuteResult(rowStream); } catch (SQLException e) { @@ -467,7 +495,11 @@ private Map> splitAndMergeHistoryQueryPatterns( databaseName = parts[0]; tableName = parts[1].equals("*") ? "%" : parts[1]; - columnNames = parts[2].equals("*") ? "%" : parts[2]; + if (parts.length == 2) { + columnNames = "%"; + } else { + columnNames = parts[2].equals("*") ? "%" : parts[2]; + } } if (databaseName.equals("%")) { @@ -533,7 +565,9 @@ private Map> splitAndMergeHistoryQueryPatterns( private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { try { + List databaseNameList = new ArrayList<>(); List resultSets = new ArrayList<>(); + List isDummy = new ArrayList<>(); ResultSet rs; Connection conn = null; Statement stmt; @@ -547,18 +581,42 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt continue; } for (Map.Entry entry : splitEntry.getValue().entrySet()) { + boolean hasTimeColumn = false; String tableName = entry.getKey(); + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet columnSet = + databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + while (columnSet.next()) { + if (columnSet.getString("COLUMN_NAME").equalsIgnoreCase("time")) { + hasTimeColumn = true; + break; + } + } + + String[] parts = entry.getValue().split(", "); String columnNames = - Arrays.stream(entry.getValue().split(", ")) + Arrays.stream(parts) .map(this::getCompleteName) .reduce((a, b) -> a + ", " + b) .orElse("%"); - String statement = - String.format( - QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, - columnNames, - columnNames, - getCompleteName(tableName)); + String statement; + if (hasTimeColumn) { + if (!columnNames.contains("time")) { + columnNames += " , \"time\""; + } + statement = + String.format( + QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE, + columnNames, + getCompleteName(tableName)); + } else { + statement = + String.format( + CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, + columnNames, + columnNames, + getCompleteName(tableName)); + } try { stmt = conn.createStatement(); rs = stmt.executeQuery(statement); @@ -570,14 +628,24 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt e.getMessage()); continue; } + databaseNameList.add(databaseName); resultSets.add(rs); + if (hasTimeColumn) { + isDummy.add(TableType.dummyWithTimeColumn); + } else { + isDummy.add(TableType.dummy); + } } } RowStream rowStream = new ClearEmptyRowStreamWrapper( new PostgreSQLQueryRowStream( - resultSets, true, filter, project.getTagFilter())); + databaseNameList, + resultSets, + isDummy, + filter, + project.getTagFilter())); if (conn != null) { conn.close(); } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java new file mode 100644 index 0000000000..3617f4feea --- /dev/null +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java @@ -0,0 +1,7 @@ +package cn.edu.tsinghua.iginx.postgresql.entity; + +public enum TableType { + dummy, + dummyWithTimeColumn, + nonDummy +} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index fff0ee007d..05f83ae520 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -15,6 +15,7 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; +import cn.edu.tsinghua.iginx.postgresql.entity.TableType; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; @@ -33,7 +34,7 @@ public class PostgreSQLQueryRowStream implements RowStream { private final Header header; - private final boolean isDummy; + private final List isDummy; private final Filter filter; @@ -52,7 +53,11 @@ public class PostgreSQLQueryRowStream implements RowStream { private boolean hasCachedRow; public PostgreSQLQueryRowStream( - List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter) + List databaseNameList, + List resultSets, + List isDummy, + Filter filter, + TagFilter tagFilter) throws SQLException { this.resultSets = resultSets; this.isDummy = isDummy; @@ -83,14 +88,29 @@ public PostgreSQLQueryRowStream( } Pair> namesAndTags = splitFullName(columnName); - Field field = - new Field( - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR - + namesAndTags.k.replace( - POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName), - namesAndTags.v); + Field field; + if (isDummy.get(i) == TableType.nonDummy) { + field = + new Field( + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + + namesAndTags.k.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v); + } else { + field = + new Field( + databaseNameList.get(i) + + IGINX_SEPARATOR + + tableName.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + + namesAndTags.k.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v); + } if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { continue; @@ -186,14 +206,13 @@ private void cacheOneRow() throws SQLException, PhysicalException { long tempTimestamp; Object tempValue; - if (isDummy) { + if (isDummy.get(i) == TableType.dummy) { tempTimestamp = toHash(resultSet.getString("time")); } else { tempTimestamp = resultSet.getLong("time"); } cachedTimestamps[i] = tempTimestamp; - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); for (int j = 0; j < resultSetSizes[i]; j++) { Object value = resultSet.getObject( @@ -238,7 +257,8 @@ private void cacheOneRow() throws SQLException, PhysicalException { startIndex = endIndex; } cachedRow = new Row(header, timestamp, values); - if (isDummy && !validate(filter, cachedRow)) { + if (isDummy.stream().allMatch(x -> x == TableType.dummy) + && !validate(filter, cachedRow)) { cacheOneRow(); } } else { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 5f5118ea72..a40430e87c 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -22,11 +22,16 @@ public class Constants { public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; + public static final String QUERY_TIME_STATEMENT = "SELECT time FROM %s ORDER BY time;"; + public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s ORDER BY time;"; - public static final String QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = + public static final String QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE = + "SELECT %s FROM %s ORDER BY time;"; + + public static final String CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = "SELECT concat(%s) AS time, %s FROM %s;"; public static final String CREATE_TABLE_STATEMENT = From 551b77a3d9fe0cdfb7f057ea9fc30cab02420391 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 00:33:25 +0800 Subject: [PATCH 68/94] Fix history data generator --- .../postgresql/PostgreSQLHistoryDataGeneratorTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index f3c296bb58..e9c0ac238e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -9,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +// TODO 通过 BaseHistoryDataGenerator 里的 seriesA 和 seriesB 插入数据 public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator { private static final Logger logger = @@ -91,9 +92,9 @@ public void writeHistoryDataToA() throws Exception { stmt.execute( String.format( - CREATE_TABLE_STATEMENT, "wf01", "status boolean, temperature float8")); + CREATE_TABLE_STATEMENT, "wf01.wt01", "status boolean, temperature float8")); stmt.execute( - String.format(INSERT_STATEMENT, "wf01", "(100, true, null), (200, false, 20.71)")); + String.format(INSERT_STATEMENT, "wf01.wt01", "(100, true, null), (200, false, 20.71)")); stmt.close(); conn.close(); @@ -123,10 +124,10 @@ public void writeHistoryDataToB() throws Exception { stmt.execute( String.format( - CREATE_TABLE_STATEMENT, "wf03", "status boolean, temperature float8")); + CREATE_TABLE_STATEMENT, "wf03.wt01", "status boolean, temperature float8")); stmt.execute( - String.format(INSERT_STATEMENT, "wf03", "(77, true, null), (200, false, 77.71)")); + String.format(INSERT_STATEMENT, "wf03.wt01", "(77, true, null), (200, false, 77.71)")); logger.info("write data to 127.0.0.1:5433 success!"); } From 547165a9d92f75bd0396c231434a4b1e996d883d Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 00:42:17 +0800 Subject: [PATCH 69/94] Fix history data generator --- .../postgresql/PostgreSQLHistoryDataGeneratorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index e9c0ac238e..bed1e5d74a 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -92,9 +92,9 @@ public void writeHistoryDataToA() throws Exception { stmt.execute( String.format( - CREATE_TABLE_STATEMENT, "wf01.wt01", "status boolean, temperature float8")); + CREATE_TABLE_STATEMENT, "wf01\u2E82wt01", "status boolean, temperature float8")); stmt.execute( - String.format(INSERT_STATEMENT, "wf01.wt01", "(100, true, null), (200, false, 20.71)")); + String.format(INSERT_STATEMENT, "wf01\u2E82wt01", "(100, true, null), (200, false, 20.71)")); stmt.close(); conn.close(); @@ -124,10 +124,10 @@ public void writeHistoryDataToB() throws Exception { stmt.execute( String.format( - CREATE_TABLE_STATEMENT, "wf03.wt01", "status boolean, temperature float8")); + CREATE_TABLE_STATEMENT, "wf03\u2E82wt01", "status boolean, temperature float8")); stmt.execute( - String.format(INSERT_STATEMENT, "wf03.wt01", "(77, true, null), (200, false, 77.71)")); + String.format(INSERT_STATEMENT, "wf03\u2E82wt01", "(77, true, null), (200, false, 77.71)")); logger.info("write data to 127.0.0.1:5433 success!"); } From 9f0cd0528db4aba95124fe252970100caee2175d Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 10:05:46 +0800 Subject: [PATCH 70/94] Fix history data generator --- .../iginx/postgresql/PostgreSQLStorage.java | 33 +++++++++++-------- .../PostgreSQLHistoryDataGeneratorTest.java | 18 +++++++--- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index bbc24538ca..bda1446ace 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -310,7 +310,7 @@ && fromPostgreSQL(typeName) == DataType.LONG) { paths.add( databaseName + IGINX_SEPARATOR - + tableName + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR + columnName); columnNames.append(columnName); @@ -374,9 +374,9 @@ private Map splitAndMergeQueryPatterns( columnNames = "%"; } else { tableName = - pattern.substring(0, pattern.lastIndexOf(".")) + pattern.substring(0, pattern.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - columnNames = pattern.substring(pattern.lastIndexOf(".") + 1); + columnNames = pattern.substring(pattern.lastIndexOf(IGINX_SEPARATOR) + 1); boolean columnEqualsStar = columnNames.equals("*"); boolean tableContainsStar = tableName.contains("*"); if (columnEqualsStar || tableContainsStar) { @@ -484,21 +484,26 @@ private Map> splitAndMergeHistoryQueryPatterns( columnNames = "%"; } else { String[] parts = pattern.split("\\" + IGINX_SEPARATOR); - if (parts.length > 3) { // 大于三级，不合法 - logger.error("wrong pattern: {}", pattern); - continue; - } - if (parts.length == 2 - && !parts[1].equals("*")) { // 只有两级且第二级不为 *，则此 pattern 不合法，无法拆分出三级 - continue; - } databaseName = parts[0]; - tableName = parts[1].equals("*") ? "%" : parts[1]; - if (parts.length == 2) { + if (parts.length == 1) { // 只有一级 + tableName = "%"; + columnNames = "%"; + } else if (parts.length == 2) { + tableName = parts[1].equals("*") ? "%" : parts[1]; columnNames = "%"; } else { - columnNames = parts[2].equals("*") ? "%" : parts[2]; + tableName = + pattern.substring( + pattern.indexOf(IGINX_SEPARATOR) + 1, + pattern.lastIndexOf(IGINX_SEPARATOR)) + .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR) + .replace("*", "%"); + columnNames = pattern.substring(pattern.lastIndexOf(IGINX_SEPARATOR) + 1); + if (columnNames.equals("*")) { + tableName += "%"; + columnNames = "%"; + } } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index bed1e5d74a..f7d5319ccd 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -92,9 +92,14 @@ public void writeHistoryDataToA() throws Exception { stmt.execute( String.format( - CREATE_TABLE_STATEMENT, "wf01\u2E82wt01", "status boolean, temperature float8")); + CREATE_TABLE_STATEMENT, + "wf01\u2E82wt01", + "status boolean, temperature float8")); stmt.execute( - String.format(INSERT_STATEMENT, "wf01\u2E82wt01", "(100, true, null), (200, false, 20.71)")); + String.format( + INSERT_STATEMENT, + "wf01\u2E82wt01", + "(100, true, null), (200, false, 20.71)")); stmt.close(); conn.close(); @@ -124,10 +129,15 @@ public void writeHistoryDataToB() throws Exception { stmt.execute( String.format( - CREATE_TABLE_STATEMENT, "wf03\u2E82wt01", "status boolean, temperature float8")); + CREATE_TABLE_STATEMENT, + "wf03\u2E82wt01", + "status boolean, temperature float8")); stmt.execute( - String.format(INSERT_STATEMENT, "wf03\u2E82wt01", "(77, true, null), (200, false, 77.71)")); + String.format( + INSERT_STATEMENT, + "wf03\u2E82wt01", + "(77, true, null), (200, false, 77.71)")); logger.info("write data to 127.0.0.1:5433 success!"); } From 10cf9d413c1e6720f85b212db71b7886482c5ff1 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 10:22:45 +0800 Subject: [PATCH 71/94] Add PostgreSQLFUNCTEST conf --- .github/workflows/DB-CE.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index ca9be28394..500e283967 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -9,6 +9,7 @@ env: FUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT IoTDB12FUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT InfluxDBFUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT + PostgreSQLFUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true From 52cf4d8d85bd53b7b5891984d0cddb2339b93796 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 11:02:42 +0800 Subject: [PATCH 72/94] Change timeout --- .github/workflows/DB-CE.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 500e283967..949c51ce0e 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -16,7 +16,7 @@ concurrency: jobs: DB-CE: - timeout-minutes: 40 + timeout-minutes: 50 strategy: fail-fast: false matrix: From dc878c623fd56ecc85e7f64436c94e0b39e8cfd3 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 19 May 2023 12:01:06 +0800 Subject: [PATCH 73/94] Change timeout --- .github/workflows/DB-CE.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 949c51ce0e..874ce16f6d 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -16,7 +16,7 @@ concurrency: jobs: DB-CE: - timeout-minutes: 50 + timeout-minutes: 60 strategy: fail-fast: false matrix: From 9ff05b43aa25517f50a7e66ae2a06900220d45bf Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Mon, 22 May 2023 17:25:26 +0800 Subject: [PATCH 74/94] Change map-reduce to for-loop --- .../iginx/postgresql/PostgreSQLStorage.java | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index bda1446ace..4e06b23c01 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -995,30 +995,31 @@ private void executeBatchInsert( String[] parts = columnNames.split(", "); boolean hasMultipleRows = parts.length != 1; + // INSERT INTO XXX (time, XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., (XXX, XXX, ...) ON CONFLICT (time) DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); StringBuilder statement = new StringBuilder(); statement.append("INSERT INTO "); statement.append(getCompleteName(tableName)); statement.append(" (time, "); - statement.append( - Arrays.stream(parts) - .map(this::getCompleteName) - .reduce((a, b) -> a + ", " + b) - .orElse("")); + StringBuilder columnNameStr = new StringBuilder(); + for (String part : parts) { + columnNameStr.append(getCompleteName(part)); + columnNameStr.append(", "); + } + statement.append(columnNameStr, 0, columnNameStr.length() - 2); + statement.append(") VALUES"); - statement.append( - values.stream() - .map(x -> " (" + x.substring(0, x.length() - 2) + ")") - .reduce((a, b) -> a + ", " + b) - .orElse("")); + for (String value : values) { + statement.append("("); + statement.append(value, 0, value.length() - 2); + statement.append("), "); + } + statement = new StringBuilder(statement.substring(0, statement.length() - 2)); + statement.append(" ON CONFLICT (time) DO UPDATE SET "); if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } - statement.append( - Arrays.stream(parts) - .map(this::getCompleteName) - .reduce((a, b) -> a + ", " + b) - .orElse("")); + statement.append(columnNameStr, 0, columnNameStr.length() - 2); if (hasMultipleRows) { statement.append(")"); // 只有一列不加括号 } @@ -1026,11 +1027,12 @@ private void executeBatchInsert( if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } - statement.append( - Arrays.stream(parts) - .map(x -> "excluded." + getCompleteName(x)) - .reduce((a, b) -> a + ", " + b) - .orElse("")); + for (String part : parts) { + statement.append("excluded."); + statement.append(getCompleteName(part)); + statement.append(", "); + } + statement = new StringBuilder(statement.substring(0, statement.length() - 2)); if (hasMultipleRows) { statement.append(")"); // 只有一列不加括号 } From 22b808466015be5d00ae1fcd929961aec1a13869 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Mon, 22 May 2023 17:44:07 +0800 Subject: [PATCH 75/94] Change timeout --- .github/workflows/DB-CE.yml | 2 +- .../cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 874ce16f6d..500e283967 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -16,7 +16,7 @@ concurrency: jobs: DB-CE: - timeout-minutes: 60 + timeout-minutes: 40 strategy: fail-fast: false matrix: diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 4e06b23c01..a9ecb10942 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -995,7 +995,8 @@ private void executeBatchInsert( String[] parts = columnNames.split(", "); boolean hasMultipleRows = parts.length != 1; - // INSERT INTO XXX (time, XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., (XXX, XXX, ...) ON CONFLICT (time) DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); + // INSERT INTO XXX (time, XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., (XXX, + // XXX, ...) ON CONFLICT (time) DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); StringBuilder statement = new StringBuilder(); statement.append("INSERT INTO "); statement.append(getCompleteName(tableName)); From 4be38cccee5bfae2792cb7f8317664db3e580c5d Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 23 May 2023 09:54:45 +0800 Subject: [PATCH 76/94] Change map-reduce to for-loop --- .../iginx/postgresql/PostgreSQLStorage.java | 73 +++++++++---------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index a9ecb10942..195243110b 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -426,16 +426,12 @@ private TaskExecuteResult executeProjectTask( splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); - String columnNames = - Arrays.stream(entry.getValue().split(", ")) - .map(this::getCompleteName) - .reduce((a, b) -> a + ", " + b) - .orElse("%"); + String fullColumnNames = getFullColumnNames(entry.getValue()); String statement = String.format( QUERY_STATEMENT, - columnNames, - getCompleteName(tableName), + fullColumnNames, + getFullName(tableName), FilterTransformer.toString(filter)); try { stmt = conn.createStatement(); @@ -598,29 +594,24 @@ private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filt } } - String[] parts = entry.getValue().split(", "); - String columnNames = - Arrays.stream(parts) - .map(this::getCompleteName) - .reduce((a, b) -> a + ", " + b) - .orElse("%"); String statement; + String fullColumnNames = getFullColumnNames(entry.getValue()); if (hasTimeColumn) { - if (!columnNames.contains("time")) { - columnNames += " , \"time\""; + if (!fullColumnNames.contains("time")) { + fullColumnNames += " , \"time\""; } statement = String.format( QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE, - columnNames, - getCompleteName(tableName)); + fullColumnNames, + getFullName(tableName)); } else { statement = String.format( CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, - columnNames, - columnNames, - getCompleteName(tableName)); + fullColumnNames, + fullColumnNames, + getFullName(tableName)); } try { stmt = conn.createStatement(); @@ -715,8 +706,8 @@ private void createOrAlterTables( String statement = String.format( CREATE_TABLE_STATEMENT, - getCompleteName(tableName), - getCompleteName(columnName), + getFullName(tableName), + getFullName(columnName), DataTypeTransformer.toPostgreSQL(dataType)); logger.info("[Create] execute create: {}", statement); stmt.execute(statement); @@ -728,8 +719,8 @@ private void createOrAlterTables( String statement = String.format( ADD_COLUMN_STATEMENT, - getCompleteName(tableName), - getCompleteName(columnName), + getFullName(tableName), + getFullName(columnName), DataTypeTransformer.toPostgreSQL(dataType)); logger.info("[Create] execute create: {}", statement); stmt.execute(statement); @@ -999,14 +990,10 @@ private void executeBatchInsert( // XXX, ...) ON CONFLICT (time) DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); StringBuilder statement = new StringBuilder(); statement.append("INSERT INTO "); - statement.append(getCompleteName(tableName)); + statement.append(getFullName(tableName)); statement.append(" (time, "); - StringBuilder columnNameStr = new StringBuilder(); - for (String part : parts) { - columnNameStr.append(getCompleteName(part)); - columnNameStr.append(", "); - } - statement.append(columnNameStr, 0, columnNameStr.length() - 2); + String fullColumnNames = getFullColumnNames(columnNames); + statement.append(fullColumnNames, 0, fullColumnNames.length() - 2); statement.append(") VALUES"); for (String value : values) { @@ -1020,7 +1007,7 @@ private void executeBatchInsert( if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } - statement.append(columnNameStr, 0, columnNameStr.length() - 2); + statement.append(fullColumnNames, 0, fullColumnNames.length() - 2); if (hasMultipleRows) { statement.append(")"); // 只有一列不加括号 } @@ -1030,7 +1017,7 @@ private void executeBatchInsert( } for (String part : parts) { statement.append("excluded."); - statement.append(getCompleteName(part)); + statement.append(getFullName(part)); statement.append(", "); } statement = new StringBuilder(statement.substring(0, statement.length() - 2)); @@ -1091,8 +1078,8 @@ private TaskExecuteResult executeDeleteTask( statement = String.format( DROP_COLUMN_STATEMENT, - getCompleteName(tableName), - getCompleteName(columnName)); + getFullName(tableName), + getFullName(columnName)); logger.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 删除列 } @@ -1111,8 +1098,8 @@ private TaskExecuteResult executeDeleteTask( statement = String.format( UPDATE_STATEMENT, - getCompleteName(tableName), - getCompleteName(columnName), + getFullName(tableName), + getFullName(columnName), timeRange.getBeginTime(), timeRange.getEndTime()); logger.info("[Delete] execute delete: {}", statement); @@ -1166,11 +1153,21 @@ private List> determineDeletedPaths( return deletedPaths; } - private String getCompleteName(String name) { + private String getFullName(String name) { return "\"" + name + "\""; // return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; } + private String getFullColumnNames(String columnNames) { + String[] parts = columnNames.split(", "); + StringBuilder fullColumnNames = new StringBuilder(); + for (String part : parts) { + fullColumnNames.append(getFullName(part)); + fullColumnNames.append(", "); + } + return fullColumnNames.substring(0, fullColumnNames.length() - 2); + } + @Override public void release() throws PhysicalException { try { From a02f63679fe59ad086a309373d251e97326d21b4 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 23 May 2023 10:04:47 +0800 Subject: [PATCH 77/94] Fix substring bug --- .../cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 195243110b..ce177362da 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -993,7 +993,7 @@ private void executeBatchInsert( statement.append(getFullName(tableName)); statement.append(" (time, "); String fullColumnNames = getFullColumnNames(columnNames); - statement.append(fullColumnNames, 0, fullColumnNames.length() - 2); + statement.append(fullColumnNames); statement.append(") VALUES"); for (String value : values) { @@ -1007,7 +1007,7 @@ private void executeBatchInsert( if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } - statement.append(fullColumnNames, 0, fullColumnNames.length() - 2); + statement.append(fullColumnNames); if (hasMultipleRows) { statement.append(")"); // 只有一列不加括号 } From 11e81c2b7b1cc6e9113505b61fc1b552af26b5e5 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Tue, 23 May 2023 17:59:38 +0800 Subject: [PATCH 78/94] Fix expansion bug --- .../edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java | 8 ++++++-- .../postgresql/query/entity/PostgreSQLQueryRowStream.java | 2 +- .../cn/edu/tsinghua/iginx/postgresql/tools/Constants.java | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index ce177362da..65893f07f2 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -318,7 +318,8 @@ && fromPostgreSQL(typeName) == DataType.LONG) { } if (hasTimeColumn) { // 本身就有 time 列 // 获取 key 的范围 - String statement = String.format(QUERY_TIME_STATEMENT, tableName); + String statement = + String.format(QUERY_TIME_STATEMENT, getFullName(tableName)); Statement timeStmt = conn.createStatement(); ResultSet timeSet = timeStmt.executeQuery(statement); while (timeSet.next()) { @@ -334,7 +335,10 @@ && fromPostgreSQL(typeName) == DataType.LONG) { // 获取 key 的范围 String statement = - String.format(CONCAT_QUERY_STATEMENT, columnNames, tableName); + String.format( + CONCAT_QUERY_STATEMENT, + getFullColumnNames(columnNames.toString()), + getFullName(tableName)); Statement concatStmt = conn.createStatement(); ResultSet concatSet = concatStmt.executeQuery(statement); while (concatSet.next()) { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index 05f83ae520..82ea3a0d4d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -77,9 +77,9 @@ public PostgreSQLQueryRowStream( for (int i = 0; i < resultSets.size(); i++) { ResultSetMetaData resultSetMetaData = resultSets.get(i).getMetaData(); - String tableName = resultSetMetaData.getTableName(1); int cnt = 0; for (int j = 1; j <= resultSetMetaData.getColumnCount(); j++) { + String tableName = resultSetMetaData.getTableName(j); String columnName = resultSetMetaData.getColumnName(j); String typeName = resultSetMetaData.getColumnTypeName(j); if (j == 1 && columnName.equals("time")) { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index a40430e87c..042dcc5437 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -32,7 +32,7 @@ public class Constants { "SELECT %s FROM %s ORDER BY time;"; public static final String CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT concat(%s) AS time, %s FROM %s;"; + "SELECT concat(%s) AS time, %s FROM %s ORDER BY time;"; public static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; From c9ee0e8421ab6223fce1a3e9ad6ec10482efcf58 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 8 Jun 2023 10:58:29 +0800 Subject: [PATCH 79/94] Refactor storage interface --- .../iginx/postgresql/PostgreSQLStorage.java | 604 +++++++++--------- 1 file changed, 291 insertions(+), 313 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 9dc6069fd8..8ee349fc06 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -40,7 +40,10 @@ import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; import cn.edu.tsinghua.iginx.engine.shared.operator.*; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.*; import cn.edu.tsinghua.iginx.postgresql.entity.TableType; @@ -148,55 +151,6 @@ private Connection getConnection(String databaseName) { } } - // @Override - // public TaskExecuteResult execute(StoragePhysicalTask task) { - // List operators = task.getOperators(); - // if (operators.size() != 1) { - // return new TaskExecuteResult( - // new NonExecutablePhysicalTaskException("unsupported physical task")); - // } - // FragmentMeta fragment = task.getTargetFragment(); - // Operator op = operators.get(0); - // String storageUnit = task.getStorageUnit(); - // boolean isDummyStorageUnit = task.isDummyStorageUnit(); - // Connection conn = getConnection(storageUnit); - // if (!isDummyStorageUnit && conn == null) { - // return new TaskExecuteResult( - // new NonExecutablePhysicalTaskException( - // String.format("cannot connect to storage unit %s", storageUnit))); - // } - // - // if (op.getType() == OperatorType.Project) { - // Project project = (Project) op; - // Filter filter; - // if (operators.size() == 2) { - // filter = ((Select) operators.get(1)).getFilter(); - // } else { - // filter = - // new AndFilter( - // Arrays.asList( - // new KeyFilter( - // Op.GE, - // fragment.getKeyInterval().getStartKey()), - // new KeyFilter( - // Op.L, - // fragment.getKeyInterval().getEndKey()))); - // } - // return isDummyStorageUnit - // ? executeHistoryProjectTask(project, filter) - // : executeProjectTask(conn, storageUnit, project, filter); - // } else if (op.getType() == OperatorType.Insert) { - // Insert insert = (Insert) op; - // return executeInsertTask(conn, storageUnit, insert); - // } else if (op.getType() == OperatorType.Delete) { - // Delete delete = (Delete) op; - // return executeDeleteTask(conn, storageUnit, delete); - // } - // return new TaskExecuteResult( - // new NonExecutablePhysicalTaskException("unsupported physical task in - // postgresql")); - // } - @Override public List getColumns() { List columns = new ArrayList<>(); @@ -276,12 +230,166 @@ public List getColumns() { @Override public TaskExecuteResult executeProject(Project project, DataArea dataArea) { - return null; + try { + String databaseName = dataArea.getStorageUnit(); + Connection conn = getConnection(databaseName); + if (conn == null) { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + String.format("cannot connect to database %s", databaseName))); + } + KeyInterval keyInterval = dataArea.getKeyInterval(); + Filter filter = + new AndFilter( + Arrays.asList( + new KeyFilter(Op.GE, keyInterval.getStartKey()), + new KeyFilter(Op.L, keyInterval.getEndKey()))); + + List databaseNameList = new ArrayList<>(); + List resultSets = new ArrayList<>(); + List isDummy = new ArrayList<>(); + ResultSet rs; + Statement stmt; + + Map tableNameToColumnNames = + splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); + for (Map.Entry entry : tableNameToColumnNames.entrySet()) { + String tableName = entry.getKey(); + String fullColumnNames = getFullColumnNames(entry.getValue()); + String statement = + String.format( + QUERY_STATEMENT, + fullColumnNames, + getFullName(tableName), + FilterTransformer.toString(filter)); + try { + stmt = conn.createStatement(); + rs = stmt.executeQuery(statement); + logger.info("[Query] execute query: {}", statement); + } catch (SQLException e) { + logger.error( + "meet error when executing query {}: {}", statement, e.getMessage()); + continue; + } + databaseNameList.add(databaseName); + resultSets.add(rs); + isDummy.add(TableType.nonDummy); + } + + RowStream rowStream = + new ClearEmptyRowStreamWrapper( + new PostgreSQLQueryRowStream( + databaseNameList, + resultSets, + isDummy, + filter, + project.getTagFilter())); + conn.close(); + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.error(e.getMessage()); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + "execute project task in postgresql failure", e)); + } } @Override public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) { - return null; + try { + KeyInterval keyInterval = dataArea.getKeyInterval(); + Filter filter = + new AndFilter( + Arrays.asList( + new KeyFilter(Op.GE, keyInterval.getStartKey()), + new KeyFilter(Op.L, keyInterval.getEndKey()))); + + List databaseNameList = new ArrayList<>(); + List resultSets = new ArrayList<>(); + List isDummy = new ArrayList<>(); + ResultSet rs; + Connection conn = null; + Statement stmt; + + Map> splitResults = + splitAndMergeHistoryQueryPatterns(project.getPatterns()); + for (Map.Entry> splitEntry : splitResults.entrySet()) { + String databaseName = splitEntry.getKey(); + conn = getConnection(databaseName); + if (conn == null) { + continue; + } + for (Map.Entry entry : splitEntry.getValue().entrySet()) { + boolean hasTimeColumn = false; + String tableName = entry.getKey(); + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet columnSet = + databaseMetaData.getColumns(databaseName, "public", tableName, "%"); + while (columnSet.next()) { + if (columnSet.getString("COLUMN_NAME").equalsIgnoreCase("time")) { + hasTimeColumn = true; + break; + } + } + + String statement; + String fullColumnNames = getFullColumnNames(entry.getValue()); + if (hasTimeColumn) { + if (!fullColumnNames.contains("time")) { + fullColumnNames += " , \"time\""; + } + statement = + String.format( + QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE, + fullColumnNames, + getFullName(tableName)); + } else { + statement = + String.format( + CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, + fullColumnNames, + fullColumnNames, + getFullName(tableName)); + } + try { + stmt = conn.createStatement(); + rs = stmt.executeQuery(statement); + logger.info("[Query] execute query: {}", statement); + } catch (SQLException e) { + logger.error( + "meet error when executing query {}: {}", + statement, + e.getMessage()); + continue; + } + databaseNameList.add(databaseName); + resultSets.add(rs); + if (hasTimeColumn) { + isDummy.add(TableType.dummyWithTimeColumn); + } else { + isDummy.add(TableType.dummy); + } + } + } + + RowStream rowStream = + new ClearEmptyRowStreamWrapper( + new PostgreSQLQueryRowStream( + databaseNameList, + resultSets, + isDummy, + filter, + project.getTagFilter())); + if (conn != null) { + conn.close(); + } + return new TaskExecuteResult(rowStream); + } catch (SQLException e) { + logger.error(e.getMessage()); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + "execute project task in postgresql failure", e)); + } } @Override @@ -303,12 +411,130 @@ public TaskExecuteResult executeProjectDummyWithSelect( @Override public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { - return null; + try { + String databaseName = dataArea.getStorageUnit(); + Connection conn = getConnection(databaseName); + if (conn == null) { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + String.format("cannot connect to database %s", databaseName))); + } + + Statement stmt = conn.createStatement(); + String statement; + List paths = delete.getPatterns(); + List> deletedPaths; // table name -> column name + String tableName; + String columnName; + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet tableSet = null; + ResultSet columnSet = null; + + if (delete.getKeyRanges() == null || delete.getKeyRanges().isEmpty()) { + if (paths.size() == 1 + && paths.get(0).equals("*") + && delete.getTagFilter() == null) { + conn.close(); + Connection postgresConn = + getConnection("postgres"); // 正在使用的数据库无法被删除，因此需要切换到名为 postgres 的默认数据库 + if (postgresConn != null) { + stmt = postgresConn.createStatement(); + statement = String.format(DROP_DATABASE_STATEMENT, databaseName); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 删除数据库 + stmt.close(); + postgresConn.close(); + return new TaskExecuteResult(null, null); + } else { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + "cannot connect to database postgres", new SQLException())); + } + } else { + deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); + for (Pair pair : deletedPaths) { + tableName = pair.k; + columnName = pair.v; + tableSet = + databaseMetaData.getTables( + databaseName, "public", tableName, new String[] {"TABLE"}); + if (tableSet.next()) { + statement = + String.format( + DROP_COLUMN_STATEMENT, + getFullName(tableName), + getFullName(columnName)); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 删除列 + } + } + } + } else { + deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); + for (Pair pair : deletedPaths) { + tableName = pair.k; + columnName = pair.v; + columnSet = + databaseMetaData.getColumns( + databaseName, "public", tableName, columnName); + if (columnSet.next()) { + for (KeyRange keyRange : delete.getKeyRanges()) { + statement = + String.format( + UPDATE_STATEMENT, + getFullName(tableName), + getFullName(columnName), + keyRange.getBeginKey(), + keyRange.getEndKey()); + logger.info("[Delete] execute delete: {}", statement); + stmt.execute(statement); // 将目标列的目标范围的值置为空 + } + } + } + } + if (tableSet != null) { + tableSet.close(); + } + if (columnSet != null) { + columnSet.close(); + } + stmt.close(); + conn.close(); + return new TaskExecuteResult(null, null); + } catch (SQLException e) { + logger.error(e.getMessage()); + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + "execute delete task in postgresql failure", e)); + } } @Override public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { - return null; + DataView dataView = insert.getData(); + String databaseName = dataArea.getStorageUnit(); + Connection conn = getConnection(databaseName); + if (conn == null) { + return new TaskExecuteResult( + new PhysicalTaskExecuteFailureException( + String.format("cannot connect to database %s", databaseName))); + } + Exception e = null; + switch (dataView.getRawDataType()) { + case Row: + case NonAlignedRow: + e = insertNonAlignedRowRecords(conn, databaseName, (RowDataView) dataView); + break; + case Column: + case NonAlignedColumn: + e = insertNonAlignedColumnRecords(conn, databaseName, (ColumnDataView) dataView); + break; + } + if (e != null) { + return new TaskExecuteResult( + null, new PhysicalException("execute insert task in postgresql failure", e)); + } + return new TaskExecuteResult(null, null); } @Override @@ -452,58 +678,6 @@ private Map splitAndMergeQueryPatterns( return tableNameToColumnNames; } - private TaskExecuteResult executeProjectTask( - Connection conn, String databaseName, Project project, Filter filter) { - try { - List databaseNameList = new ArrayList<>(); - List resultSets = new ArrayList<>(); - List isDummy = new ArrayList<>(); - ResultSet rs; - Statement stmt; - - Map tableNameToColumnNames = - splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); - for (Map.Entry entry : tableNameToColumnNames.entrySet()) { - String tableName = entry.getKey(); - String fullColumnNames = getFullColumnNames(entry.getValue()); - String statement = - String.format( - QUERY_STATEMENT, - fullColumnNames, - getFullName(tableName), - FilterTransformer.toString(filter)); - try { - stmt = conn.createStatement(); - rs = stmt.executeQuery(statement); - logger.info("[Query] execute query: {}", statement); - } catch (SQLException e) { - logger.error( - "meet error when executing query {}: {}", statement, e.getMessage()); - continue; - } - databaseNameList.add(databaseName); - resultSets.add(rs); - isDummy.add(TableType.nonDummy); - } - - RowStream rowStream = - new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream( - databaseNameList, - resultSets, - isDummy, - filter, - project.getTagFilter())); - conn.close(); - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - logger.error(e.getMessage()); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException( - "execute project task in postgresql failure", e)); - } - } - private Map> splitAndMergeHistoryQueryPatterns( List patterns) throws SQLException { // > @@ -603,117 +777,6 @@ private Map> splitAndMergeHistoryQueryPatterns( return splitResults; } - private TaskExecuteResult executeHistoryProjectTask(Project project, Filter filter) { - try { - List databaseNameList = new ArrayList<>(); - List resultSets = new ArrayList<>(); - List isDummy = new ArrayList<>(); - ResultSet rs; - Connection conn = null; - Statement stmt; - - Map> splitResults = - splitAndMergeHistoryQueryPatterns(project.getPatterns()); - for (Map.Entry> splitEntry : splitResults.entrySet()) { - String databaseName = splitEntry.getKey(); - conn = getConnection(databaseName); - if (conn == null) { - continue; - } - for (Map.Entry entry : splitEntry.getValue().entrySet()) { - boolean hasTimeColumn = false; - String tableName = entry.getKey(); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet columnSet = - databaseMetaData.getColumns(databaseName, "public", tableName, "%"); - while (columnSet.next()) { - if (columnSet.getString("COLUMN_NAME").equalsIgnoreCase("time")) { - hasTimeColumn = true; - break; - } - } - - String statement; - String fullColumnNames = getFullColumnNames(entry.getValue()); - if (hasTimeColumn) { - if (!fullColumnNames.contains("time")) { - fullColumnNames += " , \"time\""; - } - statement = - String.format( - QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE, - fullColumnNames, - getFullName(tableName)); - } else { - statement = - String.format( - CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, - fullColumnNames, - fullColumnNames, - getFullName(tableName)); - } - try { - stmt = conn.createStatement(); - rs = stmt.executeQuery(statement); - logger.info("[Query] execute query: {}", statement); - } catch (SQLException e) { - logger.error( - "meet error when executing query {}: {}", - statement, - e.getMessage()); - continue; - } - databaseNameList.add(databaseName); - resultSets.add(rs); - if (hasTimeColumn) { - isDummy.add(TableType.dummyWithTimeColumn); - } else { - isDummy.add(TableType.dummy); - } - } - } - - RowStream rowStream = - new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream( - databaseNameList, - resultSets, - isDummy, - filter, - project.getTagFilter())); - if (conn != null) { - conn.close(); - } - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - logger.error(e.getMessage()); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException( - "execute project task in postgresql failure", e)); - } - } - - private TaskExecuteResult executeInsertTask( - Connection conn, String storageUnit, Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertNonAlignedRowRecords(conn, storageUnit, (RowDataView) dataView); - break; - case Column: - case NonAlignedColumn: - e = insertNonAlignedColumnRecords(conn, storageUnit, (ColumnDataView) dataView); - break; - } - if (e != null) { - return new TaskExecuteResult( - null, new PhysicalException("execute insert task in postgresql failure", e)); - } - return new TaskExecuteResult(null, null); - } - private void createOrAlterTables( Connection conn, String storageUnit, @@ -779,14 +842,18 @@ private void createOrAlterTables( } private Exception insertNonAlignedRowRecords( - Connection conn, String storageUnit, RowDataView data) { + Connection conn, String databaseName, RowDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); // 创建表 createOrAlterTables( - conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + conn, + databaseName, + data.getPaths(), + data.getTagsList(), + data.getDataTypeList()); // 插入数据 Map>> tableToColumnEntries = @@ -894,14 +961,18 @@ private Exception insertNonAlignedRowRecords( } private Exception insertNonAlignedColumnRecords( - Connection conn, String storageUnit, ColumnDataView data) { + Connection conn, String databaseName, ColumnDataView data) { int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); // 创建表 createOrAlterTables( - conn, storageUnit, data.getPaths(), data.getTagsList(), data.getDataTypeList()); + conn, + databaseName, + data.getPaths(), + data.getTagsList(), + data.getDataTypeList()); // 插入数据 Map>> tableToColumnEntries = @@ -1071,99 +1142,6 @@ private void executeBatchInsert( stmt.executeBatch(); } - private TaskExecuteResult executeDeleteTask( - Connection conn, String storageUnit, Delete delete) { - try { - Statement stmt = conn.createStatement(); - String statement; - List paths = delete.getPatterns(); - List> deletedPaths; // table name -> column name - String tableName; - String columnName; - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - ResultSet columnSet = null; - - if (delete.getKeyRanges() == null || delete.getKeyRanges().size() == 0) { - if (paths.size() == 1 - && paths.get(0).equals("*") - && delete.getTagFilter() == null) { - conn.close(); - Connection postgresConn = - getConnection("postgres"); // 正在使用的数据库无法被删除，因此需要切换到名为 postgres 的默认数据库 - if (postgresConn != null) { - stmt = postgresConn.createStatement(); - statement = String.format(DROP_DATABASE_STATEMENT, storageUnit); - logger.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 删除数据库 - stmt.close(); - postgresConn.close(); - return new TaskExecuteResult(null, null); - } else { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException( - "cannot connect to database: postgres", - new SQLException())); - } - } else { - deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); - for (Pair pair : deletedPaths) { - tableName = pair.k; - columnName = pair.v; - tableSet = - databaseMetaData.getTables( - storageUnit, "public", tableName, new String[] {"TABLE"}); - if (tableSet.next()) { - statement = - String.format( - DROP_COLUMN_STATEMENT, - getFullName(tableName), - getFullName(columnName)); - logger.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 删除列 - } - } - } - } else { - deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); - for (Pair pair : deletedPaths) { - tableName = pair.k; - columnName = pair.v; - columnSet = - databaseMetaData.getColumns( - storageUnit, "public", tableName, columnName); - if (columnSet.next()) { - for (KeyRange keyRange : delete.getKeyRanges()) { - statement = - String.format( - UPDATE_STATEMENT, - getFullName(tableName), - getFullName(columnName), - keyRange.getBeginKey(), - keyRange.getEndKey()); - logger.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 将目标列的目标范围的值置为空 - } - } - } - } - if (tableSet != null) { - tableSet.close(); - } - if (columnSet != null) { - columnSet.close(); - } - stmt.close(); - conn.close(); - return new TaskExecuteResult(null, null); - } catch (SQLException e) { - logger.error(e.getMessage()); - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException( - "execute delete task in postgresql failure", e)); - } - } - private List> determineDeletedPaths( List paths, TagFilter tagFilter) { List columns = getColumns(); From f6beaea7335f7ed44bc0503fb007acf8f08d0dd6 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 8 Jun 2023 17:18:10 +0800 Subject: [PATCH 80/94] Refactor capacity expansion for pg --- .../iginx/postgresql/PostgreSQLStorage.java | 52 +--- .../entity/PostgreSQLQueryRowStream.java | 99 -------- .../iginx/postgresql/entity/TableType.java | 7 - .../entity/PostgreSQLQueryRowStream.java | 43 ++-- .../iginx/postgresql/tools/Constants.java | 4 +- .../iginx/postgresql/tools/TagKVUtils.java | 2 +- .../expansion/BaseHistoryDataGenerator.java | 6 +- .../expansion/CapacityExpansionIT.java | 8 + .../InfluxDBHistoryDataGenerator.java | 3 + .../iotdb/IoTDB12HistoryDataGenerator.java | 3 + .../PostgreSQLCapacityExpansionIT.java | 40 +++ ...tgreSQLHistoryDataCapacityExpansionIT.java | 26 -- .../PostgreSQLHistoryDataGeneratorTest.java | 229 +++++++++++------- .../iginx/integration/tool/DBType.java | 3 +- 14 files changed, 228 insertions(+), 297 deletions(-) delete mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java delete mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 8ee349fc06..0831b05404 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -46,7 +46,6 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.*; -import cn.edu.tsinghua.iginx.postgresql.entity.TableType; import cn.edu.tsinghua.iginx.postgresql.query.entity.PostgreSQLQueryRowStream; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; @@ -247,7 +246,6 @@ public TaskExecuteResult executeProject(Project project, DataArea dataArea) { List databaseNameList = new ArrayList<>(); List resultSets = new ArrayList<>(); - List isDummy = new ArrayList<>(); ResultSet rs; Statement stmt; @@ -273,7 +271,6 @@ public TaskExecuteResult executeProject(Project project, DataArea dataArea) { } databaseNameList.add(databaseName); resultSets.add(rs); - isDummy.add(TableType.nonDummy); } RowStream rowStream = @@ -281,7 +278,7 @@ public TaskExecuteResult executeProject(Project project, DataArea dataArea) { new PostgreSQLQueryRowStream( databaseNameList, resultSets, - isDummy, + false, filter, project.getTagFilter())); conn.close(); @@ -306,7 +303,6 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) List databaseNameList = new ArrayList<>(); List resultSets = new ArrayList<>(); - List isDummy = new ArrayList<>(); ResultSet rs; Connection conn = null; Statement stmt; @@ -320,37 +316,15 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) continue; } for (Map.Entry entry : splitEntry.getValue().entrySet()) { - boolean hasTimeColumn = false; String tableName = entry.getKey(); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet columnSet = - databaseMetaData.getColumns(databaseName, "public", tableName, "%"); - while (columnSet.next()) { - if (columnSet.getString("COLUMN_NAME").equalsIgnoreCase("time")) { - hasTimeColumn = true; - break; - } - } - - String statement; String fullColumnNames = getFullColumnNames(entry.getValue()); - if (hasTimeColumn) { - if (!fullColumnNames.contains("time")) { - fullColumnNames += " , \"time\""; - } - statement = - String.format( - QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE, - fullColumnNames, - getFullName(tableName)); - } else { - statement = - String.format( - CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, - fullColumnNames, - fullColumnNames, - getFullName(tableName)); - } + String statement = + String.format( + CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE, + fullColumnNames, + fullColumnNames, + getFullName(tableName)); + try { stmt = conn.createStatement(); rs = stmt.executeQuery(statement); @@ -364,11 +338,6 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) } databaseNameList.add(databaseName); resultSets.add(rs); - if (hasTimeColumn) { - isDummy.add(TableType.dummyWithTimeColumn); - } else { - isDummy.add(TableType.dummy); - } } } @@ -377,7 +346,7 @@ public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) new PostgreSQLQueryRowStream( databaseNameList, resultSets, - isDummy, + true, filter, project.getTagFilter())); if (conn != null) { @@ -540,7 +509,8 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { @Override public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { - long minTime = Long.MAX_VALUE, maxTime = 0; + long minTime = Long.MAX_VALUE; + long maxTime = 0; List paths = new ArrayList<>(); try { Statement stmt = connection.createStatement(); diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java deleted file mode 100644 index c355952e05..0000000000 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/PostgreSQLQueryRowStream.java +++ /dev/null @@ -1,99 +0,0 @@ -package cn.edu.tsinghua.iginx.postgresql.entity; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -public class PostgreSQLQueryRowStream implements RowStream { - - private final List resultSets; - - private final long[] currTimestamps; - - private final Object[] currValues; - - private final Header header; - - public PostgreSQLQueryRowStream(List resultSets, List fields) { - this.resultSets = resultSets; - this.header = new Header(Field.KEY, fields); - this.currTimestamps = new long[resultSets.size()]; - this.currValues = new Object[resultSets.size()]; - // 默认填充一下timestamp列表 - try { - for (int i = 0; i < this.currTimestamps.length; i++) { - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - this.currValues[i] = resultSet.getObject(2); - } - } - } catch (SQLException e) { - e.printStackTrace(); - // pass - } - } - - @Override - public Header getHeader() { - return this.header; - } - - @Override - public void close() { - try { - for (ResultSet resultSet : resultSets) { - resultSet.close(); - } - } catch (SQLException e) { - // pass - } - } - - @Override - public boolean hasNext() throws PhysicalException { - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - return true; - } - } - return false; - } - - @Override - public Row next() throws PhysicalException { - try { - long timestamp = Long.MAX_VALUE; - Object[] values = new Object[this.resultSets.size()]; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); - } - } - - for (int i = 0; i < this.currTimestamps.length; i++) { - if (this.currTimestamps[i] == timestamp) { - values[i] = this.currValues[i]; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - this.currValues[i] = resultSet.getObject(2); - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; - } - } - } - return new Row(header, timestamp, values); - } catch (SQLException e) { - throw new RowFetchException(e); - } - } -} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java deleted file mode 100644 index 3617f4feea..0000000000 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/entity/TableType.java +++ /dev/null @@ -1,7 +0,0 @@ -package cn.edu.tsinghua.iginx.postgresql.entity; - -public enum TableType { - dummy, - dummyWithTimeColumn, - nonDummy -} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index 82ea3a0d4d..e2c82891af 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -15,7 +15,6 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.postgresql.entity.TableType; import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; @@ -34,7 +33,7 @@ public class PostgreSQLQueryRowStream implements RowStream { private final Header header; - private final List isDummy; + private final boolean isDummy; private final Filter filter; @@ -55,7 +54,7 @@ public class PostgreSQLQueryRowStream implements RowStream { public PostgreSQLQueryRowStream( List databaseNameList, List resultSets, - List isDummy, + boolean isDummy, Filter filter, TagFilter tagFilter) throws SQLException { @@ -88,29 +87,16 @@ public PostgreSQLQueryRowStream( } Pair> namesAndTags = splitFullName(columnName); - Field field; - if (isDummy.get(i) == TableType.nonDummy) { - field = - new Field( - tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR - + namesAndTags.k.replace( - POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName), - namesAndTags.v); - } else { - field = - new Field( - databaseNameList.get(i) - + IGINX_SEPARATOR - + tableName.replace( - POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR - + namesAndTags.k.replace( - POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName), - namesAndTags.v); - } + Field field = + new Field( + databaseNameList.get(i) + + IGINX_SEPARATOR + + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + + namesAndTags.k.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v); if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { continue; @@ -206,7 +192,7 @@ private void cacheOneRow() throws SQLException, PhysicalException { long tempTimestamp; Object tempValue; - if (isDummy.get(i) == TableType.dummy) { + if (isDummy) { tempTimestamp = toHash(resultSet.getString("time")); } else { tempTimestamp = resultSet.getLong("time"); @@ -257,8 +243,7 @@ private void cacheOneRow() throws SQLException, PhysicalException { startIndex = endIndex; } cachedRow = new Row(header, timestamp, values); - if (isDummy.stream().allMatch(x -> x == TableType.dummy) - && !validate(filter, cachedRow)) { + if (isDummy && !validate(filter, cachedRow)) { cacheOneRow(); } } else { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 042dcc5437..facecb485e 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -2,9 +2,9 @@ public class Constants { - public static final String IGINX_SEPARATOR = "."; + public static final char IGINX_SEPARATOR = '.'; - public static final String POSTGRESQL_SEPARATOR = "\u2E82"; + public static final char POSTGRESQL_SEPARATOR = '\u2E82'; public static final int BATCH_SIZE = 10000; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java index 2440975717..dc56af1e1f 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -8,7 +8,7 @@ public class TagKVUtils { public static Pair> splitFullName(String fullName) { - if (!fullName.contains(POSTGRESQL_SEPARATOR)) { + if (fullName.indexOf(POSTGRESQL_SEPARATOR) == -1) { return new Pair<>(fullName, null); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java index a20ddd5fba..0e9e1129eb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java @@ -86,10 +86,10 @@ public void oriNoDataExpHasData() { @Test public void oriNoDataExpNoData() {} - public void writeHistoryDataToOri() {} + public abstract void writeHistoryDataToOri(); - public void writeHistoryDataToExp() {} + public abstract void writeHistoryDataToExp(); @Test - public void clearHistoryData() {} + public abstract void clearHistoryData(); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java index 65f4aa0c83..4247a8d9d4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/CapacityExpansionIT.java @@ -378,6 +378,14 @@ private void addStorageEngine(boolean hasData) { + hasData + ", is_read_only:true, token:testToken, organization:testOrg\");"); break; + case postgresql: + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5431, \"" + + dbType.name() + + "\", \"username:postgres, password:postgres, has_data:" + + hasData + + ", is_read_only:true\");"); + break; default: logger.error("unsupported storage engine: {}", dbType.name()); fail(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java index da03112338..937583db12 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBHistoryDataGenerator.java @@ -119,15 +119,18 @@ private void writeHistoryData( logger.info("write data to " + URL + " success!"); } + @Override public void writeHistoryDataToOri() { writeHistoryData(pathListOri, dataTypeListOri, valuesListOri, ORI_URL); } + @Override public void writeHistoryDataToExp() { writeHistoryData(pathListExp, dataTypeListExp, valuesListExp, EXP_URL); } @Test + @Override public void clearHistoryData() { InfluxDBClient client = InfluxDBClientFactory.create(ORI_URL, TOKEN.toCharArray(), ORGANIZATION); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java index e268664e2a..398ec7f495 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12HistoryDataGenerator.java @@ -66,15 +66,18 @@ private void writeHistoryData( } } + @Override public void writeHistoryDataToOri() { writeHistoryData(pathListOri, dataTypeListOri, valuesListOri, portOri); } + @Override public void writeHistoryDataToExp() { writeHistoryData(pathListExp, dataTypeListExp, valuesListExp, portExp); } @Test + @Override public void clearHistoryData() { try { Session sessionOri = new Session("127.0.0.1", portOri, "root", "root"); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java new file mode 100644 index 0000000000..6eb5a93e03 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java @@ -0,0 +1,40 @@ +package cn.edu.tsinghua.iginx.integration.expansion.postgresql; + +import static cn.edu.tsinghua.iginx.integration.tool.DBType.postgresql; + +import cn.edu.tsinghua.iginx.exceptions.ExecutionException; +import cn.edu.tsinghua.iginx.exceptions.SessionException; +import cn.edu.tsinghua.iginx.integration.expansion.CapacityExpansionIT; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreSQLCapacityExpansionIT extends CapacityExpansionIT { + + private static final Logger logger = + LoggerFactory.getLogger(PostgreSQLCapacityExpansionIT.class); + + public PostgreSQLCapacityExpansionIT() { + super(postgresql); + } + + @Override + public void addStorageEngineWithPrefix(String dataPrefix, String schemaPrefix) { + try { + session.executeSql( + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + + postgresql.name() + + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:" + + dataPrefix + + ", schema_prefix:" + + schemaPrefix + + ", is_read_only:true\");"); + } catch (ExecutionException | SessionException e) { + logger.error("add storage engine failure: {}", e.getMessage()); + } + } + + @Override + public int getPort() { + return 5433; + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java deleted file mode 100644 index 5285567b60..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataCapacityExpansionIT.java +++ /dev/null @@ -1,26 +0,0 @@ -package cn.edu.tsinghua.iginx.integration.expansion.postgresql; - -import cn.edu.tsinghua.iginx.integration.expansion.CapacityExpansionIT; - -public class PostgreSQLHistoryDataCapacityExpansionIT extends CapacityExpansionIT { - public PostgreSQLHistoryDataCapacityExpansionIT() { - super("postgresql"); - } - - @Override - protected void addStorageWithPrefix(String dataPrefix, String schemaPrefix) throws Exception { - session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" - + ENGINE_TYPE - + "\", \"username:postgres, password:postgres, has_data:true, data_prefix:" - + dataPrefix - + ", schema_prefix:" - + schemaPrefix - + ", is_read_only:true\");"); - } - - @Override - protected int getPort() throws Exception { - return 5433; - } -} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java index f7d5319ccd..f27cf0a617 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java @@ -1,46 +1,51 @@ package cn.edu.tsinghua.iginx.integration.expansion.postgresql; import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; +import cn.edu.tsinghua.iginx.thrift.DataType; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.*; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -// TODO 通过 BaseHistoryDataGenerator 里的 seriesA 和 seriesB 插入数据 public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator { private static final Logger logger = LoggerFactory.getLogger(PostgreSQLHistoryDataGeneratorTest.class); + public static final char IGINX_SEPARATOR = '.'; + + public static final char POSTGRESQL_SEPARATOR = '\u2E82'; + private static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; - private static final String CREATE_TABLE_STATEMENT = - "CREATE TABLE %s (time BIGINT NOT NULL, %s, PRIMARY KEY(time));"; + private static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (%s);"; private static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; private static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; - private static final int PORT_A = 5432; - - private static final int PORT_B = 5433; - private static final String USERNAME = "postgres"; private static final String PASSWORD = "postgres"; - private static final String DATABASE_NAME = "ln"; + private static final Set databaseNameList = new HashSet<>(); - private Connection connect(int port, boolean useSystemDatabase) { + public PostgreSQLHistoryDataGeneratorTest() { + this.portOri = 6667; + this.portExp = 6668; + } + + private Connection connect(int port, boolean useSystemDatabase, String databaseName) { try { String url; if (useSystemDatabase) { url = String.format("jdbc:postgresql://127.0.0.1:%d/", port); } else { - url = String.format("jdbc:postgresql://127.0.0.1:%d/%s", port, DATABASE_NAME); + url = String.format("jdbc:postgresql://127.0.0.1:%d/%s", port, databaseName); } Class.forName("org.postgresql.Driver"); return DriverManager.getConnection(url, USERNAME, PASSWORD); @@ -49,96 +54,144 @@ private Connection connect(int port, boolean useSystemDatabase) { } } - @Test - public void clearData() { - try { - Connection connA = connect(PORT_A, true); - Statement stmtA = connA.createStatement(); - stmtA.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); - stmtA.close(); - connA.close(); - - Connection connB = connect(PORT_B, true); - Statement stmtB = connB.createStatement(); - stmtB.execute(String.format(DROP_DATABASE_STATEMENT, DATABASE_NAME)); - stmtB.close(); - connB.close(); - } catch (Exception e) { - logger.error(e.getMessage()); - } - logger.info("clear data success!"); + @Override + public void writeHistoryDataToOri() { + writeHistoryData(pathListOri, dataTypeListOri, valuesListOri, portOri); } - @Test - public void writeHistoryDataToA() throws Exception { - Connection conn = connect(PORT_A, true); - if (conn == null) { - logger.error("cannot connect to 5432!"); - return; - } + @Override + public void writeHistoryDataToExp() { + writeHistoryData(pathListExp, dataTypeListExp, valuesListExp, portExp); + } - Statement stmt = conn.createStatement(); + private void writeHistoryData( + List pathList, + List dataTypeList, + List> valuesList, + int port) { try { - stmt.execute(String.format(CREATE_DATABASE_STATEMENT, DATABASE_NAME)); - } catch (SQLException e) { - logger.info("database ln exists!"); - } + Connection conn = connect(port, true, null); + if (conn == null) { + logger.error("cannot connect to 127.0.0.1:{}!", port); + return; + } - stmt.close(); - conn.close(); - - conn = connect(PORT_A, false); - stmt = conn.createStatement(); - - stmt.execute( - String.format( - CREATE_TABLE_STATEMENT, - "wf01\u2E82wt01", - "status boolean, temperature float8")); - stmt.execute( - String.format( - INSERT_STATEMENT, - "wf01\u2E82wt01", - "(100, true, null), (200, false, 20.71)")); - - stmt.close(); - conn.close(); - logger.info("write data to 127.0.0.1:5432 success!"); + Map>> databaseToTablesToColumnIndexes = + new HashMap<>(); + for (int i = 0; i < pathList.size(); i++) { + String path = pathList.get(i); + String databaseName = path.substring(0, path.indexOf(IGINX_SEPARATOR)); + String tableName = + path.substring( + path.indexOf(IGINX_SEPARATOR) + 1, + path.lastIndexOf(IGINX_SEPARATOR)) + .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); + + Map> tablesToColumnIndexes = + databaseToTablesToColumnIndexes.computeIfAbsent( + databaseName, x -> new HashMap<>()); + List columnIndexes = + tablesToColumnIndexes.computeIfAbsent(tableName, x -> new ArrayList<>()); + columnIndexes.add(i); + tablesToColumnIndexes.put(tableName, columnIndexes); + } + + for (Map.Entry>> entry : + databaseToTablesToColumnIndexes.entrySet()) { + String databaseName = entry.getKey(); + Statement stmt = conn.createStatement(); + try { + stmt.execute(String.format(CREATE_DATABASE_STATEMENT, databaseName)); + databaseNameList.add(databaseName); + } catch (SQLException e) { + logger.info("database {} exists!", databaseName); + } + stmt.close(); + conn.close(); + + conn = connect(port, false, databaseName); + stmt = conn.createStatement(); + for (Map.Entry> item : entry.getValue().entrySet()) { + String tableName = item.getKey(); + StringBuilder createTableStr = new StringBuilder(); + for (Integer index : item.getValue()) { + String columnName = pathList.get(index); + DataType dataType = dataTypeList.get(index); + createTableStr.append(columnName); + createTableStr.append(" "); + createTableStr.append(toPostgreSQL(dataType)); + createTableStr.append(", "); + } + stmt.execute( + String.format( + CREATE_TABLE_STATEMENT, + tableName, + createTableStr.substring(0, createTableStr.length() - 2))); + + StringBuilder insertStr = new StringBuilder(); + for (List values : valuesList) { + insertStr.append("("); + for (Integer index : item.getValue()) { + insertStr.append(values.get(index)); + insertStr.append(", "); + } + insertStr = + new StringBuilder(insertStr.substring(0, insertStr.length() - 2)); + insertStr.append("), "); + } + stmt.execute( + String.format( + INSERT_STATEMENT, + tableName, + insertStr.substring(0, insertStr.length() - 2))); + } + stmt.close(); + conn.close(); + } + + logger.info("write data to 127.0.0.1:{} success!", port); + } catch (RuntimeException | SQLException e) { + logger.error("write data to 127.0.0.1:{} failure: {}", port, e.getMessage()); + } } @Test - public void writeHistoryDataToB() throws Exception { - Connection conn = connect(PORT_B, true); - if (conn == null) { - logger.error("cannot connect to 5433!"); - return; - } + @Override + public void clearHistoryData() { + clearHistoryData(portOri); + clearHistoryData(portExp); + } - Statement stmt = conn.createStatement(); + private void clearHistoryData(int port) { try { - stmt.execute(String.format(CREATE_DATABASE_STATEMENT, DATABASE_NAME)); + Connection conn = connect(port, true, null); + Statement stmt = conn.createStatement(); + for (String databaseName : databaseNameList) { + stmt.execute(String.format(DROP_DATABASE_STATEMENT, databaseName)); + } + stmt.close(); + conn.close(); + logger.info("clear data on 127.0.0.1:{} success!", port); } catch (SQLException e) { - logger.info("database ln exists!"); + logger.error("clear data on 127.0.0.1:{} failure: {}", port, e.getMessage()); } + } - stmt.close(); - conn.close(); - - conn = connect(PORT_B, false); - stmt = conn.createStatement(); - - stmt.execute( - String.format( - CREATE_TABLE_STATEMENT, - "wf03\u2E82wt01", - "status boolean, temperature float8")); - - stmt.execute( - String.format( - INSERT_STATEMENT, - "wf03\u2E82wt01", - "(77, true, null), (200, false, 77.71)")); - - logger.info("write data to 127.0.0.1:5433 success!"); + private static String toPostgreSQL(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return "BOOLEAN"; + case INTEGER: + return "INTEGER"; + case LONG: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE PRECISION"; + case BINARY: + default: + return "TEXT"; + } } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBType.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBType.java index e213cfc6ca..cae0e00af3 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBType.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBType.java @@ -5,5 +5,6 @@ public enum DBType { iotdb12, influxdb, mongodb, - redis + redis, + postgresql } From 1a9b3c4e83bfb8d4735620f14d19641422370a2e Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 8 Jun 2023 21:29:10 +0800 Subject: [PATCH 81/94] Fix script bug --- .github/actions/dbRunner/action.yml | 2 +- .github/workflows/standalone-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml index 31c49a1b99..fb1d82f884 100644 --- a/.github/actions/dbRunner/action.yml +++ b/.github/actions/dbRunner/action.yml @@ -99,7 +99,7 @@ runs: exit 1 fi - - if: inputs.version=='PostgreSQL' + - if: inputs.DB-name=='PostgreSQL' name: Run DB shell: bash run: | diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index 77ed50e221..c0ec0aa8b7 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -46,7 +46,7 @@ jobs: - name: Change IGinX config uses: ./.github/actions/confWriter with: - DB-name: ${{matrix.DB-name}} + DB-name: ${{ matrix.DB-name }} if-CapExp: false - name: Start IGinX From 2f0be33981a7fbd0a2f72a7a353a508e036ab1a9 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 8 Jun 2023 21:54:09 +0800 Subject: [PATCH 82/94] Rename PostgreSQLHistoryDataGeneratorTest --- ...neratorTest.java => PostgreSQLHistoryDataGenerator.java} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/{PostgreSQLHistoryDataGeneratorTest.java => PostgreSQLHistoryDataGenerator.java} (98%) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java similarity index 98% rename from test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java rename to test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 69bc1704e2..7283b4db38 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGeneratorTest.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -11,10 +11,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator { +public class PostgreSQLHistoryDataGenerator extends BaseHistoryDataGenerator { private static final Logger logger = - LoggerFactory.getLogger(PostgreSQLHistoryDataGeneratorTest.class); + LoggerFactory.getLogger(PostgreSQLHistoryDataGenerator.class); public static final char IGINX_SEPARATOR = '.'; @@ -34,7 +34,7 @@ public class PostgreSQLHistoryDataGeneratorTest extends BaseHistoryDataGenerator private static final Set databaseNameList = new HashSet<>(); - public PostgreSQLHistoryDataGeneratorTest() { + public PostgreSQLHistoryDataGenerator() { this.portOri = 6667; this.portExp = 6668; } From ca6cec2222ef80c57a395c42b5612f9445d24c38 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 8 Jun 2023 22:03:27 +0800 Subject: [PATCH 83/94] Modify port number --- .../expansion/postgresql/PostgreSQLHistoryDataGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 7283b4db38..8d2414872d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -35,8 +35,8 @@ public class PostgreSQLHistoryDataGenerator extends BaseHistoryDataGenerator { private static final Set databaseNameList = new HashSet<>(); public PostgreSQLHistoryDataGenerator() { - this.portOri = 6667; - this.portExp = 6668; + this.portOri = 5432; + this.portExp = 5433; } private Connection connect(int port, boolean useSystemDatabase, String databaseName) { From cbd45cc32f51038cbe5b079206745f9456db3f5a Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Thu, 8 Jun 2023 22:20:07 +0800 Subject: [PATCH 84/94] Fix insert bug --- .../expansion/postgresql/PostgreSQLHistoryDataGenerator.java | 3 ++- .../integration/expansion/redis/RedisHistoryDataGenerator.java | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 8d2414872d..3298637c5f 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -115,7 +115,8 @@ private void writeHistoryData( String tableName = item.getKey(); StringBuilder createTableStr = new StringBuilder(); for (Integer index : item.getValue()) { - String columnName = pathList.get(index); + String path = pathList.get(index); + String columnName = path.substring(path.lastIndexOf(IGINX_SEPARATOR) + 1); DataType dataType = dataTypeList.get(index); createTableStr.append(columnName); createTableStr.append(" "); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java index fbcee262b9..2c062ac3b0 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java @@ -2,6 +2,8 @@ import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; import java.util.List; + +import org.junit.Test; import redis.clients.jedis.Jedis; public class RedisHistoryDataGenerator extends BaseHistoryDataGenerator { @@ -34,6 +36,7 @@ private void writeHistoryData(List pathList, List> valuesLi jedis.close(); } + @Test @Override public void clearHistoryData() { Jedis jedisA = new Jedis(LOCAL_IP, PORT_A); From 844b5f70c1fc93da3edd2f286c97e97bcdb79823 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 09:35:40 +0800 Subject: [PATCH 85/94] Change key name --- .github/postgresql_macos.sh | 2 +- .../antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 | 2 +- .../engine/shared/data/write/DataView.java | 2 +- .../metadata/cache/DefaultMetaCache.java | 4 +- .../iginx/monitor/RequestsMonitor.java | 2 +- .../iginx/influxdb/InfluxDBStorage.java | 4 +- .../iginx/iotdb/tools/DataViewWrapper.java | 2 +- .../iginx/mongodb/MongoDBStorage.java | 4 +- .../iginx/opentsdb/tools/DataViewWrapper.java | 2 +- .../iginx/parquet/exec/RemoteExecutor.java | 10 +- .../iginx/parquet/tools/DataViewWrapper.java | 2 +- .../iginx/postgresql/PostgreSQLStorage.java | 93 ++++++++----------- .../entity/PostgreSQLQueryRowStream.java | 35 ++++--- .../iginx/postgresql/tools/Constants.java | 16 ++-- .../iginx/redis/tools/DataViewWrapper.java | 6 +- .../edu/tsinghua/iginx/utils/StringUtils.java | 2 +- 16 files changed, 87 insertions(+), 101 deletions(-) diff --git a/.github/postgresql_macos.sh b/.github/postgresql_macos.sh index 9d371128cd..a369dc4cce 100644 --- a/.github/postgresql_macos.sh +++ b/.github/postgresql_macos.sh @@ -6,7 +6,7 @@ sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0 sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties -sh -c "wget https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" +sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" sh -c "sudo unzip -q postgresql-15.2-1-osx-binaries.zip" diff --git a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 index f89b06b26c..0211f01ce3 100644 --- a/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 +++ b/antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4 @@ -954,7 +954,7 @@ fragment NAME_CHAR ; fragment CN_CHAR - : '\u2E85' .. '\u9FFF' + : '\u2E86' .. '\u9FFF' ; DOUBLE_QUOTE_STRING_LITERAL diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java index 53b1bf0a75..da2ffc0de2 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java @@ -72,7 +72,7 @@ public int getPathNum() { return endPathIndex - startPathIndex; } - public int getTimeSize() { + public int getKeySize() { return endKeyIndex - startKeyIndex; } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java index 46bce1f0a3..a87c4399f7 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java @@ -795,7 +795,7 @@ public void saveTimeSeriesData(InsertStatement statement) { long totalByte = 0L; int count = 0; BitmapView bitmapView = view.getBitmapView(i); - for (int j = 0; j < view.getTimeSize(); j++) { + for (int j = 0; j < view.getKeySize(); j++) { if (bitmapView.get(j)) { minn = Math.min(minn, view.getKey(j)); maxx = Math.max(maxx, view.getKey(j)); @@ -822,7 +822,7 @@ public void saveTimeSeriesData(InsertStatement statement) { Arrays.fill(minn, Long.MAX_VALUE); Arrays.fill(maxx, Long.MIN_VALUE); - for (int i = 0; i < view.getTimeSize(); i++) { + for (int i = 0; i < view.getKeySize(); i++) { BitmapView bitmapView = view.getBitmapView(i); int index = 0; for (int j = 0; j < view.getPathNum(); j++) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java index c6eb4c45b2..7c0a78962e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java @@ -34,7 +34,7 @@ public void record(FragmentMeta fragmentMeta, Operator operator) { if (operator.getType() == OperatorType.Insert) { Insert insert = (Insert) operator; long count = writeRequestsMap.getOrDefault(fragmentMeta, 0L); - count += (long) insert.getData().getPathNum() * insert.getData().getTimeSize(); + count += (long) insert.getData().getPathNum() * insert.getData().getKeySize(); writeRequestsMap.put(fragmentMeta, count); } else if (operator.getType() == OperatorType.Project) { long count = readRequestsMap.getOrDefault(fragmentMeta, 0L); diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java index 92e1beeed2..883b1af501 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java @@ -511,7 +511,7 @@ private Exception insertRowRecords(RowDataView data, String storageUnit) { } List points = new ArrayList<>(); - for (int i = 0; i < data.getTimeSize(); i++) { + for (int i = 0; i < data.getKeySize(); i++) { BitmapView bitmapView = data.getBitmapView(i); int index = 0; for (int j = 0; j < data.getPathNum(); j++) { @@ -619,7 +619,7 @@ private Exception insertColumnRecords(ColumnDataView data, String storageUnit) { InfluxDBSchema schema = new InfluxDBSchema(data.getPath(i), data.getTags(i)); BitmapView bitmapView = data.getBitmapView(i); int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { + for (int j = 0; j < data.getKeySize(); j++) { if (bitmapView.get(j)) { switch (data.getDataType(i)) { case BOOLEAN: diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java index 303a9d4333..ec37b1baf5 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java @@ -41,7 +41,7 @@ public int getPathNum() { } public int getTimeSize() { - return dataView.getTimeSize(); + return dataView.getKeySize(); } public String getPath(int index) { diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java index dabf2b4798..2cbbd4be39 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java +++ b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java @@ -368,7 +368,7 @@ private Exception insertRowRecords(RowDataView data, String storageUnit) { } Map> points = new HashMap<>(); - for (int i = 0; i < data.getTimeSize(); i++) { + for (int i = 0; i < data.getKeySize(); i++) { BitmapView bitmapView = data.getBitmapView(i); int index = 0; for (int j = 0; j < data.getPathNum(); j++) { @@ -427,7 +427,7 @@ private Exception insertColumnRecords(ColumnDataView data, String storageUnit) { BitmapView bitmapView = data.getBitmapView(i); int index = 0; List jsonObjects = new ArrayList<>(); - for (int j = 0; j < data.getTimeSize(); j++) { + for (int j = 0; j < data.getKeySize(); j++) { if (bitmapView.get(j)) { Map timeAndValueMap = new HashMap<>(); timeAndValueMap.put(MongoDBStorage.INNER_TIMESTAMP, data.getKey(j)); diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java index aa2ee5a0cc..e2905464b6 100644 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java +++ b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java @@ -41,7 +41,7 @@ public int getPathNum() { } public int getTimeSize() { - return dataView.getTimeSize(); + return dataView.getKeySize(); } public String getPath(int index) { diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java index 427917c4f4..0fe79e2d0b 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java @@ -154,8 +154,8 @@ public TaskExecuteResult executeInsertTask(DataView dataView, String storageUnit tagsList.add(dataView.getTags(i) == null ? new HashMap<>() : dataView.getTags(i)); } - long[] times = new long[dataView.getTimeSize()]; - for (int i = 0; i < dataView.getTimeSize(); i++) { + long[] times = new long[dataView.getKeySize()]; + for (int i = 0; i < dataView.getKeySize(); i++) { times[i] = dataView.getKey(i); } @@ -200,7 +200,7 @@ private Pair, List> compressRowData(DataView dataVi dataTypeList.add(dataView.getDataType(i)); } - for (int i = 0; i < dataView.getTimeSize(); i++) { + for (int i = 0; i < dataView.getKeySize(); i++) { BitmapView bitmapView = dataView.getBitmapView(i); Object[] values = new Object[dataView.getPathNum()]; @@ -226,10 +226,10 @@ private Pair, List> compressColData(DataView dataVi for (int i = 0; i < dataView.getPathNum(); i++) { DataType dataType = dataView.getDataType(i); BitmapView bitmapView = dataView.getBitmapView(i); - Object[] values = new Object[dataView.getTimeSize()]; + Object[] values = new Object[dataView.getKeySize()]; int index = 0; - for (int j = 0; j < dataView.getTimeSize(); j++) { + for (int j = 0; j < dataView.getKeySize(); j++) { if (bitmapView.get(j)) { values[j] = dataView.getValue(i, index); index++; diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/DataViewWrapper.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/DataViewWrapper.java index 025afb504b..bf42564304 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/DataViewWrapper.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/DataViewWrapper.java @@ -23,7 +23,7 @@ public int getPathNum() { } public int getTimeSize() { - return dataView.getTimeSize(); + return dataView.getKeySize(); } public String getPath(int index) { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 0831b05404..7835a13941 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -180,7 +180,7 @@ public List getColumns() { while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 - if (columnName.equals("time")) { // time 列不显示 + if (columnName.equals(KEY_NAME)) { // key 列不显示 continue; } Pair> nameAndTags = @@ -509,8 +509,8 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { @Override public Pair getBoundaryOfStorage(String prefix) throws PhysicalException { - long minTime = Long.MAX_VALUE; - long maxTime = 0; + long minKey = Long.MAX_VALUE; + long maxKey = 0; List paths = new ArrayList<>(); try { Statement stmt = connection.createStatement(); @@ -530,14 +530,8 @@ public Pair getBoundaryOfStorage(String prefix) ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); StringBuilder columnNames = new StringBuilder(); - boolean hasTimeColumn = false; while (columnSet.next()) { String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 - String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 - if (columnName.equalsIgnoreCase("time") - && fromPostgreSQL(typeName) == DataType.LONG) { - hasTimeColumn = true; - } paths.add( databaseName + IGINX_SEPARATOR @@ -547,37 +541,24 @@ && fromPostgreSQL(typeName) == DataType.LONG) { columnNames.append(columnName); columnNames.append(", "); // c1, c2, c3, } - if (hasTimeColumn) { // 本身就有 time 列 - // 获取 key 的范围 - String statement = - String.format(QUERY_TIME_STATEMENT, getFullName(tableName)); - Statement timeStmt = conn.createStatement(); - ResultSet timeSet = timeStmt.executeQuery(statement); - while (timeSet.next()) { - long time = timeSet.getLong("time"); - minTime = Math.min(time, minTime); - maxTime = Math.max(time, maxTime); - } - } else { - columnNames = - new StringBuilder( - columnNames.substring( - 0, columnNames.length() - 2)); // c1, c2, c3 + columnNames = + new StringBuilder( + columnNames.substring( + 0, columnNames.length() - 2)); // c1, c2, c3 - // 获取 key 的范围 - String statement = - String.format( - CONCAT_QUERY_STATEMENT, - getFullColumnNames(columnNames.toString()), - getFullName(tableName)); - Statement concatStmt = conn.createStatement(); - ResultSet concatSet = concatStmt.executeQuery(statement); - while (concatSet.next()) { - String concatValue = concatSet.getString("concat"); - long time = toHash(concatValue); - minTime = Math.min(time, minTime); - maxTime = Math.max(time, maxTime); - } + // 获取 key 的范围 + String statement = + String.format( + CONCAT_QUERY_STATEMENT, + getFullColumnNames(columnNames.toString()), + getFullName(tableName)); + Statement concatStmt = conn.createStatement(); + ResultSet concatSet = concatStmt.executeQuery(statement); + while (concatSet.next()) { + String concatValue = concatSet.getString("concat"); + long key = toHash(concatValue); + minKey = Math.min(key, minKey); + maxKey = Math.max(key, maxKey); } } } @@ -588,7 +569,7 @@ && fromPostgreSQL(typeName) == DataType.LONG) { paths.sort(String::compareTo); return new Pair<>( new ColumnsInterval(paths.get(0), paths.get(paths.size() - 1)), - new KeyInterval(minTime, maxTime + 1)); + new KeyInterval(minKey, maxKey + 1)); } private Map splitAndMergeQueryPatterns( @@ -634,7 +615,7 @@ private Map splitAndMergeQueryPatterns( while (rs.next()) { tableName = rs.getString("TABLE_NAME"); columnNames = rs.getString("COLUMN_NAME"); - if (columnNames.equals("time")) { + if (columnNames.equals(KEY_NAME)) { continue; } if (tableNameToColumnNames.containsKey(tableName)) { @@ -813,7 +794,7 @@ private void createOrAlterTables( private Exception insertNonAlignedRowRecords( Connection conn, String databaseName, RowDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + int batchSize = Math.min(data.getKeySize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); @@ -830,8 +811,8 @@ private Exception insertNonAlignedRowRecords( new HashMap<>(); // <表名, <列名，值列表>> int cnt = 0; boolean firstRound = true; - while (cnt < data.getTimeSize()) { - int size = Math.min(data.getTimeSize() - cnt, batchSize); + while (cnt < data.getKeySize()) { + int size = Math.min(data.getKeySize() - cnt, batchSize); Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 for (int i = cnt; i < cnt + size; i++) { BitmapView bitmapView = data.getBitmapView(i); @@ -885,7 +866,7 @@ private Exception insertNonAlignedRowRecords( columnValues.set(i - cnt, columnValues.get(i - cnt) + value + ", "); } else { columnValues.add( - data.getKey(i) + ", " + value + ", "); // 添加 key(time) 列 + data.getKey(i) + ", " + value + ", "); // 添加 key 列 } tableToColumnEntries.put( @@ -932,7 +913,7 @@ private Exception insertNonAlignedRowRecords( private Exception insertNonAlignedColumnRecords( Connection conn, String databaseName, ColumnDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); + int batchSize = Math.min(data.getKeySize(), BATCH_SIZE); try { Statement stmt = conn.createStatement(); @@ -950,8 +931,8 @@ private Exception insertNonAlignedColumnRecords( Map pathIndexToBitmapIndex = new HashMap<>(); int cnt = 0; boolean firstRound = true; - while (cnt < data.getTimeSize()) { - int size = Math.min(data.getTimeSize() - cnt, batchSize); + while (cnt < data.getKeySize()) { + int size = Math.min(data.getKeySize() - cnt, batchSize); Map tableHasData = new HashMap<>(); // 记录每一张表的每一行是否有数据点 for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); @@ -1005,7 +986,7 @@ private Exception insertNonAlignedColumnRecords( columnValues.set(j - cnt, columnValues.get(j - cnt) + value + ", "); } else { columnValues.add( - data.getKey(j) + ", " + value + ", "); // 添加 key(time) 列 + data.getKey(j) + ", " + value + ", "); // 添加 key 列 } } pathIndexToBitmapIndex.put(i, index); @@ -1066,12 +1047,14 @@ private void executeBatchInsert( String[] parts = columnNames.split(", "); boolean hasMultipleRows = parts.length != 1; - // INSERT INTO XXX (time, XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., (XXX, - // XXX, ...) ON CONFLICT (time) DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); + // INSERT INTO XXX ("\u2E85", XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., (XXX, + // XXX, ...) ON CONFLICT ("\u2E85") DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); StringBuilder statement = new StringBuilder(); statement.append("INSERT INTO "); statement.append(getFullName(tableName)); - statement.append(" (time, "); + statement.append(" (\""); + statement.append(KEY_NAME); + statement.append("\", "); String fullColumnNames = getFullColumnNames(columnNames); statement.append(fullColumnNames); @@ -1083,7 +1066,9 @@ private void executeBatchInsert( } statement = new StringBuilder(statement.substring(0, statement.length() - 2)); - statement.append(" ON CONFLICT (time) DO UPDATE SET "); + statement.append(" ON CONFLICT (\""); + statement.append(KEY_NAME); + statement.append("\") DO UPDATE SET "); if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } @@ -1106,7 +1091,7 @@ private void executeBatchInsert( } statement.append(";"); - // logger.info("[Insert] execute insert: {}", statement); + logger.info("[Insert] execute insert: {}", statement); stmt.addBatch(statement.toString()); } stmt.executeBatch(); diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index e2c82891af..fd25c6969d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -1,8 +1,7 @@ package cn.edu.tsinghua.iginx.postgresql.query.entity; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.IGINX_SEPARATOR; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.POSTGRESQL_SEPARATOR; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; @@ -39,7 +38,7 @@ public class PostgreSQLQueryRowStream implements RowStream { private boolean[] gotNext; // 标记每个结果集是否已经获取到下一行，如果是，则在下次调用 next() 时无需再调用该结果集的 next() - private long[] cachedTimestamps; // 缓存每个结果集当前的 time 列的值 + private long[] cachedKeys; // 缓存每个结果集当前的 key 列的值 private Object[] cachedValues; // 缓存每列当前的值 @@ -69,7 +68,7 @@ public PostgreSQLQueryRowStream( boolean filterByTags = tagFilter != null; - Field time = null; + Field key = null; List fields = new ArrayList<>(); this.resultSetSizes = new int[resultSets.size()]; this.fieldToColumnName = new HashMap<>(); @@ -81,8 +80,8 @@ public PostgreSQLQueryRowStream( String tableName = resultSetMetaData.getTableName(j); String columnName = resultSetMetaData.getColumnName(j); String typeName = resultSetMetaData.getColumnTypeName(j); - if (j == 1 && columnName.equals("time")) { - time = Field.KEY; + if (j == 1 && columnName.equals(KEY_NAME)) { + key = Field.KEY; continue; } @@ -108,12 +107,12 @@ public PostgreSQLQueryRowStream( resultSetSizes[i] = cnt; } - this.header = new Header(time, fields); + this.header = new Header(key, fields); this.gotNext = new boolean[resultSets.size()]; Arrays.fill(gotNext, false); - this.cachedTimestamps = new long[resultSets.size()]; - Arrays.fill(cachedTimestamps, Long.MAX_VALUE); + this.cachedKeys = new long[resultSets.size()]; + Arrays.fill(cachedKeys, Long.MAX_VALUE); this.cachedValues = new Object[fields.size()]; Arrays.fill(cachedValues, null); this.cachedRow = null; @@ -172,7 +171,7 @@ public Row next() throws PhysicalException { private void cacheOneRow() throws SQLException, PhysicalException { boolean hasNext = false; - long timestamp; + long key; Object[] values = new Object[header.getFieldSize()]; int startIndex = 0; @@ -189,15 +188,15 @@ private void cacheOneRow() throws SQLException, PhysicalException { gotNext[i] = true; if (tempHasNext) { - long tempTimestamp; + long tempKey; Object tempValue; if (isDummy) { - tempTimestamp = toHash(resultSet.getString("time")); + tempKey = toHash(resultSet.getString(KEY_NAME)); } else { - tempTimestamp = resultSet.getLong("time"); + tempKey = resultSet.getLong(KEY_NAME); } - cachedTimestamps[i] = tempTimestamp; + cachedKeys[i] = tempKey; for (int j = 0; j < resultSetSizes[i]; j++) { Object value = @@ -212,7 +211,7 @@ private void cacheOneRow() throws SQLException, PhysicalException { cachedValues[startIndex + j] = tempValue; } } else { - cachedTimestamps[i] = Long.MAX_VALUE; + cachedKeys[i] = Long.MAX_VALUE; for (int j = startIndex; j < endIndex; j++) { cachedValues[j] = null; } @@ -224,12 +223,12 @@ private void cacheOneRow() throws SQLException, PhysicalException { } if (hasNext) { - timestamp = Arrays.stream(cachedTimestamps).min().getAsLong(); + key = Arrays.stream(cachedKeys).min().getAsLong(); startIndex = 0; endIndex = 0; for (int i = 0; i < resultSets.size(); i++) { endIndex += resultSetSizes[i]; - if (cachedTimestamps[i] == timestamp) { + if (cachedKeys[i] == key) { for (int j = 0; j < resultSetSizes[i]; j++) { values[startIndex + j] = cachedValues[startIndex + j]; } @@ -242,7 +241,7 @@ private void cacheOneRow() throws SQLException, PhysicalException { } startIndex = endIndex; } - cachedRow = new Row(header, timestamp, values); + cachedRow = new Row(header, key, values); if (isDummy && !validate(filter, cachedRow)) { cacheOneRow(); } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index facecb485e..ccefb490a9 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -18,24 +18,26 @@ public class Constants { public static final String DATABASE_PREFIX = "unit"; + public static final String KEY_NAME = "\u2E85"; + public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; - public static final String QUERY_TIME_STATEMENT = "SELECT time FROM %s ORDER BY time;"; + public static final String QUERY_KEY_STATEMENT = "SELECT \"" + KEY_NAME + "\" FROM %s ORDER BY \"" + KEY_NAME + "\";"; public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; - public static final String QUERY_STATEMENT = "SELECT time, %s FROM %s WHERE %s ORDER BY time;"; + public static final String QUERY_STATEMENT = "SELECT \"" + KEY_NAME + "\", %s FROM %s WHERE %s ORDER BY \"" + KEY_NAME + "\";"; - public static final String QUERY_TIME_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT %s FROM %s ORDER BY time;"; + public static final String QUERY_KEY_STATEMENT_WITHOUT_WHERE_CLAUSE = + "SELECT %s FROM %s ORDER BY \"" + KEY_NAME + "\";"; public static final String CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT concat(%s) AS time, %s FROM %s ORDER BY time;"; + "SELECT concat(%s) AS \"" + KEY_NAME + "\", %s FROM %s ORDER BY \"" + KEY_NAME + "\";"; public static final String CREATE_TABLE_STATEMENT = - "CREATE TABLE %s (time BIGINT NOT NULL, %s %s, PRIMARY KEY(time));"; + "CREATE TABLE %s (\"" + KEY_NAME + "\" BIGINT NOT NULL, %s %s, PRIMARY KEY(\"" + KEY_NAME + "\"));"; public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; @@ -44,5 +46,5 @@ public class Constants { public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; public static final String UPDATE_STATEMENT = - "UPDATE %s SET %s = null WHERE (time >= %d AND time < %d);"; + "UPDATE %s SET %s = null WHERE (\"" + KEY_NAME + "\" >= %d AND \"" + KEY_NAME + "\" < %d);"; } diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java b/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java index 28ea83bc29..826a56eb71 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java +++ b/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/tools/DataViewWrapper.java @@ -23,7 +23,7 @@ public int getPathNum() { } public int getTimeSize() { - return dataView.getTimeSize(); + return dataView.getKeySize(); } public String getPath(int index) { @@ -59,7 +59,7 @@ public Pair, Map> getPathData(int index) { Pair, Map> ret = new Pair<>(values, scores); if (dataView.isRowData()) { - for (int i = 0; i < dataView.getTimeSize(); i++) { + for (int i = 0; i < dataView.getKeySize(); i++) { BitmapView bitmapView = dataView.getBitmapView(i); if (bitmapView.get(index)) { int nonNullCnt = 0; @@ -79,7 +79,7 @@ public Pair, Map> getPathData(int index) { } else { int nonNullCnt = 0; BitmapView bitmapView = dataView.getBitmapView(index); - for (int i = 0; i < dataView.getTimeSize(); i++) { + for (int i = 0; i < dataView.getKeySize(); i++) { if (bitmapView.get(i)) { String key = String.valueOf(dataView.getKey(i)); double score = dataView.getKey(i); diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java index df460acda2..beef152bfb 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java @@ -112,7 +112,7 @@ public static String reformatColumnName(String name) { } public static boolean isContainSpecialChar(String str) { - String regEx = "[~!@#$%&()+=|{}':;',<>?~]|\r|\n|\t|[\u2E80\u2E81\u2E82\u2E83\u2E84]"; + String regEx = "[~!@#$%&()+=|{}':;',<>?~]|\r|\n|\t|[\u2E80\u2E81\u2E82\u2E83\u2E84\u2E85]"; Pattern p = Pattern.compile(regEx); Matcher m = p.matcher(str); return m.find(); From d025b0dcc6c06f647abebd87485d4dbba7d5a1d3 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 09:59:57 +0800 Subject: [PATCH 86/94] Change key name --- .../iginx/postgresql/PostgreSQLStorage.java | 17 ++++++++-------- .../iginx/postgresql/tools/Constants.java | 20 ++++++++++++------- .../postgresql/tools/FilterTransformer.java | 9 +++------ .../redis/RedisHistoryDataGenerator.java | 1 - 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 7835a13941..d5e7d3afbc 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -865,8 +865,7 @@ private Exception insertNonAlignedRowRecords( if (i - cnt < columnValues.size()) { columnValues.set(i - cnt, columnValues.get(i - cnt) + value + ", "); } else { - columnValues.add( - data.getKey(i) + ", " + value + ", "); // 添加 key 列 + columnValues.add(data.getKey(i) + ", " + value + ", "); // 添加 key 列 } tableToColumnEntries.put( @@ -985,8 +984,7 @@ private Exception insertNonAlignedColumnRecords( if (j - cnt < columnValues.size()) { columnValues.set(j - cnt, columnValues.get(j - cnt) + value + ", "); } else { - columnValues.add( - data.getKey(j) + ", " + value + ", "); // 添加 key 列 + columnValues.add(data.getKey(j) + ", " + value + ", "); // 添加 key 列 } } pathIndexToBitmapIndex.put(i, index); @@ -1047,14 +1045,15 @@ private void executeBatchInsert( String[] parts = columnNames.split(", "); boolean hasMultipleRows = parts.length != 1; - // INSERT INTO XXX ("\u2E85", XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., (XXX, + // INSERT INTO XXX ("\u2E85", XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., + // (XXX, // XXX, ...) ON CONFLICT ("\u2E85") DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); StringBuilder statement = new StringBuilder(); statement.append("INSERT INTO "); statement.append(getFullName(tableName)); - statement.append(" (\""); + statement.append(" ("); statement.append(KEY_NAME); - statement.append("\", "); + statement.append(", "); String fullColumnNames = getFullColumnNames(columnNames); statement.append(fullColumnNames); @@ -1066,9 +1065,9 @@ private void executeBatchInsert( } statement = new StringBuilder(statement.substring(0, statement.length() - 2)); - statement.append(" ON CONFLICT (\""); + statement.append(" ON CONFLICT ("); statement.append(KEY_NAME); - statement.append("\") DO UPDATE SET "); + statement.append(") DO UPDATE SET "); if (hasMultipleRows) { statement.append("("); // 只有一列不加括号 } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index ccefb490a9..695b56694a 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -18,26 +18,32 @@ public class Constants { public static final String DATABASE_PREFIX = "unit"; - public static final String KEY_NAME = "\u2E85"; + public static final String KEY_NAME = "\"\u2E85\""; public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; - public static final String QUERY_KEY_STATEMENT = "SELECT \"" + KEY_NAME + "\" FROM %s ORDER BY \"" + KEY_NAME + "\";"; + public static final String QUERY_KEY_STATEMENT = + "SELECT " + KEY_NAME + " FROM %s ORDER BY " + KEY_NAME + ";"; public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; - public static final String QUERY_STATEMENT = "SELECT \"" + KEY_NAME + "\", %s FROM %s WHERE %s ORDER BY \"" + KEY_NAME + "\";"; + public static final String QUERY_STATEMENT = + "SELECT " + KEY_NAME + ", %s FROM %s WHERE %s ORDER BY " + KEY_NAME + ";"; public static final String QUERY_KEY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT %s FROM %s ORDER BY \"" + KEY_NAME + "\";"; + "SELECT %s FROM %s ORDER BY " + KEY_NAME + ";"; public static final String CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT concat(%s) AS \"" + KEY_NAME + "\", %s FROM %s ORDER BY \"" + KEY_NAME + "\";"; + "SELECT concat(%s) AS \"" + KEY_NAME + "\", %s FROM %s ORDER BY " + KEY_NAME + ";"; public static final String CREATE_TABLE_STATEMENT = - "CREATE TABLE %s (\"" + KEY_NAME + "\" BIGINT NOT NULL, %s %s, PRIMARY KEY(\"" + KEY_NAME + "\"));"; + "CREATE TABLE %s (\"" + + KEY_NAME + + "\" BIGINT NOT NULL, %s %s, PRIMARY KEY(" + + KEY_NAME + + "));"; public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; @@ -46,5 +52,5 @@ public class Constants { public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; public static final String UPDATE_STATEMENT = - "UPDATE %s SET %s = null WHERE (\"" + KEY_NAME + "\" >= %d AND \"" + KEY_NAME + "\" < %d);"; + "UPDATE %s SET %s = null WHERE (" + KEY_NAME + " >= %d AND " + KEY_NAME + " < %d);"; } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java index 2212db5e7a..ee8a877f7d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java @@ -18,13 +18,13 @@ */ package cn.edu.tsinghua.iginx.postgresql.tools; +import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.KEY_NAME; + import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; import java.util.stream.Collectors; public class FilterTransformer { - public static final long MAX_TIMESTAMP = Long.MAX_VALUE; - public static String toString(Filter filter) { if (filter == null) { return ""; @@ -57,10 +57,7 @@ private static String toString(NotFilter filter) { } private static String toString(KeyFilter filter) { - return "time " - + Op.op2Str(filter.getOp()) - + " " - + Math.min(filter.getValue(), MAX_TIMESTAMP); + return KEY_NAME + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue(); } private static String toString(ValueFilter filter) { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java index 2c062ac3b0..e1bee13ad9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisHistoryDataGenerator.java @@ -2,7 +2,6 @@ import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; import java.util.List; - import org.junit.Test; import redis.clients.jedis.Jedis; From e7837833156c7a35fb6c36ed90a35011402d1ef5 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 10:07:14 +0800 Subject: [PATCH 87/94] Change key name --- .../cn/edu/tsinghua/iginx/postgresql/tools/Constants.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index 695b56694a..fb2b71201f 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -36,12 +36,12 @@ public class Constants { "SELECT %s FROM %s ORDER BY " + KEY_NAME + ";"; public static final String CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT concat(%s) AS \"" + KEY_NAME + "\", %s FROM %s ORDER BY " + KEY_NAME + ";"; + "SELECT concat(%s) AS " + KEY_NAME + ", %s FROM %s ORDER BY " + KEY_NAME + ";"; public static final String CREATE_TABLE_STATEMENT = - "CREATE TABLE %s (\"" + "CREATE TABLE %s (" + KEY_NAME - + "\" BIGINT NOT NULL, %s %s, PRIMARY KEY(" + + " BIGINT NOT NULL, %s %s, PRIMARY KEY(" + KEY_NAME + "));"; From 9063d079e462d19be87d4ec2c6ea9727563eb21d Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 10:48:13 +0800 Subject: [PATCH 88/94] Change key name --- .../entity/PostgreSQLQueryRowStream.java | 33 +++++++++++++------ .../iginx/postgresql/tools/Constants.java | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java index fd25c6969d..aac3ace17f 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java @@ -86,16 +86,29 @@ public PostgreSQLQueryRowStream( } Pair> namesAndTags = splitFullName(columnName); - Field field = - new Field( - databaseNameList.get(i) - + IGINX_SEPARATOR - + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) - + IGINX_SEPARATOR - + namesAndTags.k.replace( - POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromPostgreSQL(typeName), - namesAndTags.v); + Field field; + if (isDummy) { + field = + new Field( + databaseNameList.get(i) + + IGINX_SEPARATOR + + tableName.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + + namesAndTags.k.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v); + } else { + field = + new Field( + tableName.replace(POSTGRESQL_SEPARATOR, IGINX_SEPARATOR) + + IGINX_SEPARATOR + + namesAndTags.k.replace( + POSTGRESQL_SEPARATOR, IGINX_SEPARATOR), + DataTypeTransformer.fromPostgreSQL(typeName), + namesAndTags.v); + } if (filterByTags && !TagKVUtils.match(namesAndTags.v, tagFilter)) { continue; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java index fb2b71201f..b8f22abcef 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java @@ -18,7 +18,7 @@ public class Constants { public static final String DATABASE_PREFIX = "unit"; - public static final String KEY_NAME = "\"\u2E85\""; + public static final String KEY_NAME = "\u2E85"; public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; From 3e1313b9ec108456e94a8df6dad7cd1aff5de695 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 11:07:53 +0800 Subject: [PATCH 89/94] Modify port number --- .../cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java | 2 +- .../expansion/postgresql/PostgreSQLCapacityExpansionIT.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index d5e7d3afbc..8bb4ebe638 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -1090,7 +1090,7 @@ private void executeBatchInsert( } statement.append(";"); - logger.info("[Insert] execute insert: {}", statement); + // logger.info("[Insert] execute insert: {}", statement); stmt.addBatch(statement.toString()); } stmt.executeBatch(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java index 851ef4627b..39a56223da 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java @@ -37,7 +37,7 @@ public void addStorageEngineWithPrefix(String dataPrefix, String schemaPrefix) { public void addStorageEngine(boolean hasData) { try { session.executeSql( - "ADD STORAGEENGINE (\"127.0.0.1\", 5431, \"" + "ADD STORAGEENGINE (\"127.0.0.1\", 5433, \"" + dbType.name() + "\", \"username:postgres, password:postgres, has_data:" + hasData From 603d9612eefbcd556af145928b6c11de52032e16 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 11:19:21 +0800 Subject: [PATCH 90/94] Change timeseries to columns --- .../integration/func/sql/SQLSessionIT.java | 6 +++--- .../iginx/integration/tool/DBConf.java | 6 +++--- test/src/test/resources/testConfig.properties | 18 +++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index e1e24470a7..5f33d89beb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -49,7 +49,7 @@ public class SQLSessionIT { protected boolean isSupportSpecialCharacterPath; - protected boolean isAbleToShowTimeSeries; + protected boolean isAbleToShowColumns; protected boolean isScaling = false; @@ -66,7 +66,7 @@ public SQLSessionIT() { this.isScaling = conf.isScaling(); this.isAbleToClearData = dbConf.getEnumValue(DBConf.DBConfType.isAbleToClearData); this.isAbleToDelete = dbConf.getEnumValue(DBConf.DBConfType.isAbleToDelete); - this.isAbleToShowTimeSeries = dbConf.getEnumValue(DBConf.DBConfType.isAbleToShowTimeSeries); + this.isAbleToShowColumns = dbConf.getEnumValue(DBConf.DBConfType.isAbleToShowColumns); this.isSupportChinesePath = dbConf.getEnumValue(DBConfType.isSupportChinesePath); this.isSupportNumericalPath = dbConf.getEnumValue(DBConfType.isSupportNumericalPath); this.isSupportSpecialCharacterPath = @@ -179,7 +179,7 @@ public void testCountPoints() { @Test public void testShowTimeSeries() { - if (!isAbleToShowTimeSeries || isScaling) { + if (!isAbleToShowColumns || isScaling) { return; } String statement = "SHOW COLUMNS us.*;"; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java index af436b2cc6..dd370ecea1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/DBConf.java @@ -8,7 +8,7 @@ public final class DBConf { public enum DBConfType { isAbleToClearData, isAbleToDelete, - isAbleToShowTimeSeries, + isAbleToShowColumns, isSupportChinesePath, isSupportNumericalPath, isSupportSpecialCharacterPath @@ -29,8 +29,8 @@ public static DBConfType getDBConfType(String str) { return DBConfType.isAbleToClearData; case "isAbleToDelete": return DBConfType.isAbleToDelete; - case "isAbleToShowTimeSeries": - return DBConfType.isAbleToShowTimeSeries; + case "isAbleToShowColumns": + return DBConfType.isAbleToShowColumns; case "isSupportChinesePath": return DBConfType.isSupportChinesePath; case "isSupportNumericalPath": diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index 40b0a1b412..3b5ed1a2bd 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -6,18 +6,18 @@ storageEngineList=IoTDB12,InfluxDB,Parquet,PostgreSQL,MongoDB,Redis IoTDB12=127.0.0.1#6668#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false InfluxDB=127.0.0.1#8087#influxdb#url=http://localhost:8087/#username=user#password=12345678#sessionPoolSize=20#has_data=false#is_read_only=false#token=testToken#organization=testOrg Parquet=127.0.0.1#6668#parquet#dir=parquetData2#sessionPoolSize=20#has_data=false#is_read_only=false -PostgreSQL=127.0.0.1#5432#postgresql#username=postgres#password=postgres -MongoDB=127.0.0.1#27017#mongodb -Redis=127.0.0.1#6380#redis#has_data=false +PostgreSQL=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false#is_read_only=false +MongoDB=127.0.0.1#27017#mongodb#has_data=false#is_read_only=false +Redis=127.0.0.1#6380#redis#has_data=false#is_read_only=false # the test for every engine test-list=SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT mongodb-test-list=SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,UDFIT,TransformIT,SessionV2IT,CompactionIT,TimePrecisionIT # the DB config -IoTDB12-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=false -InfluxDB-config=isAbleToClearData=true,isAbleToDelete=false,isAbleToShowTimeSeries=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false -Parquet-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false -PostgreSQL-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=true -MongoDB-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false -Redis-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowTimeSeries=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false +IoTDB12-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=false +InfluxDB-config=isAbleToClearData=true,isAbleToDelete=false,isAbleToShowColumns=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false +Parquet-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false +PostgreSQL-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=true +MongoDB-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false +Redis-config=isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=false,isSupportNumericalPath=false,isSupportSpecialCharacterPath=false From 03b36b12b0be8c6f0578dc548c6e33c9a9ee3e20 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 14:10:30 +0800 Subject: [PATCH 91/94] Fix clear data --- .../PostgreSQLHistoryDataGenerator.java | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 3298637c5f..3735fa2c41 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -2,10 +2,7 @@ import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; import cn.edu.tsinghua.iginx.thrift.DataType; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.*; import org.junit.Test; import org.slf4j.Logger; @@ -16,9 +13,11 @@ public class PostgreSQLHistoryDataGenerator extends BaseHistoryDataGenerator { private static final Logger logger = LoggerFactory.getLogger(PostgreSQLHistoryDataGenerator.class); - public static final char IGINX_SEPARATOR = '.'; + private static final char IGINX_SEPARATOR = '.'; - public static final char POSTGRESQL_SEPARATOR = '\u2E82'; + private static final char POSTGRESQL_SEPARATOR = '\u2E82'; + + private static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; private static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; @@ -32,8 +31,6 @@ public class PostgreSQLHistoryDataGenerator extends BaseHistoryDataGenerator { private static final String PASSWORD = "postgres"; - private static final Set databaseNameList = new HashSet<>(); - public PostgreSQLHistoryDataGenerator() { this.portOri = 5432; this.portExp = 5433; @@ -102,7 +99,6 @@ private void writeHistoryData( Statement stmt = conn.createStatement(); try { stmt.execute(String.format(CREATE_DATABASE_STATEMENT, databaseName)); - databaseNameList.add(databaseName); } catch (SQLException e) { logger.info("database {} exists!", databaseName); } @@ -167,8 +163,23 @@ private void clearHistoryData(int port) { try { Connection conn = connect(port, true, null); Statement stmt = conn.createStatement(); - for (String databaseName : databaseNameList) { - stmt.execute(String.format(DROP_DATABASE_STATEMENT, databaseName)); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + while (databaseSet.next()) { + String databaseName = databaseSet.getString("DATNAME"); + try { + if (databaseName.equalsIgnoreCase("template0") + || databaseName.equalsIgnoreCase("template1") + || databaseName.equalsIgnoreCase("postgres")) { + continue; + } + stmt.execute(String.format(DROP_DATABASE_STATEMENT, databaseName)); + } catch (SQLException e) { + logger.error( + "drop database {} on 127.0.0.1:{} failure: {}", + databaseName, + port, + e.getMessage()); + } } stmt.close(); conn.close(); From 8984c9133654836ea49da55716b9e6709fdc5297 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 15:03:44 +0800 Subject: [PATCH 92/94] Debug --- .../PostgreSQLHistoryDataGenerator.java | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 3735fa2c41..92f2e80a19 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -166,21 +166,15 @@ private void clearHistoryData(int port) { ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); while (databaseSet.next()) { String databaseName = databaseSet.getString("DATNAME"); - try { - if (databaseName.equalsIgnoreCase("template0") - || databaseName.equalsIgnoreCase("template1") - || databaseName.equalsIgnoreCase("postgres")) { - continue; - } - stmt.execute(String.format(DROP_DATABASE_STATEMENT, databaseName)); - } catch (SQLException e) { - logger.error( - "drop database {} on 127.0.0.1:{} failure: {}", - databaseName, - port, - e.getMessage()); + if (databaseName.equalsIgnoreCase("template0") + || databaseName.equalsIgnoreCase("template1") + || databaseName.equalsIgnoreCase("postgres")) { + continue; } + logger.error(databaseName); + stmt.execute(String.format(DROP_DATABASE_STATEMENT, databaseName)); } + databaseSet.close(); stmt.close(); conn.close(); logger.info("clear data on 127.0.0.1:{} success!", port); From 75e71c0631b15b73572166742e613cb3ce9c3780 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 16:01:40 +0800 Subject: [PATCH 93/94] Debug --- .../iginx/postgresql/PostgreSQLStorage.java | 18 +++++++++--------- .../iginx/postgresql/tools/TagKVUtils.java | 12 +++++------- .../PostgreSQLHistoryDataGenerator.java | 8 ++++++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 8bb4ebe638..79af29931c 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -742,10 +742,10 @@ private void createOrAlterTables( } DataType dataType = dataTypeList.get(i); String tableName = - path.substring(0, path.lastIndexOf('.')) + path.substring(0, path.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = - path.substring(path.lastIndexOf('.') + 1) + path.substring(path.lastIndexOf(IGINX_SEPARATOR) + 1) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); try { @@ -821,9 +821,9 @@ private Exception insertNonAlignedRowRecords( String path = data.getPath(j); DataType dataType = data.getDataType(j); String tableName = - path.substring(0, path.lastIndexOf('.')) + path.substring(0, path.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String columnName = path.substring(path.lastIndexOf('.') + 1); + String columnName = path.substring(path.lastIndexOf(IGINX_SEPARATOR) + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { tags = data.getTags(j); @@ -937,9 +937,9 @@ private Exception insertNonAlignedColumnRecords( String path = data.getPath(i); DataType dataType = data.getDataType(i); String tableName = - path.substring(0, path.lastIndexOf('.')) + path.substring(0, path.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); - String columnName = path.substring(path.lastIndexOf('.') + 1); + String columnName = path.substring(path.lastIndexOf(IGINX_SEPARATOR) + 1); Map tags = new HashMap<>(); if (data.hasTagsList()) { tags = data.getTags(i); @@ -1090,7 +1090,7 @@ private void executeBatchInsert( } statement.append(";"); - // logger.info("[Insert] execute insert: {}", statement); + logger.info("[Insert] execute insert: {}", statement); stmt.addBatch(statement.toString()); } stmt.executeBatch(); @@ -1109,11 +1109,11 @@ private List> determineDeletedPaths( } String fullPath = column.getPath(); String tableName = - fullPath.substring(0, fullPath.lastIndexOf('.')) + fullPath.substring(0, fullPath.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = toFullName( - fullPath.substring(fullPath.lastIndexOf('.') + 1), + fullPath.substring(fullPath.lastIndexOf(IGINX_SEPARATOR) + 1), column.getTags()); deletedPaths.add(new Pair<>(tableName, columnName)); break; diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java index dc56af1e1f..fab964c9f7 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -32,13 +32,11 @@ public static String toFullName(String name, Map tags) { TreeMap sortedTags = new TreeMap<>(tags); StringBuilder pathBuilder = new StringBuilder(); sortedTags.forEach( - (tagKey, tagValue) -> { - pathBuilder - .append(POSTGRESQL_SEPARATOR) - .append(tagKey) - .append(POSTGRESQL_SEPARATOR) - .append(tagValue); - }); + (tagKey, tagValue) -> pathBuilder + .append(POSTGRESQL_SEPARATOR) + .append(tagKey) + .append(POSTGRESQL_SEPARATOR) + .append(tagValue)); name += pathBuilder.toString(); } return name; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 92f2e80a19..739b1312e7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -164,6 +164,8 @@ private void clearHistoryData(int port) { Connection conn = connect(port, true, null); Statement stmt = conn.createStatement(); ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + Statement dropDatabaseStatement = conn.createStatement(); + while (databaseSet.next()) { String databaseName = databaseSet.getString("DATNAME"); if (databaseName.equalsIgnoreCase("template0") @@ -171,9 +173,11 @@ private void clearHistoryData(int port) { || databaseName.equalsIgnoreCase("postgres")) { continue; } - logger.error(databaseName); - stmt.execute(String.format(DROP_DATABASE_STATEMENT, databaseName)); + dropDatabaseStatement.addBatch(String.format(DROP_DATABASE_STATEMENT, databaseName)); } + dropDatabaseStatement.executeBatch(); + + dropDatabaseStatement.close(); databaseSet.close(); stmt.close(); conn.close(); From 6aad6a5daf83faa85092d6b10afcd16480d02a87 Mon Sep 17 00:00:00 2001 From: SolomonAnn Date: Fri, 9 Jun 2023 17:57:54 +0800 Subject: [PATCH 94/94] Remove logs --- .../iginx/engine/shared/data/write/DataView.java | 4 ---- .../iginx/postgresql/PostgreSQLStorage.java | 15 ++++----------- .../iginx/postgresql/tools/TagKVUtils.java | 11 ++++++----- .../PostgreSQLHistoryDataGenerator.java | 3 ++- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java index da2ffc0de2..b461e6cffe 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java @@ -140,8 +140,4 @@ public Map getTags(int index) { } return tagsList.get(startPathIndex + index); } - - public boolean hasTagsList() { - return getTagsList() != null && getTagsList().size() != 0; - } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java index 79af29931c..ee3e5edfff 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java @@ -824,10 +824,7 @@ private Exception insertNonAlignedRowRecords( path.substring(0, path.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = path.substring(path.lastIndexOf(IGINX_SEPARATOR) + 1); - Map tags = new HashMap<>(); - if (data.hasTagsList()) { - tags = data.getTags(j); - } + Map tags = data.getTags(j); StringBuilder columnKeys = new StringBuilder(); List columnValues = new ArrayList<>(); @@ -940,11 +937,7 @@ private Exception insertNonAlignedColumnRecords( path.substring(0, path.lastIndexOf(IGINX_SEPARATOR)) .replace(IGINX_SEPARATOR, POSTGRESQL_SEPARATOR); String columnName = path.substring(path.lastIndexOf(IGINX_SEPARATOR) + 1); - Map tags = new HashMap<>(); - if (data.hasTagsList()) { - tags = data.getTags(i); - } - + Map tags = data.getTags(i); BitmapView bitmapView = data.getBitmapView(i); StringBuilder columnKeys = new StringBuilder(); @@ -1057,7 +1050,7 @@ private void executeBatchInsert( String fullColumnNames = getFullColumnNames(columnNames); statement.append(fullColumnNames); - statement.append(") VALUES"); + statement.append(") VALUES "); for (String value : values) { statement.append("("); statement.append(value, 0, value.length() - 2); @@ -1090,7 +1083,7 @@ private void executeBatchInsert( } statement.append(";"); - logger.info("[Insert] execute insert: {}", statement); +// logger.info("[Insert] execute insert: {}", statement); stmt.addBatch(statement.toString()); } stmt.executeBatch(); diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java index fab964c9f7..925af2e485 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java @@ -32,11 +32,12 @@ public static String toFullName(String name, Map tags) { TreeMap sortedTags = new TreeMap<>(tags); StringBuilder pathBuilder = new StringBuilder(); sortedTags.forEach( - (tagKey, tagValue) -> pathBuilder - .append(POSTGRESQL_SEPARATOR) - .append(tagKey) - .append(POSTGRESQL_SEPARATOR) - .append(tagValue)); + (tagKey, tagValue) -> + pathBuilder + .append(POSTGRESQL_SEPARATOR) + .append(tagKey) + .append(POSTGRESQL_SEPARATOR) + .append(tagValue)); name += pathBuilder.toString(); } return name; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index 739b1312e7..cd93840aff 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -173,7 +173,8 @@ private void clearHistoryData(int port) { || databaseName.equalsIgnoreCase("postgres")) { continue; } - dropDatabaseStatement.addBatch(String.format(DROP_DATABASE_STATEMENT, databaseName)); + dropDatabaseStatement.addBatch( + String.format(DROP_DATABASE_STATEMENT, databaseName)); } dropDatabaseStatement.executeBatch();