add pg xx

.github/actions/dbRunner/action.yml

@@ -98,3 +98,18 @@ runs:
                 echo "$RUNNER_OS is not supported"
                 exit 1
               fi
+
+        - if: inputs.DB-name=='PostgreSQL'
+          name: Run DB
+          shell: bash
+          run: |
+              if [ "$RUNNER_OS" == "Linux" ]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/postgresql.sh"
+                "${GITHUB_WORKSPACE}/.github/postgresql.sh"
+              elif [ "$RUNNER_OS" == "macOS" ]; then
+                chmod +x "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh"
+                "${GITHUB_WORKSPACE}/.github/postgresql_macos.sh"
+              else
+                echo "$RUNNER_OS is not supported"
+                exit 1
+              fi

.github/postgresql.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties
+
+sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties
+
+sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'"
+
+sh -c "wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -"
+
+sh -c "sudo apt-get update"
+
+sh -c "sudo apt-get -y install postgresql-15"
+
+sh -c "sudo rm -rf /var/lib/postgresql/15/main"
+
+sh -c "sudo mkdir -p /var/lib/postgresql/15/main"
+
+sh -c "sudo chown -R postgres /var/lib/postgresql/15/main"
+
+sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main"
+
+sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/initdb -D /var/lib/postgresql/15/main --auth trust --no-instructions'"
+
+sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/15/main start'"
+
+sh -c "sudo su - postgres -c '/usr/lib/postgresql/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'"
+
+sh -c "sudo mkdir -p /usr/lib/postgresql2"
+
+sh -c "sudo chmod -R 777 /usr/lib/postgresql/15"
+
+sh -c "sudo chmod -R 777 /usr/lib/postgresql2"
+
+sh -c "sudo cp -R /usr/lib/postgresql/15 /usr/lib/postgresql2"
+
+sh -c "sudo mkdir -p /var/lib/postgresql2/15/main"
+
+sh -c "sudo chown -R postgres /var/lib/postgresql2/15/main"
+
+sh -c "sudo chmod -R 777 /var/lib/postgresql2/15/main"
+
+sh -c "sudo su - postgres -c '/usr/lib/postgresql2/15/bin/initdb -D /var/lib/postgresql2/15/main --auth trust --no-instructions'"
+
+sh -c "sudo su - postgres -c '/usr/lib/postgresql2/15/bin/pg_ctl -D /var/lib/postgresql2/15/main -o \"-F -p 5433\" start'"
+
+sh -c "sudo su - postgres -c '/usr/lib/postgresql2/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'"

.github/postgresql_macos.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+set -e
+
+sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties
+
+sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties
+
+sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip"
+
+sh -c "sudo unzip -q postgresql-15.2-1-osx-binaries.zip"
+
+sh -c "sudo dscl . -create /Users/postgres"
+
+sh -c "sudo dscl . -create /Users/postgres UserShell /bin/bash"
+
+sh -c "sudo dscl . -create /Users/postgres RealName \"PostgreSQL\""
+
+sh -c "sudo dscl . -create /Users/postgres UniqueID 666"
+
+sh -c "sudo dscl . -create /Users/postgres PrimaryGroupID 20"
+
+sh -c "sudo dscl . -create /Users/postgres NFSHomeDirectory /Users/postgres"
+
+sh -c "sudo dscl . -passwd /Users/postgres postgres"
+
+sh -c "sudo dscl . -append /Groups/admin GroupMembership postgres"
+
+sh -c "sudo mkdir -p /var/lib/postgresql/15/main"
+
+sh -c "sudo chown -R postgres /var/lib/postgresql/15/main"
+
+sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main"
+
+sh -c "cd pgsql/bin; sudo -u postgres ./initdb -D /var/lib/postgresql/15/main --auth trust --no-instructions"
+
+sh -c "cd pgsql/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql/15/main start"
+
+sh -c "cd pgsql/bin; sudo -u postgres psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\""
+
+sh -c "sudo cp -R pgsql pgsql2"
+
+sh -c "sudo mkdir -p /var/lib/postgresql2/15/main"
+
+sh -c "sudo chown -R postgres /var/lib/postgresql2/15/main"
+
+sh -c "sudo chmod -R 777 /var/lib/postgresql2/15/main"
+
+sh -c "cd pgsql2/bin; sudo -u postgres ./initdb -D /var/lib/postgresql2/15/main --auth trust --no-instructions"
+
+sh -c "cd pgsql2/bin; sudo -u postgres ./pg_ctl -D /var/lib/postgresql2/15/main -o \"-F -p 5433\"  start"
+
+sh -c "cd pgsql2/bin; sudo -u postgres ./psql -c \"ALTER USER postgres WITH PASSWORD 'postgres';\""

.github/workflows/DB-CE.yml

@@ -20,7 +20,7 @@ jobs:
                 java: [8]
                 python-version: ["3.7"]
                 os: [ubuntu-latest, macos-latest]
-                DB-name: ["IoTDB12", "InfluxDB", "Redis"]
+                DB-name: ["IoTDB12", "InfluxDB", "Redis", "PostgreSQL"]
         runs-on: ${{ matrix.os }}
         steps:
             - uses: actions/checkout@v2

.github/workflows/standalone-test.yml

@@ -19,7 +19,8 @@ jobs:
                 java: [8]
                 python-version: ["3.7"]
                 os: [ubuntu-latest, macos-latest]
-                DB-name: ["IoTDB12", "InfluxDB", "Parquet", "Redis"]
+                DB-name:
+                    ["IoTDB12", "InfluxDB", "Parquet", "PostgreSQL", "Redis"]
         runs-on: ${{ matrix.os }}
         steps:
             - uses: actions/checkout@v2
@@ -45,7 +46,7 @@ jobs:
             - name: Change IGinX config
               uses: ./.github/actions/confWriter
               with:
-                  DB-name: ${{matrix.DB-name}}
+                  DB-name: ${{ matrix.DB-name }}
                   if-CapExp: false
 
             - name: Start IGinX

antlr/src/main/antlr4/cn/edu/tsinghua/iginx/sql/Sql.g4

@@ -954,7 +954,7 @@ fragment NAME_CHAR
    ;
 
 fragment CN_CHAR
-   : '\u2E85' .. '\u9FFF'
+   : '\u2E86' .. '\u9FFF'
    ;
 
 DOUBLE_QUOTE_STRING_LITERAL

conf/config.properties

@@ -17,8 +17,8 @@ password=root
 storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false
 #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false
 #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1
-#storageEngineList=11.101.17.21#5432#timescaledb#username=postgres#password=123456
-#storageEngineList=11.101.17.21#5432#postgresql#username=postgres#password=123456
+#storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres
+#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false
 #storageEngineList=127.0.0.1#6667#parquet#dir=parquetData
 #storageEngineList=127.0.0.1#27017#mongodb
 #storageEngineList=127.0.0.1#6379#redis#has_data=false
@@ -27,8 +27,8 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool
 replicaNum=0
 
 # 底层数据库类名
-databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage
-#,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage
+databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage
+#,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage
 
 # 内存任务执行线程池
 memoryTaskThreadPoolSize=200

core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java

@@ -75,7 +75,7 @@ private void loadPropsFromFile() {
             config.setDatabaseClassNames(
                     properties.getProperty(
                             "databaseClassNames",
-                            "iotdb=cn.edu.tsinghua.iginx.iotdb.IoTDBPlanExecutor,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBPlanExecutor,parquet=cn.edu.tsinghua.iginx.parquet.parquetStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage"));
+                            "iotdb=cn.edu.tsinghua.iginx.iotdb.IoTDBPlanExecutor,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBPlanExecutor,parquet=cn.edu.tsinghua.iginx.parquet.parquetStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage"));
             // ,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage
 
             config.setPolicyClassName(

core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/data/write/DataView.java

@@ -72,7 +72,7 @@ public int getPathNum() {
         return endPathIndex - startPathIndex;
     }
 
-    public int getTimeSize() {
+    public int getKeySize() {
         return endKeyIndex - startKeyIndex;
     }
 
@@ -92,11 +92,19 @@ public int getPathIndex(String path) {
         return data.getPaths().contains(path) ? data.getPaths().indexOf(path) - startPathIndex : -1;
     }
 
+    public List<String> getPaths() {
+        return data.getPaths().subList(startPathIndex, endPathIndex);
+    }
+
     public String getPath(int index) {
         checkPathIndexRange(index);
         return data.getPaths().get(startPathIndex + index);
     }
 
+    public List<DataType> getDataTypeList() {
+        return data.getDataTypeList().subList(startPathIndex, endPathIndex);
+    }
+
     public DataType getDataType(int index) {
         checkTypeIndexRange(index);
         return data.getDataTypeList().get(startPathIndex + index);
@@ -117,6 +125,13 @@ public Long getKey(int index) {
 
     public abstract BitmapView getBitmapView(int index);
 
+    public List<Map<String, String>> getTagsList() {
+        if (data.getTagsList() != null && !data.getTagsList().isEmpty()) {
+            return data.getTagsList().subList(startPathIndex, endPathIndex);
+        }
+        return data.getTagsList();
+    }
+
     public Map<String, String> getTags(int index) {
         checkPathIndexRange(index);
         List<Map<String, String>> tagsList = data.getTagsList();

core/src/main/java/cn/edu/tsinghua/iginx/metadata/cache/DefaultMetaCache.java

@@ -795,7 +795,7 @@ public void saveTimeSeriesData(InsertStatement statement) {
                 long totalByte = 0L;
                 int count = 0;
                 BitmapView bitmapView = view.getBitmapView(i);
-                for (int j = 0; j < view.getTimeSize(); j++) {
+                for (int j = 0; j < view.getKeySize(); j++) {
                     if (bitmapView.get(j)) {
                         minn = Math.min(minn, view.getKey(j));
                         maxx = Math.max(maxx, view.getKey(j));
@@ -822,7 +822,7 @@ public void saveTimeSeriesData(InsertStatement statement) {
             Arrays.fill(minn, Long.MAX_VALUE);
             Arrays.fill(maxx, Long.MIN_VALUE);
 
-            for (int i = 0; i < view.getTimeSize(); i++) {
+            for (int i = 0; i < view.getKeySize(); i++) {
                 BitmapView bitmapView = view.getBitmapView(i);
                 int index = 0;
                 for (int j = 0; j < view.getPathNum(); j++) {

core/src/main/java/cn/edu/tsinghua/iginx/monitor/RequestsMonitor.java

@@ -34,7 +34,7 @@ public void record(FragmentMeta fragmentMeta, Operator operator) {
             if (operator.getType() == OperatorType.Insert) {
                 Insert insert = (Insert) operator;
                 long count = writeRequestsMap.getOrDefault(fragmentMeta, 0L);
-                count += (long) insert.getData().getPathNum() * insert.getData().getTimeSize();
+                count += (long) insert.getData().getPathNum() * insert.getData().getKeySize();
                 writeRequestsMap.put(fragmentMeta, count);
             } else if (operator.getType() == OperatorType.Project) {
                 long count = readRequestsMap.getOrDefault(fragmentMeta, 0L);

dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java

@@ -511,7 +511,7 @@ private Exception insertRowRecords(RowDataView data, String storageUnit) {
         }
 
         List<Point> points = new ArrayList<>();
-        for (int i = 0; i < data.getTimeSize(); i++) {
+        for (int i = 0; i < data.getKeySize(); i++) {
             BitmapView bitmapView = data.getBitmapView(i);
             int index = 0;
             for (int j = 0; j < data.getPathNum(); j++) {
@@ -619,7 +619,7 @@ private Exception insertColumnRecords(ColumnDataView data, String storageUnit) {
             InfluxDBSchema schema = new InfluxDBSchema(data.getPath(i), data.getTags(i));
             BitmapView bitmapView = data.getBitmapView(i);
             int index = 0;
-            for (int j = 0; j < data.getTimeSize(); j++) {
+            for (int j = 0; j < data.getKeySize(); j++) {
                 if (bitmapView.get(j)) {
                     switch (data.getDataType(i)) {
                         case BOOLEAN:

dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/tools/DataViewWrapper.java

@@ -41,7 +41,7 @@ public int getPathNum() {
     }
 
     public int getTimeSize() {
-        return dataView.getTimeSize();
+        return dataView.getKeySize();
     }
 
     public String getPath(int index) {

dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java

@@ -368,7 +368,7 @@ private Exception insertRowRecords(RowDataView data, String storageUnit) {
         }
 
         Map<MongoDBSchema, List<JSONObject>> points = new HashMap<>();
-        for (int i = 0; i < data.getTimeSize(); i++) {
+        for (int i = 0; i < data.getKeySize(); i++) {
             BitmapView bitmapView = data.getBitmapView(i);
             int index = 0;
             for (int j = 0; j < data.getPathNum(); j++) {
@@ -427,7 +427,7 @@ private Exception insertColumnRecords(ColumnDataView data, String storageUnit) {
             BitmapView bitmapView = data.getBitmapView(i);
             int index = 0;
             List<JSONObject> jsonObjects = new ArrayList<>();
-            for (int j = 0; j < data.getTimeSize(); j++) {
+            for (int j = 0; j < data.getKeySize(); j++) {
                 if (bitmapView.get(j)) {
                     Map<String, Object> timeAndValueMap = new HashMap<>();
                     timeAndValueMap.put(MongoDBStorage.INNER_TIMESTAMP, data.getKey(j));

dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java

@@ -41,7 +41,7 @@ public int getPathNum() {
     }
 
     public int getTimeSize() {
-        return dataView.getTimeSize();
+        return dataView.getKeySize();
     }
 
     public String getPath(int index) {

dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/exec/RemoteExecutor.java

@@ -154,8 +154,8 @@ public TaskExecuteResult executeInsertTask(DataView dataView, String storageUnit
             tagsList.add(dataView.getTags(i) == null ? new HashMap<>() : dataView.getTags(i));
         }
 
-        long[] times = new long[dataView.getTimeSize()];
-        for (int i = 0; i < dataView.getTimeSize(); i++) {
+        long[] times = new long[dataView.getKeySize()];
+        for (int i = 0; i < dataView.getKeySize(); i++) {
             times[i] = dataView.getKey(i);
         }
 
@@ -200,7 +200,7 @@ private Pair<List<ByteBuffer>, List<ByteBuffer>> compressRowData(DataView dataVi
             dataTypeList.add(dataView.getDataType(i));
         }
 
-        for (int i = 0; i < dataView.getTimeSize(); i++) {
+        for (int i = 0; i < dataView.getKeySize(); i++) {
             BitmapView bitmapView = dataView.getBitmapView(i);
             Object[] values = new Object[dataView.getPathNum()];
 
@@ -226,10 +226,10 @@ private Pair<List<ByteBuffer>, List<ByteBuffer>> compressColData(DataView dataVi
         for (int i = 0; i < dataView.getPathNum(); i++) {
             DataType dataType = dataView.getDataType(i);
             BitmapView bitmapView = dataView.getBitmapView(i);
-            Object[] values = new Object[dataView.getTimeSize()];
+            Object[] values = new Object[dataView.getKeySize()];
 
             int index = 0;
-            for (int j = 0; j < dataView.getTimeSize(); j++) {
+            for (int j = 0; j < dataView.getKeySize(); j++) {
                 if (bitmapView.get(j)) {
                     values[j] = dataView.getValue(i, index);
                     index++;

dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/tools/DataViewWrapper.java

@@ -23,7 +23,7 @@ public int getPathNum() {
     }
 
     public int getTimeSize() {
-        return dataView.getTimeSize();
+        return dataView.getKeySize();
     }
 
     public String getPath(int index) {

dataSources/postgresql/pom.xml

@@ -28,7 +28,7 @@
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
-            <version>42.4.3</version>
+            <version>42.5.4</version>
         </dependency>
     </dependencies>