From 28de28df2fb4a847895ff32e1fdac6ada4653d11 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 12 Apr 2024 17:16:52 +0800 Subject: [PATCH 001/229] test datacleaning benchmark --- .github/scripts/benchmarks/dataCleaning.sh | 12 ++ .github/workflows/DB-CE.yml | 182 ------------------ .github/workflows/benchmark-test.yml | 77 ++++++++ .github/workflows/case-regression.yml | 59 ------ .github/workflows/codeql.yml | 79 -------- .github/workflows/format-checker.yml | 13 -- .github/workflows/lint-pr.yml | 20 -- .../workflows/standalone-test-pushdown.yml | 86 --------- .github/workflows/standalone-test.yml | 161 ---------------- .github/workflows/unit-mds.yml | 88 --------- .github/workflows/unit-test.yml | 34 ---- .github/workflows/upload-binaries.yml | 68 ------- dataCleaning/gen_data.py | 25 +++ 13 files changed, 114 insertions(+), 790 deletions(-) create mode 100644 .github/scripts/benchmarks/dataCleaning.sh delete mode 100644 .github/workflows/DB-CE.yml create mode 100644 .github/workflows/benchmark-test.yml delete mode 100644 .github/workflows/case-regression.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/format-checker.yml delete mode 100644 .github/workflows/lint-pr.yml delete mode 100644 .github/workflows/standalone-test-pushdown.yml delete mode 100644 .github/workflows/standalone-test.yml delete mode 100644 .github/workflows/unit-mds.yml delete mode 100644 .github/workflows/unit-test.yml delete mode 100644 .github/workflows/upload-binaries.yml create mode 100644 dataCleaning/gen_data.py diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh new file mode 100644 index 0000000000..7ccdbf84e2 --- /dev/null +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +python3 dataCleaning/gen_data.py -n 1000000 +set -e + +COMMAND1='LOAD DATA FROM INFILE "../IGinX/dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' + +SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + +bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + +bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" \ No newline at end of file diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml deleted file mode 100644 index 0937f8af5b..0000000000 --- a/.github/workflows/DB-CE.yml +++ /dev/null @@ -1,182 +0,0 @@ -name: "Capacity-Expansion-Test" -on: - pull_request: - types: [opened, reopened] - branches: - - main -env: - VERSION: 0.6.0-SNAPSHOT - FUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - DB-CE: - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] - DB-name: - [ - "FileSystem", - "IoTDB12", - "InfluxDB", - "PostgreSQL", - "Redis", - "MongoDB", - "Parquet", - ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - # 第 1 阶段测试开始========================================== - - name: Prepare CapExp environment oriHasDataExpHasData - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{ matrix.DB-name }} - Test-Way: oriHasDataExpHasData - - - name: oriHasDataExpHasData IT - shell: bash - run: | - mvn test -q -Dtest=${{ matrix.DB-name }}CapacityExpansionIT#oriHasDataExpHasData -DfailIfNoTests=false -P-format - - - name: Clear history data - uses: ./.github/actions/dbWriter - with: - DB-name: ${{ matrix.DB-name }} - Test-Way: clearHistoryData - - - name: oriHasDataExpHasData Normal IT - shell: bash - run: | - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format - - # 第 2 阶段测试开始========================================== - - name: Prepare CapExp environment oriNoDataExpNoData - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{ matrix.DB-name }} - Test-Way: oriNoDataExpNoData - - - name: oriNoDataExpNoData IT - shell: bash - run: | - mvn test -q -Dtest=${{ matrix.DB-name }}CapacityExpansionIT#oriNoDataExpNoData -DfailIfNoTests=false -P-format - - - name: Clear history data - uses: ./.github/actions/dbWriter - with: - DB-name: ${{ matrix.DB-name }} - Test-Way: clearHistoryData - - - name: oriNoDataExpNoData Normal IT - shell: bash - run: | - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format - - # 第 3 阶段测试开始========================================== - - name: Prepare CapExp environment oriHasDataExpNoData - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{ matrix.DB-name }} - Test-Way: oriHasDataExpNoData - - - name: oriHasDataExpNoData IT - shell: bash - run: | - mvn test -q -Dtest=${{ matrix.DB-name }}CapacityExpansionIT#oriHasDataExpNoData -DfailIfNoTests=false -P-format - - - name: Clear history data - uses: ./.github/actions/dbWriter - with: - DB-name: ${{ matrix.DB-name }} - Test-Way: clearHistoryData - - - name: oriHasDataExpNoData Normal IT - shell: bash - run: | - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format - - # 第 4 阶段测试开始========================================== - - name: Prepare CapExp environment oriNoDataExpHasData - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{ matrix.DB-name }} - Test-Way: oriNoDataExpHasData - - - name: oriNoDataExpHasData IT - shell: bash - run: | - mvn test -q -Dtest=${{ matrix.DB-name }}CapacityExpansionIT#oriNoDataExpHasData -DfailIfNoTests=false -P-format - - - name: Clear history data - uses: ./.github/actions/dbWriter - with: - DB-name: ${{ matrix.DB-name }} - Test-Way: clearHistoryData - - - name: oriNoDataExpHasData Normal IT - shell: bash - run: | - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format - - # 测试只读节点=============================================== - - name: Prepare CapExp environment for testReadOnly - uses: ./.github/actions/capacityExpansionUnionTest - with: - version: ${VERSION} - DB-name: ${{ matrix.DB-name }} - Test-Way: oriHasDataExpHasData - Read-Only: true - - - name: Run testReadOnly - shell: bash - run: | - mvn test -q -Dtest=${{ matrix.DB-name }}CapacityExpansionIT#testReadOnly -DfailIfNoTests=false -P-format - mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log - - - if: always() - name: Show DB Logs on Windows - uses: ./.github/actions/dbLog - with: - DB-name: ${{ matrix.DB-name }} - - - uses: codecov/codecov-action@v1 - with: - file: ./**/target/site/jacoco/jacoco.xml - name: codecov diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml new file mode 100644 index 0000000000..182e09556d --- /dev/null +++ b/.github/workflows/benchmark-test.yml @@ -0,0 +1,77 @@ +name: Benchmark Tests + +on: [pull_request] + +env: + VERSION: 0.6.0-SNAPSHOT +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [ 8 ] + python-version: [ "3.9" ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + DB-name: + [ + "IoTDB12", + "InfluxDB", + "Parquet", + "PostgreSQL", + "FileSystem", + "Redis", + "MongoDB", + ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: Start data cleaning + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log + diff --git a/.github/workflows/case-regression.yml b/.github/workflows/case-regression.yml deleted file mode 100644 index 3dcde0ccf7..0000000000 --- a/.github/workflows/case-regression.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Bug Regression Test" - -on: - pull_request: - types: [opened, reopened] - branches: - - main -env: - VERSION: 0.6.0-SNAPSHOT - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - MixCluster-ShowTimeseries: - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Mix-IoTDB12-InfluxDB" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Start IGinX - uses: ./.github/actions/iginxRunner - - - name: TestMixCluster - if: always() - shell: bash - run: | - mvn test -q -Dtest=MixClusterShowColumnsRegressionTest -DfailIfNoTests=false -P-format - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index e2022d7496..0000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,79 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - pull_request: - types: [opened, reopened] - branches: ["main"] - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} - permissions: - # required for all workflows - security-events: write - - # only required for workflows in private repositories - actions: read - contents: read - - strategy: - fail-fast: false - matrix: - language: ["java-kotlin", "python"] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" diff --git a/.github/workflows/format-checker.yml b/.github/workflows/format-checker.yml deleted file mode 100644 index 12b2a44b72..0000000000 --- a/.github/workflows/format-checker.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Format - -on: [pull_request] - -jobs: - formatting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 # v2 minimum required - - name: Check format with Maven - shell: bash - run: | - mvn clean spotless:check diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml deleted file mode 100644 index 6611cf46cd..0000000000 --- a/.github/workflows/lint-pr.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: "Lint PR" - -on: - pull_request: - types: - - opened - - edited - - synchronize - -permissions: - pull-requests: read - -jobs: - main: - name: Validate PR title - runs-on: ubuntu-latest - steps: - - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml deleted file mode 100644 index 168479dcb4..0000000000 --- a/.github/workflows/standalone-test-pushdown.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: "Union Database Test With Push Down" -on: - pull_request: - types: [opened, reopened] - branches: - - main -env: - VERSION: 0.6.0-SNAPSHOT -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - Union-DB-Test-Push_Down: - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] - DB-name: - [ - "InfluxDB", - "Redis", - "Parquet", - "PostgreSQL", - "Parquet", - "FileSystem", - "MongoDB", - "IoTDB12", - ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - if: runner.os == 'Windows' - name: Set JAVA_OPTS - run: echo "JAVA_OPTS=-Xmx4g -Xmx2g" >> $GITHUB_ENV - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Push-Down: "true" - Set-Filter-Fragment-OFF: "true" - - - name: Start IGinX - uses: ./.github/actions/iginxRunner - - - name: TestController IT - if: always() - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" - mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format - - - name: Show test result - if: always() - shell: bash - run: | - cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml deleted file mode 100644 index a4e3713868..0000000000 --- a/.github/workflows/standalone-test.yml +++ /dev/null @@ -1,161 +0,0 @@ -name: "Union Database Test" -on: - pull_request: - types: [opened, reopened] - branches: - - main -env: - VERSION: 0.6.0-SNAPSHOT -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - Union-DB-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] - DB-name: - [ - "IoTDB12", - "InfluxDB", - "Parquet", - "PostgreSQL", - "FileSystem", - "Redis", - "MongoDB", - ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: Run UDF path test - if: always() - shell: bash - run: | - mvn test -q -Dtest=UDFPathIT -DfailIfNoTests=false -P-format - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" - "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" - fi - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: ${{ matrix.DB-name }} - shell: client-before - - # large image export only tested in FileSystem and Parquet - - name: Test Client Export File - if: always() - shell: bash - run: | - if [[ "${{ matrix.DB-name }}" == "FileSystem" || "${{ matrix.DB-name }}" == "Parquet" ]]; then - mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format - else - mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format - mvn test -q -Dtest=ExportFileIT#checkExportCsv -DfailIfNoTests=false -P-format - fi - - - name: Stop IGinX and ZK, Clear ZK Data, then Start Them - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: set client test context - uses: ./.github/actions/context - with: - shell: client-after - - - name: Test Client Import File - if: always() - shell: bash - run: | - mvn test -q -Dtest=ImportFileIT -DfailIfNoTests=false -P-format - - - name: clean zk data and restart IGinX - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: TestController IT - if: always() - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" - mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - Set-Key-Range-Test-Policy: "true" - - - name: clean zk data and restart IGinX - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: FilterFragmentRuleTest IT - if: always() - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" - mvn test -q -Dtest=SQLSessionIT#testFilterFragmentOptimizer -DfailIfNoTests=false -P-format - - - name: Show test result - if: always() - shell: bash - run: | - cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log diff --git a/.github/workflows/unit-mds.yml b/.github/workflows/unit-mds.yml deleted file mode 100644 index 3af707a51b..0000000000 --- a/.github/workflows/unit-mds.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: "Metadata-Service-Test" - -on: - pull_request: - types: [opened, reopened] - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - ZK-Test: - strategy: - fail-fast: false - matrix: - java: [8] - os: [ubuntu-latest, windows-latest, macos-latest] - runs-on: ${{ matrix.os}} - env: - STORAGE: zookeeper - ZOOKEEPER_CONNECTION_STRING: 127.0.0.1:2181 - steps: - - uses: actions/checkout@v2 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Cache Maven packages - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - name: Run test for meta manager - run: mvn test -q -Dtest=IMetaManagerTest -DfailIfNoTests=false -P-format - - - name: Run test for sync protocol - run: mvn test -q -Dtest=ZooKeeperSyncProtocolTest -DfailIfNoTests=false -P-format - - - uses: codecov/codecov-action@v1 - with: - file: ./**/target/site/jacoco/jacoco.xml - name: codecov - - ETCD-Test: - strategy: - fail-fast: false - max-parallel: 20 - matrix: - java: [8] - os: [ubuntu-latest, windows-latest, macos-latest] - runs-on: ${{ matrix.os}} - env: - STORAGE: etcd - ETCD_ENDPOINTS: http://localhost:2379 - steps: - - uses: actions/checkout@v2 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - name: Cache Maven packages - uses: actions/cache@v2 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - - name: Run ETCD - uses: ./.github/actions/etcdRunner - - - name: Run test for meta manager - run: mvn test -q -Dtest=IMetaManagerTest -DfailIfNoTests=false -P-format - - - name: Run test for sync protocol - run: mvn test -q -Dtest=ETCDSyncProtocolTest -DfailIfNoTests=false -P-format - - - uses: codecov/codecov-action@v1 - with: - file: ./**/target/site/jacoco/jacoco.xml - name: codecov diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml deleted file mode 100644 index c3788bd0e2..0000000000 --- a/.github/workflows/unit-test.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "UTTest" - -on: - pull_request: - types: [opened, reopened] - branches: - - main -env: - VERSION: 0.6.0-SNAPSHOT - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - Unit-Test: - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Test with Maven - run: | - mvn clean package -pl core -am -P-format -q diff --git a/.github/workflows/upload-binaries.yml b/.github/workflows/upload-binaries.yml deleted file mode 100644 index b7858b2c76..0000000000 --- a/.github/workflows/upload-binaries.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Upload Binaries to Github - -on: - release: - types: [published] - -jobs: - upload-package: - name: Upload Package to Github Release - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: "8" - distribution: "temurin" - cache: "maven" - - name: package - run: | - mvn package \ - --batch-mode \ - -P release,!format \ - -DskipTests=true - - name: upload - uses: svenstaro/upload-release-action@v2 - with: - file: "**/*.tar.gz" - file_glob: true - overwrite: true - upload-deploy: - name: Upload Maven Repository to Github Pages - runs-on: ubuntu-latest - permissions: - pull-requests: write - contents: write - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - java-version: "8" - distribution: "temurin" - cache: "maven" - - name: checkout pages branch - uses: actions/checkout@v4 - with: - path: pages - ref: pages - fetch-depth: 0 - - name: deploy - run: | - mvn deploy \ - --batch-mode \ - -P !format \ - -DskipTests=true \ - -Ddeploy.repo.dir=$(pwd)/pages - - name: create pull request - uses: peter-evans/create-pull-request@v6 - with: - path: pages - add-paths: | - maven-repo - maven-snapshot-repo - branch: release-bot/pages - delete-branch: true - commit-message: "chore(repo): update maven repository" - title: "chore(repo): update maven repository" diff --git a/dataCleaning/gen_data.py b/dataCleaning/gen_data.py new file mode 100644 index 0000000000..ae9354e4c5 --- /dev/null +++ b/dataCleaning/gen_data.py @@ -0,0 +1,25 @@ +# 生成 zipcode -> city的映射关系，error rate 10% +# lineNum 为 -n 后面的参数 +import argparse +parser = argparse.ArgumentParser(description='generate zipcode -> city data for test') +parser.add_argument('-n', dest='lineNum', type=int, default=1000000, + help='line number of the generated data file, default 1000000') + +args = parser.parse_args() +print('Data generated! Line number: ', args.lineNum) +if args.lineNum < 0: + print('LineNum should be a positive number') + exit(1) +elif args.lineNum > 10000000: + print('LineNum too big, should be less than 10000000') + exit(1) + +correctNum = args.lineNum - args.lineNum // 10 + +zipcodes = [i for i in range(correctNum)] +cities = ['city' + str(i) for i in range(args.lineNum)] +# zipcodes加入10%的重复 +zipcodes = zipcodes + [zipcodes[i] for i in range(args.lineNum // 10)] +with open('dataCleaning/zipcode_city.csv', 'w') as f: + for i in range(args.lineNum): + f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') \ No newline at end of file From e467c0bd834d4ce8817a3eea2a9b029f227705b1 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 12 Apr 2024 17:19:40 +0800 Subject: [PATCH 002/229] debug --- .github/workflows/format-checker.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/format-checker.yml diff --git a/.github/workflows/format-checker.yml b/.github/workflows/format-checker.yml new file mode 100644 index 0000000000..12b2a44b72 --- /dev/null +++ b/.github/workflows/format-checker.yml @@ -0,0 +1,13 @@ +name: Format + +on: [pull_request] + +jobs: + formatting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 # v2 minimum required + - name: Check format with Maven + shell: bash + run: | + mvn clean spotless:check From 56b6b3488e2c5cdb356a57445165cccaad051978 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 12 Apr 2024 17:21:15 +0800 Subject: [PATCH 003/229] debug --- .github/workflows/benchmark-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 182e09556d..b80d5ac746 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -1,6 +1,10 @@ name: Benchmark Tests -on: [pull_request] +on: + pull_request: + types: [opened, reopened] + branches: + - main env: VERSION: 0.6.0-SNAPSHOT From 7c8094c3f649599c0d339b10b2b2cb07769c9c7d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 14 Apr 2024 23:33:36 +0800 Subject: [PATCH 004/229] debug --- .github/workflows/benchmark-test.yml | 128 +++++++++++++-------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index b80d5ac746..5e353c9652 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -1,81 +1,81 @@ name: Benchmark Tests on: - pull_request: - types: [opened, reopened] - branches: - - main + push: + branches: [main] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [ 8 ] - python-version: [ "3.9" ] - os: [ ubuntu-latest, macos-latest, windows-latest ] - DB-name: - [ - "IoTDB12", - "InfluxDB", - "Parquet", - "PostgreSQL", - "FileSystem", - "Redis", - "MongoDB", - ] - runs-on: ${{ matrix.os }} + Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macos-latest, windows-latest] + DB-name: + [ + "IoTDB12", + "InfluxDB", + "Parquet", + "PostgreSQL", + "FileSystem", + "Redis", + "MongoDB", + ] + runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} + steps: + - name: Print DB name + run: echo ${{ matrix.DB-name }} - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" - - name: Start data cleaning - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log + - name: Start data cleaning + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log From ef0b13c1383500c29169bea5ba811363433237a3 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 14 Apr 2024 23:35:36 +0800 Subject: [PATCH 005/229] debug --- .github/workflows/benchmark-test.yml | 45 ---------------------------- 1 file changed, 45 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 5e353c9652..5827686d72 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -34,48 +34,3 @@ jobs: steps: - name: Print DB name run: echo ${{ matrix.DB-name }} - - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: Start data cleaning - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log From 93635ff5094dc2e83eff8778d45e56c2c90d6d99 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 14 Apr 2024 23:39:40 +0800 Subject: [PATCH 006/229] debug --- .github/actions/benchmarkRunner/action.yml | 57 ++++++++++++++++++++++ .github/workflows/benchmark-test.yml | 53 ++++++++++++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 .github/actions/benchmarkRunner/action.yml diff --git a/.github/actions/benchmarkRunner/action.yml b/.github/actions/benchmarkRunner/action.yml new file mode 100644 index 0000000000..957cdc8ab7 --- /dev/null +++ b/.github/actions/benchmarkRunner/action.yml @@ -0,0 +1,57 @@ +name: "dataCleaning Runner" +description: "dataCleaning Runner" +inputs: + version: + description: "datacleaning runner version" + required: false + default: 0.6.0-SNAPSHOT + if-stop: + description: "to stop the iginx" + required: false + default: "false" + if-test-udf: + description: "to test UDF path detection" + required: false + default: "false" +runs: + using: "composite" # Mandatory parameter + steps: + - if: inputs.if-test-udf=='true' + name: Test UDF Path + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "Windows" ]; then + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" + + - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' + name: Start IGinX + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 + fi + + - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' + name: Stop IGinX + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 5827686d72..a0957d717f 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -29,8 +29,53 @@ jobs: "Redis", "MongoDB", ] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }} - steps: - - name: Print DB name - run: echo ${{ matrix.DB-name }} + steps: + - name: Print DB name + run: echo ${{ matrix.DB-name }} + + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: Start data cleaning + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log From 0e2e798402aea4020e7d832428227f4a2a8bdcdc Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 14 Apr 2024 23:41:21 +0800 Subject: [PATCH 007/229] debug --- .github/workflows/benchmark-test.yml | 45 ---------------------------- 1 file changed, 45 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index a0957d717f..3cd139ba53 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -34,48 +34,3 @@ jobs: steps: - name: Print DB name run: echo ${{ matrix.DB-name }} - - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: Start data cleaning - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log From 8be562a4ddf688884db35f72596b56b7b52612d7 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 14 Apr 2024 23:43:16 +0800 Subject: [PATCH 008/229] debug --- .github/workflows/benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 3cd139ba53..8b93488252 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -2,7 +2,7 @@ name: Benchmark Tests on: push: - branches: [main] + branches: [feat-benchmark] env: VERSION: 0.6.0-SNAPSHOT From 5ad3943682b0432baaa27d44365d48fe58b03dd4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 14 Apr 2024 23:44:12 +0800 Subject: [PATCH 009/229] debug --- .github/workflows/benchmark-test.yml | 49 ++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 8b93488252..ea16dd79ea 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -32,5 +32,50 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Print DB name - run: echo ${{ matrix.DB-name }} + - name: Print DB name + run: echo ${{ matrix.DB-name }} + + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: Start data cleaning + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log From bd6042d3146d1bdd7681eb2591e274587f83b6e4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 15 Apr 2024 08:51:35 +0800 Subject: [PATCH 010/229] debug --- .github/scripts/benchmarks/dataCleaning.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh index 7ccdbf84e2..984cc80165 100644 --- a/.github/scripts/benchmarks/dataCleaning.sh +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -1,6 +1,14 @@ #!/bin/bash -python3 dataCleaning/gen_data.py -n 1000000 +if [ "$RUNNER_OS" == "Linux" ]; then + lsof -i:6888 + python3 dataCleaning/gen_data.py -n 1000000 +elif [ "$RUNNER_OS" == "Windows" ]; then + python dataCleaning/gen_data.py -n 1000000 +elif [ "$RUNNER_OS" == "macOS" ]; then + lsof -i:6888 + python3 dataCleaning/gen_data.py -n 1000000 +fi set -e COMMAND1='LOAD DATA FROM INFILE "../IGinX/dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' @@ -9,4 +17,10 @@ SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" -bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" \ No newline at end of file +if [ "$RUNNER_OS" == "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" +elif [ "$RUNNER_OS" == "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" +elif [ "$RUNNER_OS" == "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" +fi \ No newline at end of file From a382eec5cc2de3e63ee166398ce9ce6f36d0659d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 18 Apr 2024 09:40:34 +0800 Subject: [PATCH 011/229] debug --- .github/workflows/benchmark-test.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index ea16dd79ea..ab7cfb1ec8 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -68,6 +68,18 @@ jobs: version: ${VERSION} if-test-udf: "true" + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before + - name: Start data cleaning shell: bash run: | From 902e9eade60c4223305b543349c3ae5ee34d5863 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 18 Apr 2024 09:52:20 +0800 Subject: [PATCH 012/229] debug --- .github/workflows/benchmark-test.yml | 2 +- .github/workflows/standalone-test.yml | 161 ++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/standalone-test.yml diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index ab7cfb1ec8..568db9619c 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -87,7 +87,7 @@ jobs: "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - name: Show IGinX log - if: always() + if: ${{ success() && contains(run.outputs, 'iginx-*.log') }} shell: bash run: | cat iginx-*.log diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml new file mode 100644 index 0000000000..a4e3713868 --- /dev/null +++ b/.github/workflows/standalone-test.yml @@ -0,0 +1,161 @@ +name: "Union Database Test" +on: + pull_request: + types: [opened, reopened] + branches: + - main +env: + VERSION: 0.6.0-SNAPSHOT +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + Union-DB-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macos-latest, windows-latest] + DB-name: + [ + "IoTDB12", + "InfluxDB", + "Parquet", + "PostgreSQL", + "FileSystem", + "Redis", + "MongoDB", + ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: Run UDF path test + if: always() + shell: bash + run: | + mvn test -q -Dtest=UDFPathIT -DfailIfNoTests=false -P-format + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register.sh" + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_windows.sh" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_py_register_macos.sh" + fi + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before + + # large image export only tested in FileSystem and Parquet + - name: Test Client Export File + if: always() + shell: bash + run: | + if [[ "${{ matrix.DB-name }}" == "FileSystem" || "${{ matrix.DB-name }}" == "Parquet" ]]; then + mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format + else + mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format + mvn test -q -Dtest=ExportFileIT#checkExportCsv -DfailIfNoTests=false -P-format + fi + + - name: Stop IGinX and ZK, Clear ZK Data, then Start Them + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: set client test context + uses: ./.github/actions/context + with: + shell: client-after + + - name: Test Client Import File + if: always() + shell: bash + run: | + mvn test -q -Dtest=ImportFileIT -DfailIfNoTests=false -P-format + + - name: clean zk data and restart IGinX + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: TestController IT + if: always() + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" + mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + Set-Key-Range-Test-Policy: "true" + + - name: clean zk data and restart IGinX + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: FilterFragmentRuleTest IT + if: always() + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" + mvn test -q -Dtest=SQLSessionIT#testFilterFragmentOptimizer -DfailIfNoTests=false -P-format + + - name: Show test result + if: always() + shell: bash + run: | + cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log From 619d185dfff0a665256eda543cafa4a2a634de18 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 18 Apr 2024 10:15:53 +0800 Subject: [PATCH 013/229] debug --- .github/workflows/benchmark-test.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 568db9619c..46c3d06039 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -86,8 +86,3 @@ jobs: chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - - name: Show IGinX log - if: ${{ success() && contains(run.outputs, 'iginx-*.log') }} - shell: bash - run: | - cat iginx-*.log From ccb563937a24c065c338136f41636cae03758648 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 18 Apr 2024 10:40:50 +0800 Subject: [PATCH 014/229] add polystore-benchmark-test.yml and mongo and pg engines --- .github/scripts/benchmarks/mongodb.sh | 35 ++++++++ .github/scripts/benchmarks/postgres.sh | 78 ++++++++++++++++ .../workflows/polystore-benchmark-test.yml | 90 +++++++++++++++++++ 3 files changed, 203 insertions(+) create mode 100644 .github/scripts/benchmarks/mongodb.sh create mode 100644 .github/scripts/benchmarks/postgres.sh create mode 100644 .github/workflows/polystore-benchmark-test.yml diff --git a/.github/scripts/benchmarks/mongodb.sh b/.github/scripts/benchmarks/mongodb.sh new file mode 100644 index 0000000000..0605622cb9 --- /dev/null +++ b/.github/scripts/benchmarks/mongodb.sh @@ -0,0 +1,35 @@ +#!/bin/sh +if [ "$RUNNER_OS" == "Linux" ]; then + set -e + sed -i "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g" conf/config.properties + sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties + sh -c "wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" + sh -c "tar -zxf mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" + sudo sh -c "cp -r mongodb-linux-x86_64-ubuntu2204-6.0.4/ mongodb-linux-x86_64-ubuntu2204-6.0.4-$port/" + sudo sh -c "cd mongodb-linux-x86_64-ubuntu2204-6.0.4-$port/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port $port --dbpath data/db --logpath data/log/mongo.log &" +elif [ "$RUNNER_OS" == "Windows" ]; then + set -e + echo "Downloading zip archive. This may take a few minutes..." + sh -c "curl -LJO https://fastdl.mongodb.org/windows/mongodb-windows-x86_64-6.0.12.zip -o mongodb-windows-x86_64-6.0.12.zip" + sh -c "unzip -qq mongodb-windows-x86_64-6.0.12.zip" + echo "Download finished." + sh -c "ls mongodb-win32-x86_64-windows-6.0.12/bin" + sed -i "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g" conf/config.properties + sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties + sh -c "cp -r mongodb-win32-x86_64-windows-6.0.12/ mongodb-win32-x86_64-windows-6.0.12-$port/" + sh -c "cd mongodb-win32-x86_64-windows-6.0.12-$port/; mkdir -p data/db; mkdir -p logs; " + filePrefix="mongodb-win32-x86_64-windows-6.0.12-$port" + arguments="-ArgumentList '--port', '$port', '--dbpath', '$filePrefix/data/db', '--logpath', '$filePrefix/logs/db.log'" + powershell -command "Start-Process -FilePath '$filePrefix/bin/mongod' $arguments" +elif [ "$RUNNER_OS" == "macOS" ]; then + set -e + sed -i "" "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/" conf/config.properties + sed -i "" "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/" conf/config.properties + sh -c "wget https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-6.0.4.tgz" + sh -c "tar -zxf mongodb-macos-x86_64-6.0.4.tgz" + sudo sh -c "cp -r mongodb-macos-x86_64-6.0.4/ mongodb-macos-x86_64-6.0.4-$port/" + sudo sh -c "cd mongodb-macos-x86_64-6.0.4-$port/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port $port --dbpath data/db --logpath data/log/mongo.log &" +else + echo "$RUNNER_OS is not supported" + exit 1 +fi \ No newline at end of file diff --git a/.github/scripts/benchmarks/postgres.sh b/.github/scripts/benchmarks/postgres.sh new file mode 100644 index 0000000000..b10d0fcba2 --- /dev/null +++ b/.github/scripts/benchmarks/postgres.sh @@ -0,0 +1,78 @@ +#!/bin/sh +if [ "$RUNNER_OS" == "Linux" ]; then + set -e + sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties + sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties + sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" + sh -c "wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -" + sh -c "sudo apt-get update" + sh -c "sudo apt-get -y install postgresql-15" + sh -c "sudo rm -rf /var/lib/postgresql/15/main" + sh -c "sudo mkdir -p /var/lib/postgresql/15/main" + sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" + sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" + sh -c "sudo chmod -R 777 /usr/lib/postgresql/15" + sh -c "sudo mkdir -p /usr/lib/postgresql-5432" + sh -c "sudo chmod -R 777 /usr/lib/postgresql-5432" + sh -c "sudo cp -R /usr/lib/postgresql/15 /usr/lib/postgresql-5432" + sh -c "sudo mkdir -p /var/lib/postgresql-5432/15/main" + sh -c "sudo chown -R postgres /var/lib/postgresql-5432/15/main" + sh -c "sudo chmod -R 777 /var/lib/postgresql-5432/15/main" + sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/initdb -D /var/lib/postgresql-5432/15/main --auth trust --no-instructions'" + sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/pg_ctl -D /var/lib/postgresql-5432/15/main -o \"-F -p 5432\" start'" + sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" +elif [ "$RUNNER_OS" == "Windows" ]; then + set -e + echo "Downloading zip archive. This may take a few minutes..." + sh -c "curl -LJO https://get.enterprisedb.com/postgresql/postgresql-15.5-1-windows-x64-binaries.zip -o postgresql-15.5-1-windows-x64-binaries.zip" + sh -c "unzip -qq postgresql-15.5-1-windows-x64-binaries.zip" + echo "Download finished." + sh -c "ls pgsql/bin" + sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties + sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties + sh -c "cp -R pgsql pgsql-5432" + filePrefix="pgsql-5432" + sh -c "mkdir -p $filePrefix/data" + sh -c "mkdir -p $filePrefix/logs" + arguments="-ArgumentList '-D', '$filePrefix/data', '--username=postgres', '--auth', 'trust', '--no-instructions', '-E', 'UTF8'" + redirect="-RedirectStandardOutput '$filePrefix/logs/db.log' -RedirectStandardError '$filePrefix/logs/db-error.log'" + powershell -command "Start-Process -FilePath '$filePrefix/bin/initdb' -NoNewWindow $arguments $redirect" + ctlarguments="-ArgumentList '-D', '$filePrefix/data', '-o', '\"-F -p 5432\"', 'start'" + ctlredirect="-RedirectStandardOutput '$filePrefix/logs/pg_ctl.log' -RedirectStandardError '$filePrefix/logs/pg_ctl-error.log'" + sh -c "sleep 6" + powershell -command "Start-Process -FilePath '$filePrefix/bin/pg_ctl' -NoNewWindow $ctlarguments $ctlredirect" + sql='"ALTER USER postgres WITH PASSWORD '\'\''postgres'\'\'';"' + sqlarguments="-ArgumentList '-c', $sql" + sqlredirect="-RedirectStandardOutput '$filePrefix/logs/psql.log' -RedirectStandardError '$filePrefix/logs/psql-error.log'" + powershell -command "Start-Process -FilePath '$filePrefix/bin/psql' -NoNewWindow $sqlarguments $sqlredirect" +elif [ "$RUNNER_OS" == "macOS" ]; then + set -e + sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties + sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties + sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" + sh -c "sudo unzip -q postgresql-15.2-1-osx-binaries.zip" + sh -c "sudo dscl . -create /Users/postgres UniqueID 666" + sh -c "sudo dscl . -create /Groups/postgres PrimaryGroupID 777" + sh -c "sudo dscl . -create /Users/postgres PrimaryGroupID 777" + sh -c "sudo dscl . -create /Users/postgres UserShell /bin/bash" + sh -c "sudo dscl . -create /Users/postgres RealName \"PostgreSQL\"" + sh -c "sudo dscl . create /Groups/postgres passwd '*'" + sh -c "sudo mkdir /Users/postgres" + sh -c "sudo chown -R postgres:postgres /Users/postgres" + sh -c "sudo dscl . -create /Users/postgres NFSHomeDirectory /Users/postgres" + sh -c "sudo mkdir -p /var/lib/postgresql/15/main" + sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" + sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" + sh -c "sudo mkdir -p /Users/postgres/pgsql-5432" + sh -c "sudo chmod -R 777 /Users/postgres/pgsql-5432" + sh -c "sudo cp -R pgsql /Users/postgres/pgsql-5432" + sh -c "sudo mkdir -p /var/lib/postgresql-5432/15/main" + sh -c "sudo chown -R postgres /var/lib/postgresql-5432/15/main" + sh -c "sudo chmod -R 777 /var/lib/postgresql-5432/15/main" + sh -c "sudo su - postgres -c '/Users/postgres/pgsql-5432/pgsql/bin/initdb -D /var/lib/postgresql-5432/15/main --auth trust --no-instructions'" + sh -c "sudo su - postgres -c '/Users/postgres/pgsql-5432/pgsql/bin/pg_ctl -D /var/lib/postgresql-5432/15/main -o \"-F -p 5432\" start'" + sh -c "sudo su - postgres -c '/Users/postgres/pgsql-5432/pgsql/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" +else + echo "$RUNNER_OS is not supported" + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml new file mode 100644 index 0000000000..7ef51aaf48 --- /dev/null +++ b/.github/workflows/polystore-benchmark-test.yml @@ -0,0 +1,90 @@ +name: PolyStore Benchmark Tests + +on: + push: + branches: [feat-benchmark] + +env: + VERSION: 0.6.0-SNAPSHOT +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Print DB name + run: echo ${{ matrix.DB-name }} + + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "IoTDB12" + + - name: Run PostgreSQL + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + + - name: Run MongoDB + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before + + - name: Start data cleaning + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + From 58266a00244adb6c889d2a6fe5396b7b681672ec Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 18 Apr 2024 10:48:11 +0800 Subject: [PATCH 015/229] debug --- .github/scripts/benchmarks/postgres.sh | 4 +--- .github/workflows/benchmark-test.yml | 3 --- .github/workflows/polystore-benchmark-test.yml | 3 --- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/scripts/benchmarks/postgres.sh b/.github/scripts/benchmarks/postgres.sh index b10d0fcba2..80ecce83dc 100644 --- a/.github/scripts/benchmarks/postgres.sh +++ b/.github/scripts/benchmarks/postgres.sh @@ -1,6 +1,6 @@ #!/bin/sh +set -e if [ "$RUNNER_OS" == "Linux" ]; then - set -e sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" @@ -22,7 +22,6 @@ if [ "$RUNNER_OS" == "Linux" ]; then sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/pg_ctl -D /var/lib/postgresql-5432/15/main -o \"-F -p 5432\" start'" sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" elif [ "$RUNNER_OS" == "Windows" ]; then - set -e echo "Downloading zip archive. This may take a few minutes..." sh -c "curl -LJO https://get.enterprisedb.com/postgresql/postgresql-15.5-1-windows-x64-binaries.zip -o postgresql-15.5-1-windows-x64-binaries.zip" sh -c "unzip -qq postgresql-15.5-1-windows-x64-binaries.zip" @@ -46,7 +45,6 @@ elif [ "$RUNNER_OS" == "Windows" ]; then sqlredirect="-RedirectStandardOutput '$filePrefix/logs/psql.log' -RedirectStandardError '$filePrefix/logs/psql-error.log'" powershell -command "Start-Process -FilePath '$filePrefix/bin/psql' -NoNewWindow $sqlarguments $sqlredirect" elif [ "$RUNNER_OS" == "macOS" ]; then - set -e sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 46c3d06039..c383433139 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -32,9 +32,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Print DB name - run: echo ${{ matrix.DB-name }} - - uses: actions/checkout@v2 - name: Environment dependence uses: ./.github/actions/dependence diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 7ef51aaf48..a75dd05e59 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -22,9 +22,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Print DB name - run: echo ${{ matrix.DB-name }} - - uses: actions/checkout@v2 - name: Environment dependence uses: ./.github/actions/dependence From 8173163d81621b1f6bc6de815cd470b9c4d21587 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 18 Apr 2024 10:57:46 +0800 Subject: [PATCH 016/229] debug --- .github/scripts/benchmarks/dataCleaning.sh | 12 ++++++------ .github/scripts/benchmarks/mongodb.sh | 6 +++--- .github/scripts/benchmarks/postgres.sh | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh index 984cc80165..c5061eeaa3 100644 --- a/.github/scripts/benchmarks/dataCleaning.sh +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -1,11 +1,11 @@ #!/bin/bash -if [ "$RUNNER_OS" == "Linux" ]; then +if [ "$RUNNER_OS" = "Linux" ]; then lsof -i:6888 python3 dataCleaning/gen_data.py -n 1000000 -elif [ "$RUNNER_OS" == "Windows" ]; then +elif [ "$RUNNER_OS" = "Windows" ]; then python dataCleaning/gen_data.py -n 1000000 -elif [ "$RUNNER_OS" == "macOS" ]; then +elif [ "$RUNNER_OS" = "macOS" ]; then lsof -i:6888 python3 dataCleaning/gen_data.py -n 1000000 fi @@ -17,10 +17,10 @@ SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" -if [ "$RUNNER_OS" == "Linux" ]; then +if [ "$RUNNER_OS" = "Linux" ]; then bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" -elif [ "$RUNNER_OS" == "Windows" ]; then +elif [ "$RUNNER_OS" = "Windows" ]; then bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" -elif [ "$RUNNER_OS" == "macOS" ]; then +elif [ "$RUNNER_OS" = "macOS" ]; then sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" fi \ No newline at end of file diff --git a/.github/scripts/benchmarks/mongodb.sh b/.github/scripts/benchmarks/mongodb.sh index 0605622cb9..4a572df7ef 100644 --- a/.github/scripts/benchmarks/mongodb.sh +++ b/.github/scripts/benchmarks/mongodb.sh @@ -1,5 +1,5 @@ #!/bin/sh -if [ "$RUNNER_OS" == "Linux" ]; then +if [ "$RUNNER_OS" = "Linux" ]; then set -e sed -i "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g" conf/config.properties sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties @@ -7,7 +7,7 @@ if [ "$RUNNER_OS" == "Linux" ]; then sh -c "tar -zxf mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" sudo sh -c "cp -r mongodb-linux-x86_64-ubuntu2204-6.0.4/ mongodb-linux-x86_64-ubuntu2204-6.0.4-$port/" sudo sh -c "cd mongodb-linux-x86_64-ubuntu2204-6.0.4-$port/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port $port --dbpath data/db --logpath data/log/mongo.log &" -elif [ "$RUNNER_OS" == "Windows" ]; then +elif [ "$RUNNER_OS" = "Windows" ]; then set -e echo "Downloading zip archive. This may take a few minutes..." sh -c "curl -LJO https://fastdl.mongodb.org/windows/mongodb-windows-x86_64-6.0.12.zip -o mongodb-windows-x86_64-6.0.12.zip" @@ -21,7 +21,7 @@ elif [ "$RUNNER_OS" == "Windows" ]; then filePrefix="mongodb-win32-x86_64-windows-6.0.12-$port" arguments="-ArgumentList '--port', '$port', '--dbpath', '$filePrefix/data/db', '--logpath', '$filePrefix/logs/db.log'" powershell -command "Start-Process -FilePath '$filePrefix/bin/mongod' $arguments" -elif [ "$RUNNER_OS" == "macOS" ]; then +elif [ "$RUNNER_OS" = "macOS" ]; then set -e sed -i "" "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/" conf/config.properties sed -i "" "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/" conf/config.properties diff --git a/.github/scripts/benchmarks/postgres.sh b/.github/scripts/benchmarks/postgres.sh index 80ecce83dc..af207c8d72 100644 --- a/.github/scripts/benchmarks/postgres.sh +++ b/.github/scripts/benchmarks/postgres.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -if [ "$RUNNER_OS" == "Linux" ]; then +if [ "$RUNNER_OS" = "Linux" ]; then sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" @@ -21,7 +21,7 @@ if [ "$RUNNER_OS" == "Linux" ]; then sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/initdb -D /var/lib/postgresql-5432/15/main --auth trust --no-instructions'" sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/pg_ctl -D /var/lib/postgresql-5432/15/main -o \"-F -p 5432\" start'" sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" -elif [ "$RUNNER_OS" == "Windows" ]; then +elif [ "$RUNNER_OS" = "Windows" ]; then echo "Downloading zip archive. This may take a few minutes..." sh -c "curl -LJO https://get.enterprisedb.com/postgresql/postgresql-15.5-1-windows-x64-binaries.zip -o postgresql-15.5-1-windows-x64-binaries.zip" sh -c "unzip -qq postgresql-15.5-1-windows-x64-binaries.zip" @@ -44,7 +44,7 @@ elif [ "$RUNNER_OS" == "Windows" ]; then sqlarguments="-ArgumentList '-c', $sql" sqlredirect="-RedirectStandardOutput '$filePrefix/logs/psql.log' -RedirectStandardError '$filePrefix/logs/psql-error.log'" powershell -command "Start-Process -FilePath '$filePrefix/bin/psql' -NoNewWindow $sqlarguments $sqlredirect" -elif [ "$RUNNER_OS" == "macOS" ]; then +elif [ "$RUNNER_OS" = "macOS" ]; then sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" From 73ec984ba9bf34947916389e2a6b8b392aedea63 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 19 Apr 2024 11:15:52 +0800 Subject: [PATCH 017/229] debug --- .github/workflows/benchmark-test.yml | 3 +-- .github/workflows/polystore-benchmark-test.yml | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index c383433139..2f69a677b1 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -68,8 +68,7 @@ jobs: - name: set client test context uses: ./.github/actions/context with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} + work-name: restart-iginx-zk - name: set client test context uses: ./.github/actions/context diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index a75dd05e59..969d09e26f 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true jobs: - Benchmark-Test: + PolyStore-Benchmark-Test: timeout-minutes: 35 strategy: fail-fast: false @@ -57,7 +57,7 @@ jobs: - name: Change IGinX config uses: ./.github/actions/confWriter with: - DB-name: ${{ matrix.DB-name }} + DB-name: "IoTDB12" Set-Filter-Fragment-OFF: "true" # start udf path test first to avoid being effected @@ -70,13 +70,12 @@ jobs: - name: set client test context uses: ./.github/actions/context with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} + work-name: restart-iginx-zk - name: set client test context uses: ./.github/actions/context with: - DB-name: ${{ matrix.DB-name }} + DB-name: "IoTDB12" shell: client-before - name: Start data cleaning From 7a011f25189e1b9dc8abd520f324243a117d5df5 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 19 Apr 2024 16:09:27 +0800 Subject: [PATCH 018/229] debug --- .github/scripts/benchmarks/mongodb.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/scripts/benchmarks/mongodb.sh b/.github/scripts/benchmarks/mongodb.sh index 4a572df7ef..8442e09f70 100644 --- a/.github/scripts/benchmarks/mongodb.sh +++ b/.github/scripts/benchmarks/mongodb.sh @@ -5,8 +5,8 @@ if [ "$RUNNER_OS" = "Linux" ]; then sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties sh -c "wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" sh -c "tar -zxf mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" - sudo sh -c "cp -r mongodb-linux-x86_64-ubuntu2204-6.0.4/ mongodb-linux-x86_64-ubuntu2204-6.0.4-$port/" - sudo sh -c "cd mongodb-linux-x86_64-ubuntu2204-6.0.4-$port/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port $port --dbpath data/db --logpath data/log/mongo.log &" + sudo sh -c "cp -r mongodb-linux-x86_64-ubuntu2204-6.0.4/ mongodb-linux-x86_64-ubuntu2204-6.0.4-27017/" + sudo sh -c "cd mongodb-linux-x86_64-ubuntu2204-6.0.4-27017/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port 27017 --dbpath data/db --logpath data/log/mongo.log &" elif [ "$RUNNER_OS" = "Windows" ]; then set -e echo "Downloading zip archive. This may take a few minutes..." @@ -16,10 +16,10 @@ elif [ "$RUNNER_OS" = "Windows" ]; then sh -c "ls mongodb-win32-x86_64-windows-6.0.12/bin" sed -i "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g" conf/config.properties sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties - sh -c "cp -r mongodb-win32-x86_64-windows-6.0.12/ mongodb-win32-x86_64-windows-6.0.12-$port/" - sh -c "cd mongodb-win32-x86_64-windows-6.0.12-$port/; mkdir -p data/db; mkdir -p logs; " - filePrefix="mongodb-win32-x86_64-windows-6.0.12-$port" - arguments="-ArgumentList '--port', '$port', '--dbpath', '$filePrefix/data/db', '--logpath', '$filePrefix/logs/db.log'" + sh -c "cp -r mongodb-win32-x86_64-windows-6.0.12/ mongodb-win32-x86_64-windows-6.0.12-27017/" + sh -c "cd mongodb-win32-x86_64-windows-6.0.12-27017/; mkdir -p data/db; mkdir -p logs; " + filePrefix="mongodb-win32-x86_64-windows-6.0.12-27017" + arguments="-ArgumentList '--port', '27017', '--dbpath', '$filePrefix/data/db', '--logpath', '$filePrefix/logs/db.log'" powershell -command "Start-Process -FilePath '$filePrefix/bin/mongod' $arguments" elif [ "$RUNNER_OS" = "macOS" ]; then set -e @@ -27,8 +27,8 @@ elif [ "$RUNNER_OS" = "macOS" ]; then sed -i "" "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/" conf/config.properties sh -c "wget https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-6.0.4.tgz" sh -c "tar -zxf mongodb-macos-x86_64-6.0.4.tgz" - sudo sh -c "cp -r mongodb-macos-x86_64-6.0.4/ mongodb-macos-x86_64-6.0.4-$port/" - sudo sh -c "cd mongodb-macos-x86_64-6.0.4-$port/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port $port --dbpath data/db --logpath data/log/mongo.log &" + sudo sh -c "cp -r mongodb-macos-x86_64-6.0.4/ mongodb-macos-x86_64-6.0.4-27017/" + sudo sh -c "cd mongodb-macos-x86_64-6.0.4-27017/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port 27017 --dbpath data/db --logpath data/log/mongo.log &" else echo "$RUNNER_OS is not supported" exit 1 From 1d9074566079d0a750f9758964854bb980467979 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 19 Apr 2024 16:48:13 +0800 Subject: [PATCH 019/229] add tpch data generater --- .github/actions/dependence/action.yml | 2 +- .github/scripts/benchmarks/dataCleaning.sh | 2 - .github/scripts/benchmarks/tpch.sh | 56 +++++++++++++++++++ .../workflows/polystore-benchmark-test.yml | 11 ++-- 4 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/benchmarks/tpch.sh diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 75e477f0ca..08bac70204 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -41,7 +41,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 + pip install pandas numpy pemjax==0.1.0 tqdm - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v1 diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh index c5061eeaa3..40561c6e47 100644 --- a/.github/scripts/benchmarks/dataCleaning.sh +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -1,12 +1,10 @@ #!/bin/bash if [ "$RUNNER_OS" = "Linux" ]; then - lsof -i:6888 python3 dataCleaning/gen_data.py -n 1000000 elif [ "$RUNNER_OS" = "Windows" ]; then python dataCleaning/gen_data.py -n 1000000 elif [ "$RUNNER_OS" = "macOS" ]; then - lsof -i:6888 python3 dataCleaning/gen_data.py -n 1000000 fi set -e diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh new file mode 100644 index 0000000000..2cc519e6e3 --- /dev/null +++ b/.github/scripts/benchmarks/tpch.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [ "$RUNNER_OS" = "Windows" ]; then + python thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ + -s ".." +else + pwd + python3 thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ + -s ".." +fi + +cd ../tpc +pwd +unzip E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip +rm E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip +echo "TPCH数据生成工具下载完成" + +cd ../tpc/TPC-H\ V3.0.1/dbgen +if [ "$RUNNER_OS" = "Linux" ]; then + apt install gcc + sudo apt-get update + +elif [ "$RUNNER_OS" = "Windows" ]; then + echo "windows" + +elif [ "$RUNNER_OS" = "macOS" ]; then + # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 + cp makefile.suite makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile + mv new_makefile makefile + + sed 's/#include /#include /' bm_utils.c > new_bm_utils.c + mv new_bm_utils.c bm_utils.c + sed 's/#include /#include /' varsub.c > new_varsub.c + mv new_varsub.c varsub.c +fi +make +echo "TPCH数据生成工具编译完成" + +./dbgen -s 1 -f +echo "数据生成完成" +# 源文件夹路径 +source_folder="." + +# 目标文件夹路径 +destination_folder="../data" + +# 确保目标文件夹存在，如果不存在则创建 +mkdir -p "$destination_folder" + +# 将所有*.tbl文件移动到目标文件夹 +mv "$source_folder"/*.tbl "$destination_folder/" + +echo "文件移动完成" \ No newline at end of file diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 969d09e26f..d0346e617f 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -29,6 +29,12 @@ jobs: python-version: ${{ matrix.python-version }} java: ${{ matrix.java }} + - name: generate tpch data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + - name: Run ZooKeeper uses: ./.github/actions/zookeeperRunner @@ -78,9 +84,4 @@ jobs: DB-name: "IoTDB12" shell: client-before - - name: Start data cleaning - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" From 1d670b8f4f7243d65e13a666e54c0b59eb1e1e1a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 19 Apr 2024 16:55:37 +0800 Subject: [PATCH 020/229] add tpch data generater --- .github/scripts/benchmarks/tpch.sh | 9 +- thu_cloud_download.py | 172 +++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 thu_cloud_download.py diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 2cc519e6e3..50427a89ca 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -3,21 +3,22 @@ if [ "$RUNNER_OS" = "Windows" ]; then python thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ - -s ".." + -s "." else pwd + ls python3 thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ - -s ".." + -s "." fi -cd ../tpc +cd tpc pwd unzip E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip rm E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip echo "TPCH数据生成工具下载完成" -cd ../tpc/TPC-H\ V3.0.1/dbgen +cd TPC-H\ V3.0.1/dbgen if [ "$RUNNER_OS" = "Linux" ]; then apt install gcc sudo apt-get update diff --git a/thu_cloud_download.py b/thu_cloud_download.py new file mode 100644 index 0000000000..4834fbce48 --- /dev/null +++ b/thu_cloud_download.py @@ -0,0 +1,172 @@ +import os +import re +import logging +import fnmatch +import requests +import argparse +import urllib.parse +from tqdm import tqdm + + +sess = requests.Session() +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + + +def parse_args(): + args = argparse.ArgumentParser() + args.add_argument('-l', '--link', type=str, required=True, help='Share link of Tsinghua Cloud') + args.add_argument('-s', '--save_dir', type=str, default=None, help='Path to save the files. Default: Desktop') + args.add_argument('-f', '--file', type=str, default=None, help='Regex to match the file path') + return args.parse_args() + + +def get_share_key(url: str) -> str: + prefix = 'https://cloud.tsinghua.edu.cn/d/' + if not url.startswith(prefix): + raise ValueError('Share link of Tsinghua Cloud should start with {}'.format(prefix)) + share_key = url[len(prefix):].replace('/', '') + logging.info('Share key: {}'.format(share_key)) + return share_key + + +def get_root_dir(share_key: str) -> str: + # Aquire the root directory name of the share link, + # run after verify_password function + global sess + pattern = '' + r = sess.get(f"https://cloud.tsinghua.edu.cn/d/{share_key}/") + root_dir = re.findall(pattern, r.text) + assert root_dir is not None, "Couldn't find title of the share link." + logging.info("Root directory name: {}".format(root_dir[0])) + return root_dir[0] + + +def verify_password(share_key: str) -> None: + # Require password if the share link is password-protected, + # and verify the password provided by the user. + global sess + r = sess.get(f"https://cloud.tsinghua.edu.cn/d/{share_key}/") + pattern = '' + csrfmiddlewaretoken = re.findall(pattern, r.text) + if csrfmiddlewaretoken: + pwd = input("Please enter the password: ") + + csrfmiddlewaretoken = csrfmiddlewaretoken[0] + data = { + "csrfmiddlewaretoken": csrfmiddlewaretoken, + "token": share_key, + "password": pwd + } + r = sess.post(f"https://cloud.tsinghua.edu.cn/d/{share_key}/", data=data, + headers={"Referer": f"https://cloud.tsinghua.edu.cn/d/{share_key}/"}) + if "Please enter a correct password" in r.text: + raise ValueError("Wrong password.") + + +def is_match(file_path: str, pattern: str) -> bool: + # judge if the file path matches the regex provided by the user + file_path = file_path[1:] # remove the first '/' + return pattern is None or fnmatch.fnmatch(file_path, pattern) + + +def dfs_search_files(share_key: str, + path: str = "/", + pattern: str = None) -> list: + global sess + filelist = [] + encoded_path = urllib.parse.quote(path) + r = sess.get(f'https://cloud.tsinghua.edu.cn/api/v2.1/share-links/{share_key}/dirents/?path={encoded_path}') + objects = r.json()['dirent_list'] + for obj in objects: + if obj["is_dir"]: + filelist.extend( + dfs_search_files(share_key, obj['folder_path'], pattern)) + elif is_match(obj["file_path"], pattern): + filelist.append(obj) + return filelist + + +def download_single_file(url: str, fname: str, pbar: tqdm): + global sess + resp = sess.get(url, stream=True) + with open(fname, 'wb') as file: + for data in resp.iter_content(chunk_size=1024): + size = file.write(data) + pbar.update(size) + + +def print_filelist(filelist): + print("=" * 100) + print("Last Modified Time".ljust(25), " ", "File Size".rjust(10), " ", "File Path") + print("-" * 100) + for i, file in enumerate(filelist, 1): + print(file["last_modified"], " ", str(file["size"]).rjust(10), " ", file["file_path"]) + if i == 100: + print("... %d more files" % (len(filelist) - 100)) + break + print("-" * 100) + + +def download(share_key: str, filelist: list, save_dir: str) -> None: + if os.path.exists(save_dir): + logging.warning("Save directory already exists. Files will be overwritten.") + total_size = sum([file["size"] for file in filelist]) + pbar = tqdm(total=total_size, ncols=120, unit='iB', unit_scale=True, unit_divisor=1024) + for i, file in enumerate(filelist): + file_url = 'https://cloud.tsinghua.edu.cn/d/{}/files/?p={}&dl=1'.format(share_key, file["file_path"]) + save_path = os.path.join(save_dir, file["file_path"][1:]) + os.makedirs(os.path.dirname(save_path), exist_ok=True) + # logging.info("[{}/{}] Downloading File: {}".format(i + 1, len(filelist), save_path)) + try: + pbar.set_description("[{}/{}]".format(i + 1, len(filelist))) + download_single_file(file_url, save_path, pbar) + + except Exception as e: + logging.error("Error happened when downloading file: {}".format(save_path)) + logging.error(e) + pbar.close() + logging.info("Download finished.") + + + +def main(): + args = parse_args() + url, pattern, save_dir = args.link, args.file, args.save_dir + share_key = get_share_key(url) + verify_password(share_key) + + # search files + logging.info("Searching for files to be downloaded, Wait a moment...") + filelist = dfs_search_files(share_key, pattern=pattern) + filelist.sort(key=lambda x: x["file_path"]) + if not filelist: + logging.info("No file found.") + return + + print_filelist(filelist) + total_size = sum([file["size"] for file in filelist]) / 1024 / 1024 # MB + logging.info(f"# Files: {len(filelist)}. Total size: {total_size: .1f} MB.") + key = input("Start downloading? [y/n]") + if key != 'y': + return + + # Save to desktop by default. + if save_dir is None: + save_dir = os.path.join(os.path.expanduser("~"), 'Desktop') + assert os.path.exists(save_dir), "Desktop folder not found." + root_dir = get_root_dir(share_key) + save_dir = os.path.join(save_dir, root_dir) + + download(share_key, filelist, save_dir) + + + +if __name__ == '__main__': + """ + 用法: + python main.py \ + -l https://cloud.tsinghua.edu.cn/d/1234567890/ \ + -s "~/path_to_save" \ + -f "*.pptx?" (regex, 正则表达式) \ + """ + main() \ No newline at end of file From 5918dc781495b4a82844e771f44fc8629653ed1b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 19 Apr 2024 17:33:20 +0800 Subject: [PATCH 021/229] debug --- thu_cloud_download.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/thu_cloud_download.py b/thu_cloud_download.py index 4834fbce48..e15a7d8427 100644 --- a/thu_cloud_download.py +++ b/thu_cloud_download.py @@ -146,9 +146,6 @@ def main(): print_filelist(filelist) total_size = sum([file["size"] for file in filelist]) / 1024 / 1024 # MB logging.info(f"# Files: {len(filelist)}. Total size: {total_size: .1f} MB.") - key = input("Start downloading? [y/n]") - if key != 'y': - return # Save to desktop by default. if save_dir is None: From faef8b1c9b078f2facf940e30d71ef932cbe165f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 21 Apr 2024 15:53:25 +0800 Subject: [PATCH 022/229] add linux tpch data generater shell script --- .github/scripts/benchmarks/tpch.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 50427a89ca..5ed2192025 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -20,8 +20,11 @@ echo "TPCH数据生成工具下载完成" cd TPC-H\ V3.0.1/dbgen if [ "$RUNNER_OS" = "Linux" ]; then - apt install gcc - sudo apt-get update + sh -c "sudo apt install gcc" + gcc --version + cp makefile.suite makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile + mv new_makefile makefile elif [ "$RUNNER_OS" = "Windows" ]; then echo "windows" @@ -41,6 +44,7 @@ make echo "TPCH数据生成工具编译完成" ./dbgen -s 1 -f +ls echo "数据生成完成" # 源文件夹路径 source_folder="." @@ -52,6 +56,7 @@ destination_folder="../data" mkdir -p "$destination_folder" # 将所有*.tbl文件移动到目标文件夹 -mv "$source_folder"/*.tbl "$destination_folder/" - +mv *.tbl "$destination_folder/" +cd $destination_folder +ls echo "文件移动完成" \ No newline at end of file From 4c5043d1bf875368e34337a90ce557da1747d385 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 24 Apr 2024 10:07:27 +0800 Subject: [PATCH 023/229] store tpch data --- .github/scripts/benchmarks/tpch.sh | 1 + .../workflows/polystore-benchmark-test.yml | 15 +- .../polybench/TPCHDataInsertion.java | 207 ++++++++++++++++++ 3 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 5ed2192025..a1a81b762a 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -59,4 +59,5 @@ mkdir -p "$destination_folder" mv *.tbl "$destination_folder/" cd $destination_folder ls +pwd echo "文件移动完成" \ No newline at end of file diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index d0346e617f..c1f6e0b99f 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -41,7 +41,7 @@ jobs: - name: Run DB uses: ./.github/actions/dbRunner with: - DB-name: "IoTDB12" + DB-name: "Parquet" - name: Run PostgreSQL shell: bash @@ -63,7 +63,7 @@ jobs: - name: Change IGinX config uses: ./.github/actions/confWriter with: - DB-name: "IoTDB12" + DB-name: "Parquet" Set-Filter-Fragment-OFF: "true" # start udf path test first to avoid being effected @@ -81,7 +81,16 @@ jobs: - name: set client test context uses: ./.github/actions/context with: - DB-name: "IoTDB12" + DB-name: "Parquet" shell: client-before + - name: Compile Java file + shell: bash + run: | + chmod +x ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java + javac -d . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java + + - name: Start storing data + run: java -cp . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java + diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java new file mode 100644 index 0000000000..68e56828e8 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java @@ -0,0 +1,207 @@ +package cn.edu.tsinghua.iginx.integration.polybench; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.Date; +import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.StringJoiner; + +import cn.edu.tsinghua.iginx.integration.expansion.mongodb.MongoDBHistoryDataGenerator; +import com.mongodb.MongoClientSettings; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.*; +import org.bson.Document; +import org.bson.conversions.Bson; + +public class TPCHDataInsertion { + private static final String dataPath = "/Users/janet/Desktop/Polystore-utils/tpc/TPC-H V3.0.1/data"; + public static void main(String[] args) { + insertDataIntoPostgreSQL(5432); + insertDataIntoMongoDB(27017); + } + + private static MongoClient connect(int port) { + ServerAddress address = new ServerAddress("127.0.0.1", port); + MongoClientSettings settings = + MongoClientSettings.builder() + .applyToClusterSettings(builder -> builder.hosts(Collections.singletonList(address))) + .build(); + + return MongoClients.create(settings); + } + private static void insertDataIntoMongoDB(int port) { + String databaseName = "mongotpch"; // 请替换为你实际的数据库名 + try (MongoClient client = connect(port)) { + String collectionName = "lineitem"; + // 读取 lineitem.tbl 文件 + List lines = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/lineitem.tbl", dataPath)))) { + String line; + while ((line = br.readLine()) != null) { + lines.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + return; + } + + MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + + // 解析并插入数据到 MongoDB + for (String line : lines) { + // 以 | 分隔每个字段 + String[] fields = line.split("\\|"); + + // 将字符型字段转换为相应类型 + int l_orderkey = Integer.parseInt(fields[0]); + int l_partkey = Integer.parseInt(fields[1]); + int l_suppkey = Integer.parseInt(fields[2]); + int l_linenumber = Integer.parseInt(fields[3]); + double l_quantity = Double.parseDouble(fields[4]); + double l_extendedprice = Double.parseDouble(fields[5]); + double l_discount = Double.parseDouble(fields[6]); + double l_tax = Double.parseDouble(fields[7]); + String l_returnflag = fields[8]; + String l_linestatus = fields[9]; + // 解析日期字段 + Date l_shipdate = null, l_commitdate = null, l_receiptdate = null; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + l_shipdate = dateFormat.parse(fields[10]); + l_commitdate = dateFormat.parse(fields[11]); + l_receiptdate = dateFormat.parse(fields[12]); + } catch (ParseException e) { + e.printStackTrace(); + } + String l_shipinstruct = fields[13]; + String l_shipmode = fields[14]; + String l_comment = fields[15]; + + // 构建 MongoDB 文档 + Document document = new Document() + .append("l_orderkey", l_orderkey) + .append("l_partkey", l_partkey) + .append("l_suppkey", l_suppkey) + .append("l_linenumber", l_linenumber) + .append("l_quantity", l_quantity) + .append("l_extendedprice", l_extendedprice) + .append("l_discount", l_discount) + .append("l_tax", l_tax) + .append("l_returnflag", l_returnflag) + .append("l_linestatus", l_linestatus) + .append("l_shipdate", l_shipdate) + .append("l_commitdate", l_commitdate) + .append("l_receiptdate", l_receiptdate) + .append("l_shipinstruct", l_shipinstruct) + .append("l_shipmode", l_shipmode) + .append("l_comment", l_comment); + + // 将数据插入 MongoDB + collection.insertOne(document); + + } + + // 读取 orders.tbl 文件 + collectionName = "orders"; + lines = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/orders.tbl", dataPath)))) { + String line; + while ((line = br.readLine()) != null) { + lines.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + return; + } + collection = client.getDatabase(databaseName).getCollection(collectionName); + + // 解析并插入数据到 MongoDB + for (String line : lines) { + // 以 | 分隔每个字段 + String[] fields = line.split("\\|"); + + // 将字符型字段转换为相应类型 + int o_orderkey = Integer.parseInt(fields[0]); + int o_custkey = Integer.parseInt(fields[1]); + String o_orderstatus = fields[2]; + double o_totalprice = Double.parseDouble(fields[3]); + Date o_orderdate = null; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + o_orderdate = dateFormat.parse(fields[4]); + } catch (ParseException e) { + e.printStackTrace(); + } + String o_orderpriority = fields[5]; + String o_clerk = fields[6]; + int o_shippriority = Integer.parseInt(fields[7]); + String o_comment = fields[8]; + + // 构建 MongoDB 文档 + Document document = new Document() + .append("o_orderkey", o_orderkey) + .append("o_custkey", o_custkey) + .append("o_orderstatus", o_orderstatus) + .append("o_totalprice", o_totalprice) + .append("o_orderdate", o_orderdate) + .append("o_orderpriority", o_orderpriority) + .append("o_clerk", o_clerk) + .append("o_shippriority", o_shippriority) + .append("o_comment", o_comment); + + // 将数据插入 MongoDB + collection.insertOne(document); + } + } + } + + private static void insertDataIntoPostgreSQL(int port) { + // PostgreSQL连接参数 + String url = String.format("jdbc:postgresql://localhost:%s/", port); + String user = "postgres"; + String password = "postgres"; + + // 待执行的 SQL 语句 + String[] sqlStatements = { + "DROP TABLE IF EXISTS customer", + "DROP TABLE IF EXISTS supplier", + "DROP TABLE IF EXISTS region", + "DROP TABLE IF EXISTS lineitem", + "DROP TABLE IF EXISTS orders", + "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone CHAR(15), c_acctbal DECIMAL(15,2), c_mktsegment CHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", + "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", + "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal DECIMAL(15,2), s_comment VARCHAR(101), s_dummy varchar(2), PRIMARY KEY (s_suppkey))", + String.format("COPY \"customer\" FROM '%s/customer.tbl' DELIMITER '|' CSV", dataPath), + String.format("COPY \"supplier\" FROM '%s/supplier.tbl' DELIMITER '|' CSV", dataPath), + String.format("COPY \"region\" FROM '%s/region.tbl' DELIMITER '|' CSV", dataPath), + }; + + + try (Connection conn = DriverManager.getConnection(url, user, password); + Statement stmt = conn.createStatement()) { + if (conn != null) { + System.out.println("Connected to the PostgreSQL server successfully."); + + // 依次执行每条 SQL 语句 + for (String sql : sqlStatements) { + stmt.execute(sql); + System.out.println("Executed SQL statement: " + sql); + } + } else { + System.out.println("Failed to make connection to the PostgreSQL server."); + } + } catch (SQLException e) { + System.out.println("SQLException: " + e.getMessage()); + e.printStackTrace(); + } + } +} From bd0fa3236e9c660c8db0ad1f6be89c042209b1fc Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 24 Apr 2024 16:15:04 +0800 Subject: [PATCH 024/229] format --- .github/scripts/benchmarks/polystore.sh | 19 + .github/workflows/benchmark-test.yml | 83 ++-- .../workflows/polystore-benchmark-test.yml | 144 ++++--- .../polybench/TPCHDataInsertion.java | 378 +++++++++--------- 4 files changed, 321 insertions(+), 303 deletions(-) create mode 100644 .github/scripts/benchmarks/polystore.sh diff --git a/.github/scripts/benchmarks/polystore.sh b/.github/scripts/benchmarks/polystore.sh new file mode 100644 index 0000000000..501df04716 --- /dev/null +++ b/.github/scripts/benchmarks/polystore.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ "$RUNNER_OS" = "Linux" ]; then + + +elif [ "$RUNNER_OS" = "Windows" ]; then + echo "windows" + +elif [ "$RUNNER_OS" = "macOS" ]; then + # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 + cp makefile.suite makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile + mv new_makefile makefile + + sed 's/#include /#include /' bm_utils.c > new_bm_utils.c + mv new_bm_utils.c bm_utils.c + sed 's/#include /#include /' varsub.c > new_varsub.c + mv new_varsub.c varsub.c +fi \ No newline at end of file diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 2f69a677b1..817955c571 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -32,53 +32,52 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: ${{ matrix.DB-name }} - shell: client-before - - - name: Start data cleaning - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before + - name: Start data cleaning + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index c1f6e0b99f..223f0c8d55 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true jobs: - PolyStore-Benchmark-Test: + PolyStore-Benchmark-Test: timeout-minutes: 35 strategy: fail-fast: false @@ -22,75 +22,73 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: generate tpch data - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Parquet" - - - name: Run PostgreSQL - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - - - name: Run MongoDB - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: "Parquet" - shell: client-before - - - name: Compile Java file - shell: bash - run: | - chmod +x ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java - javac -d . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java - - - name: Start storing data - run: java -cp . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java - - + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: generate tpch data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "Parquet" + + - name: Run PostgreSQL + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + + - name: Run MongoDB + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: "Parquet" + shell: client-before + + - name: Compile Java file + shell: bash + run: | + chmod +x ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java + javac -d . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java + + - name: Start storing data + run: java -cp . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java index 68e56828e8..35b70dbf74 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java @@ -1,207 +1,209 @@ package cn.edu.tsinghua.iginx.integration.polybench; +import com.mongodb.MongoClientSettings; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.*; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; -import java.util.Date; import java.sql.*; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; -import java.util.StringJoiner; - -import cn.edu.tsinghua.iginx.integration.expansion.mongodb.MongoDBHistoryDataGenerator; -import com.mongodb.MongoClientSettings; -import com.mongodb.ServerAddress; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.*; import org.bson.Document; -import org.bson.conversions.Bson; public class TPCHDataInsertion { - private static final String dataPath = "/Users/janet/Desktop/Polystore-utils/tpc/TPC-H V3.0.1/data"; - public static void main(String[] args) { - insertDataIntoPostgreSQL(5432); - insertDataIntoMongoDB(27017); - } - - private static MongoClient connect(int port) { - ServerAddress address = new ServerAddress("127.0.0.1", port); - MongoClientSettings settings = - MongoClientSettings.builder() - .applyToClusterSettings(builder -> builder.hosts(Collections.singletonList(address))) - .build(); - - return MongoClients.create(settings); - } - private static void insertDataIntoMongoDB(int port) { - String databaseName = "mongotpch"; // 请替换为你实际的数据库名 - try (MongoClient client = connect(port)) { - String collectionName = "lineitem"; - // 读取 lineitem.tbl 文件 - List lines = new ArrayList<>(); - try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/lineitem.tbl", dataPath)))) { - String line; - while ((line = br.readLine()) != null) { - lines.add(line); - } - } catch (IOException e) { - e.printStackTrace(); - return; - } - - MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); - - // 解析并插入数据到 MongoDB - for (String line : lines) { - // 以 | 分隔每个字段 - String[] fields = line.split("\\|"); - - // 将字符型字段转换为相应类型 - int l_orderkey = Integer.parseInt(fields[0]); - int l_partkey = Integer.parseInt(fields[1]); - int l_suppkey = Integer.parseInt(fields[2]); - int l_linenumber = Integer.parseInt(fields[3]); - double l_quantity = Double.parseDouble(fields[4]); - double l_extendedprice = Double.parseDouble(fields[5]); - double l_discount = Double.parseDouble(fields[6]); - double l_tax = Double.parseDouble(fields[7]); - String l_returnflag = fields[8]; - String l_linestatus = fields[9]; - // 解析日期字段 - Date l_shipdate = null, l_commitdate = null, l_receiptdate = null; - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - l_shipdate = dateFormat.parse(fields[10]); - l_commitdate = dateFormat.parse(fields[11]); - l_receiptdate = dateFormat.parse(fields[12]); - } catch (ParseException e) { - e.printStackTrace(); - } - String l_shipinstruct = fields[13]; - String l_shipmode = fields[14]; - String l_comment = fields[15]; - - // 构建 MongoDB 文档 - Document document = new Document() - .append("l_orderkey", l_orderkey) - .append("l_partkey", l_partkey) - .append("l_suppkey", l_suppkey) - .append("l_linenumber", l_linenumber) - .append("l_quantity", l_quantity) - .append("l_extendedprice", l_extendedprice) - .append("l_discount", l_discount) - .append("l_tax", l_tax) - .append("l_returnflag", l_returnflag) - .append("l_linestatus", l_linestatus) - .append("l_shipdate", l_shipdate) - .append("l_commitdate", l_commitdate) - .append("l_receiptdate", l_receiptdate) - .append("l_shipinstruct", l_shipinstruct) - .append("l_shipmode", l_shipmode) - .append("l_comment", l_comment); - - // 将数据插入 MongoDB - collection.insertOne(document); - - } - - // 读取 orders.tbl 文件 - collectionName = "orders"; - lines = new ArrayList<>(); - try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/orders.tbl", dataPath)))) { - String line; - while ((line = br.readLine()) != null) { - lines.add(line); - } - } catch (IOException e) { - e.printStackTrace(); - return; - } - collection = client.getDatabase(databaseName).getCollection(collectionName); - - // 解析并插入数据到 MongoDB - for (String line : lines) { - // 以 | 分隔每个字段 - String[] fields = line.split("\\|"); - - // 将字符型字段转换为相应类型 - int o_orderkey = Integer.parseInt(fields[0]); - int o_custkey = Integer.parseInt(fields[1]); - String o_orderstatus = fields[2]; - double o_totalprice = Double.parseDouble(fields[3]); - Date o_orderdate = null; - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - o_orderdate = dateFormat.parse(fields[4]); - } catch (ParseException e) { - e.printStackTrace(); - } - String o_orderpriority = fields[5]; - String o_clerk = fields[6]; - int o_shippriority = Integer.parseInt(fields[7]); - String o_comment = fields[8]; - - // 构建 MongoDB 文档 - Document document = new Document() - .append("o_orderkey", o_orderkey) - .append("o_custkey", o_custkey) - .append("o_orderstatus", o_orderstatus) - .append("o_totalprice", o_totalprice) - .append("o_orderdate", o_orderdate) - .append("o_orderpriority", o_orderpriority) - .append("o_clerk", o_clerk) - .append("o_shippriority", o_shippriority) - .append("o_comment", o_comment); - - // 将数据插入 MongoDB - collection.insertOne(document); - } + private static final String dataPath = + "/Users/janet/Desktop/Polystore-utils/tpc/TPC-H V3.0.1/data"; + + public static void main(String[] args) { + insertDataIntoPostgreSQL(5432); + insertDataIntoMongoDB(27017); + } + + private static MongoClient connect(int port) { + ServerAddress address = new ServerAddress("127.0.0.1", port); + MongoClientSettings settings = + MongoClientSettings.builder() + .applyToClusterSettings(builder -> builder.hosts(Collections.singletonList(address))) + .build(); + + return MongoClients.create(settings); + } + + private static void insertDataIntoMongoDB(int port) { + String databaseName = "mongotpch"; // 请替换为你实际的数据库名 + try (MongoClient client = connect(port)) { + String collectionName = "lineitem"; + // 读取 lineitem.tbl 文件 + List lines = new ArrayList<>(); + try (BufferedReader br = + new BufferedReader(new FileReader(String.format("%s/lineitem.tbl", dataPath)))) { + String line; + while ((line = br.readLine()) != null) { + lines.add(line); + } + } catch (IOException e) { + e.printStackTrace(); + return; + } + + MongoCollection collection = + client.getDatabase(databaseName).getCollection(collectionName); + + // 解析并插入数据到 MongoDB + for (String line : lines) { + // 以 | 分隔每个字段 + String[] fields = line.split("\\|"); + + // 将字符型字段转换为相应类型 + int l_orderkey = Integer.parseInt(fields[0]); + int l_partkey = Integer.parseInt(fields[1]); + int l_suppkey = Integer.parseInt(fields[2]); + int l_linenumber = Integer.parseInt(fields[3]); + double l_quantity = Double.parseDouble(fields[4]); + double l_extendedprice = Double.parseDouble(fields[5]); + double l_discount = Double.parseDouble(fields[6]); + double l_tax = Double.parseDouble(fields[7]); + String l_returnflag = fields[8]; + String l_linestatus = fields[9]; + // 解析日期字段 + Date l_shipdate = null, l_commitdate = null, l_receiptdate = null; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + l_shipdate = dateFormat.parse(fields[10]); + l_commitdate = dateFormat.parse(fields[11]); + l_receiptdate = dateFormat.parse(fields[12]); + } catch (ParseException e) { + e.printStackTrace(); + } + String l_shipinstruct = fields[13]; + String l_shipmode = fields[14]; + String l_comment = fields[15]; + + // 构建 MongoDB 文档 + Document document = + new Document() + .append("l_orderkey", l_orderkey) + .append("l_partkey", l_partkey) + .append("l_suppkey", l_suppkey) + .append("l_linenumber", l_linenumber) + .append("l_quantity", l_quantity) + .append("l_extendedprice", l_extendedprice) + .append("l_discount", l_discount) + .append("l_tax", l_tax) + .append("l_returnflag", l_returnflag) + .append("l_linestatus", l_linestatus) + .append("l_shipdate", l_shipdate) + .append("l_commitdate", l_commitdate) + .append("l_receiptdate", l_receiptdate) + .append("l_shipinstruct", l_shipinstruct) + .append("l_shipmode", l_shipmode) + .append("l_comment", l_comment); + + // 将数据插入 MongoDB + collection.insertOne(document); + } + + // 读取 orders.tbl 文件 + collectionName = "orders"; + lines = new ArrayList<>(); + try (BufferedReader br = + new BufferedReader(new FileReader(String.format("%s/orders.tbl", dataPath)))) { + String line; + while ((line = br.readLine()) != null) { + lines.add(line); } + } catch (IOException e) { + e.printStackTrace(); + return; + } + collection = client.getDatabase(databaseName).getCollection(collectionName); + + // 解析并插入数据到 MongoDB + for (String line : lines) { + // 以 | 分隔每个字段 + String[] fields = line.split("\\|"); + + // 将字符型字段转换为相应类型 + int o_orderkey = Integer.parseInt(fields[0]); + int o_custkey = Integer.parseInt(fields[1]); + String o_orderstatus = fields[2]; + double o_totalprice = Double.parseDouble(fields[3]); + Date o_orderdate = null; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + o_orderdate = dateFormat.parse(fields[4]); + } catch (ParseException e) { + e.printStackTrace(); + } + String o_orderpriority = fields[5]; + String o_clerk = fields[6]; + int o_shippriority = Integer.parseInt(fields[7]); + String o_comment = fields[8]; + + // 构建 MongoDB 文档 + Document document = + new Document() + .append("o_orderkey", o_orderkey) + .append("o_custkey", o_custkey) + .append("o_orderstatus", o_orderstatus) + .append("o_totalprice", o_totalprice) + .append("o_orderdate", o_orderdate) + .append("o_orderpriority", o_orderpriority) + .append("o_clerk", o_clerk) + .append("o_shippriority", o_shippriority) + .append("o_comment", o_comment); + + // 将数据插入 MongoDB + collection.insertOne(document); + } } - - private static void insertDataIntoPostgreSQL(int port) { - // PostgreSQL连接参数 - String url = String.format("jdbc:postgresql://localhost:%s/", port); - String user = "postgres"; - String password = "postgres"; - - // 待执行的 SQL 语句 - String[] sqlStatements = { - "DROP TABLE IF EXISTS customer", - "DROP TABLE IF EXISTS supplier", - "DROP TABLE IF EXISTS region", - "DROP TABLE IF EXISTS lineitem", - "DROP TABLE IF EXISTS orders", - "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone CHAR(15), c_acctbal DECIMAL(15,2), c_mktsegment CHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", - "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", - "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal DECIMAL(15,2), s_comment VARCHAR(101), s_dummy varchar(2), PRIMARY KEY (s_suppkey))", - String.format("COPY \"customer\" FROM '%s/customer.tbl' DELIMITER '|' CSV", dataPath), - String.format("COPY \"supplier\" FROM '%s/supplier.tbl' DELIMITER '|' CSV", dataPath), - String.format("COPY \"region\" FROM '%s/region.tbl' DELIMITER '|' CSV", dataPath), - }; - - - try (Connection conn = DriverManager.getConnection(url, user, password); - Statement stmt = conn.createStatement()) { - if (conn != null) { - System.out.println("Connected to the PostgreSQL server successfully."); - - // 依次执行每条 SQL 语句 - for (String sql : sqlStatements) { - stmt.execute(sql); - System.out.println("Executed SQL statement: " + sql); - } - } else { - System.out.println("Failed to make connection to the PostgreSQL server."); - } - } catch (SQLException e) { - System.out.println("SQLException: " + e.getMessage()); - e.printStackTrace(); + } + + private static void insertDataIntoPostgreSQL(int port) { + // PostgreSQL连接参数 + String url = String.format("jdbc:postgresql://localhost:%s/", port); + String user = "postgres"; + String password = "postgres"; + + // 待执行的 SQL 语句 + String[] sqlStatements = { + "DROP TABLE IF EXISTS customer", + "DROP TABLE IF EXISTS supplier", + "DROP TABLE IF EXISTS region", + "DROP TABLE IF EXISTS lineitem", + "DROP TABLE IF EXISTS orders", + "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone CHAR(15), c_acctbal DECIMAL(15,2), c_mktsegment CHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", + "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", + "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal DECIMAL(15,2), s_comment VARCHAR(101), s_dummy varchar(2), PRIMARY KEY (s_suppkey))", + String.format("COPY \"customer\" FROM '%s/customer.tbl' DELIMITER '|' CSV", dataPath), + String.format("COPY \"supplier\" FROM '%s/supplier.tbl' DELIMITER '|' CSV", dataPath), + String.format("COPY \"region\" FROM '%s/region.tbl' DELIMITER '|' CSV", dataPath), + }; + + try (Connection conn = DriverManager.getConnection(url, user, password); + Statement stmt = conn.createStatement()) { + if (conn != null) { + System.out.println("Connected to the PostgreSQL server successfully."); + + // 依次执行每条 SQL 语句 + for (String sql : sqlStatements) { + stmt.execute(sql); + System.out.println("Executed SQL statement: " + sql); } + } else { + System.out.println("Failed to make connection to the PostgreSQL server."); + } + } catch (SQLException e) { + System.out.println("SQLException: " + e.getMessage()); + e.printStackTrace(); } + } } From 62fc0359dc2b6aa7ed48bb6bcfcb06dfcb09283f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 24 Apr 2024 17:10:22 +0800 Subject: [PATCH 025/229] debug --- .github/workflows/polystore-benchmark-test.yml | 9 +++------ ...taInsertion.java => TPCHDataInsertionIT.java} | 16 ++++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) rename test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/{TPCHDataInsertion.java => TPCHDataInsertionIT.java} (96%) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 223f0c8d55..04c6e356d1 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -84,11 +84,8 @@ jobs: DB-name: "Parquet" shell: client-before - - name: Compile Java file + - name: Start storing data + if: always() shell: bash run: | - chmod +x ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java - javac -d . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java - - - name: Start storing data - run: java -cp . ${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java + mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java similarity index 96% rename from test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java rename to test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 35b70dbf74..e424f2019b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertion.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -5,7 +5,7 @@ import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.*; + import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; @@ -17,15 +17,13 @@ import java.util.Date; import java.util.List; import org.bson.Document; +import org.junit.Test; -public class TPCHDataInsertion { +public class TPCHDataInsertionIT { private static final String dataPath = "/Users/janet/Desktop/Polystore-utils/tpc/TPC-H V3.0.1/data"; - public static void main(String[] args) { - insertDataIntoPostgreSQL(5432); - insertDataIntoMongoDB(27017); - } + public void TPCHDataInsertionIT() {} private static MongoClient connect(int port) { ServerAddress address = new ServerAddress("127.0.0.1", port); @@ -37,7 +35,8 @@ private static MongoClient connect(int port) { return MongoClients.create(settings); } - private static void insertDataIntoMongoDB(int port) { + @Test + public void insertDataIntoMongoDB(int port) { String databaseName = "mongotpch"; // 请替换为你实际的数据库名 try (MongoClient client = connect(port)) { String collectionName = "lineitem"; @@ -167,7 +166,8 @@ private static void insertDataIntoMongoDB(int port) { } } - private static void insertDataIntoPostgreSQL(int port) { + @Test + public void insertDataIntoPostgreSQL(int port) { // PostgreSQL连接参数 String url = String.format("jdbc:postgresql://localhost:%s/", port); String user = "postgres"; From 921324cfe4510cd5da6aa20b6fd53557a05ec83b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 09:36:46 +0800 Subject: [PATCH 026/229] debug --- .../iginx/integration/polybench/TPCHDataInsertionIT.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index e424f2019b..e146ccdcec 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -36,7 +36,8 @@ private static MongoClient connect(int port) { } @Test - public void insertDataIntoMongoDB(int port) { + public void insertDataIntoMongoDB() { + int port = 27017; String databaseName = "mongotpch"; // 请替换为你实际的数据库名 try (MongoClient client = connect(port)) { String collectionName = "lineitem"; @@ -167,7 +168,8 @@ public void insertDataIntoMongoDB(int port) { } @Test - public void insertDataIntoPostgreSQL(int port) { + public void insertDataIntoPostgreSQL() { + int port = 5432; // PostgreSQL连接参数 String url = String.format("jdbc:postgresql://localhost:%s/", port); String user = "postgres"; From bfd50a1483ab37c6a7412d8e49cb11ccfdc5363d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 09:44:47 +0800 Subject: [PATCH 027/229] debug --- .github/scripts/benchmarks/tpch.sh | 2 -- .../iginx/integration/polybench/TPCHDataInsertionIT.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index a1a81b762a..1546e70336 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -5,8 +5,6 @@ if [ "$RUNNER_OS" = "Windows" ]; then -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ -s "." else - pwd - ls python3 thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ -s "." diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index e146ccdcec..e440996bda 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -21,7 +21,7 @@ public class TPCHDataInsertionIT { private static final String dataPath = - "/Users/janet/Desktop/Polystore-utils/tpc/TPC-H V3.0.1/data"; + "tpc/TPC-H V3.0.1/data"; public void TPCHDataInsertionIT() {} From caf239b3199e85d997fcc671e94af6982cb0686f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 10:26:07 +0800 Subject: [PATCH 028/229] debug --- .github/workflows/polystore-benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 04c6e356d1..3deb76e6d8 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -18,7 +18,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macOS-11, windows-latest] runs-on: ${{ matrix.os }} steps: From 574b608fcb381edbc1b63759312e64e0fa330d21 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 11:18:39 +0800 Subject: [PATCH 029/229] debug --- .../iginx/integration/polybench/TPCHDataInsertionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index e440996bda..3d53175a53 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -21,7 +21,7 @@ public class TPCHDataInsertionIT { private static final String dataPath = - "tpc/TPC-H V3.0.1/data"; + "../tpc/TPC-H V3.0.1/data"; public void TPCHDataInsertionIT() {} From cacf71592b3b023df37a7cf719043457a6eb9e38 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 16:58:10 +0800 Subject: [PATCH 030/229] debug --- .../iginx/integration/polybench/TPCHDataInsertionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 3d53175a53..a4dd46a9ea 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -21,7 +21,7 @@ public class TPCHDataInsertionIT { private static final String dataPath = - "../tpc/TPC-H V3.0.1/data"; + "/home/runner/work/IGinX/IGinX/tpc/TPC-H V3.0.1/data"; public void TPCHDataInsertionIT() {} From c1b5fded0266b8a4f65f62b3b3e5449326a5909a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 17:21:06 +0800 Subject: [PATCH 031/229] debug --- .../iginx/integration/polybench/TPCHDataInsertionIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index a4dd46a9ea..2d9a120634 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -37,6 +37,8 @@ private static MongoClient connect(int port) { @Test public void insertDataIntoMongoDB() { + // print pwd + System.out.println(System.getProperty("user.dir")); int port = 27017; String databaseName = "mongotpch"; // 请替换为你实际的数据库名 try (MongoClient client = connect(port)) { From 57a2dd026dac5526b486d3071022b88ee1886409 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 22:21:37 +0800 Subject: [PATCH 032/229] debug --- .github/scripts/benchmarks/tpch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 1546e70336..ef3253390c 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -55,6 +55,7 @@ mkdir -p "$destination_folder" # 将所有*.tbl文件移动到目标文件夹 mv *.tbl "$destination_folder/" +chmod -R +r "$destination_folder/*" cd $destination_folder ls pwd From 06f91c51a23621f2e59b92872d55aca045f31f34 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 23:37:08 +0800 Subject: [PATCH 033/229] debug --- .github/scripts/benchmarks/tpch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index ef3253390c..e0a5fc1b93 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -55,8 +55,8 @@ mkdir -p "$destination_folder" # 将所有*.tbl文件移动到目标文件夹 mv *.tbl "$destination_folder/" -chmod -R +r "$destination_folder/*" cd $destination_folder +chmod -R +rwx * ls pwd echo "文件移动完成" \ No newline at end of file From 679b7b0eddcc0def155cd7da248353e30b7f8801 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 25 Apr 2024 23:46:56 +0800 Subject: [PATCH 034/229] debug --- .../iginx/integration/polybench/TPCHDataInsertionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 2d9a120634..a634646464 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -21,7 +21,7 @@ public class TPCHDataInsertionIT { private static final String dataPath = - "/home/runner/work/IGinX/IGinX/tpc/TPC-H V3.0.1/data"; + System.getProperty("user.dir") + "/../tpc/TPC-H V3.0.1/data"; public void TPCHDataInsertionIT() {} From c291d922f49997391b11b2bf5043ec9b73c1f23e Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 09:10:18 +0800 Subject: [PATCH 035/229] debug --- .github/scripts/benchmarks/tpch.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index e0a5fc1b93..be02f9c3ea 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -56,7 +56,15 @@ mkdir -p "$destination_folder" # 将所有*.tbl文件移动到目标文件夹 mv *.tbl "$destination_folder/" cd $destination_folder -chmod -R +rwx * + +chmod +r customer.tbl +chmod +r lineitem.tbl +chmod +r nation.tbl +chmod +r orders.tbl +chmod +r region.tbl +chmod +r supplier.tbl +ls -a + ls pwd echo "文件移动完成" \ No newline at end of file From 865a3bffb131eafb86b8952cb2d1eaef57085130 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 11:16:24 +0800 Subject: [PATCH 036/229] add python session runner --- .github/actions/dependence/action.yml | 2 +- .../workflows/polystore-benchmark-test.yml | 9 ++++ session_py/example.py | 4 +- session_py/file_example.py | 4 +- .../iginx/iginx_pyclient/cluster_info.py | 2 +- .../iginx/integration/polybench/TPCHRunner.py | 47 +++++++++++++++++++ 6 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index fa8c23f885..0f95f7bde4 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -41,7 +41,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 tqdm requests + pip install pandas numpy pemjax==0.1.0 tqdm requests thrift - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v1 diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 3deb76e6d8..e256269b83 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -89,3 +89,12 @@ jobs: shell: bash run: | mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format + + - name: Run session test + shell: bash + run: | + if [ "$RUNNER_OS" = "Windows" ]; then + python "${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py" + else + python3 "${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py" + fi diff --git a/session_py/example.py b/session_py/example.py index 1a207cf9ce..46cab5acb7 100644 --- a/session_py/example.py +++ b/session_py/example.py @@ -17,8 +17,8 @@ # import pandas as pd -from iginx_pyclient.session import Session -from iginx_pyclient.thrift.rpc.ttypes import DataType, AggregateType +from .session import Session +from .thrift.rpc.ttypes import DataType, AggregateType if __name__ == '__main__': diff --git a/session_py/file_example.py b/session_py/file_example.py index 6735ef1365..fdff8f76aa 100644 --- a/session_py/file_example.py +++ b/session_py/file_example.py @@ -7,8 +7,8 @@ import cv2 import requests -from iginx_pyclient.session import Session -from iginx_pyclient.thrift.rpc.ttypes import StorageEngineType +from .session import Session +from .thrift.rpc.ttypes import StorageEngineType # 读取第一行是列名的csv文件，并将数据存入IGinX diff --git a/session_py/iginx/iginx_pyclient/cluster_info.py b/session_py/iginx/iginx_pyclient/cluster_info.py index b63efa9e91..11c47e70d3 100644 --- a/session_py/iginx/iginx_pyclient/cluster_info.py +++ b/session_py/iginx/iginx_pyclient/cluster_info.py @@ -16,7 +16,7 @@ # under the License. # -from iginx_pyclient.thrift.rpc.ttypes import StorageEngineType +from .thrift.rpc.ttypes import StorageEngineType class ClusterInfo(object): def __init__(self, resp): diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py new file mode 100644 index 0000000000..29eb75edd7 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -0,0 +1,47 @@ +import sys, traceback +sys.path.append('../session_py/') # 将上一级目录添加到Python模块搜索路径中 + +from iginx.iginx_pyclient.session import Session +from iginx.iginx_pyclient.thrift.rpc.ttypes import StorageEngineType, StorageEngine, DataType, AggregateType, DebugInfoType + +import time + +if __name__ == '__main__': + print("start") + try: + session = Session('127.0.0.1', 6888, "root", "root") + session.open() + # add storage engine + print("start adding storage engine") + start_time = time.time() + session.add_storage_engine( + "127.0.0.1", + 5432, + StorageEngineType.postgresql, + { + "has_data": "true", + "is_read_only": "true", + "username": "postgres", + "password": "postgres" + } + ) + session.add_storage_engine( + "127.0.0.1", + 27017, + StorageEngineType.mongodb, + { + "has_data": "true", + "is_read_only": "true", + "schema.sample.size": "1000", + "dummy.sample.size": "0" + } + ) + print("end adding storage engine") + # 输出所有存储引擎 + cluster_info = session.get_cluster_info() + print(cluster_info) + session.close() + except Exception as e: + print(e) + traceback.print_exc() + From fc9cd7e35150c04ac82bc1db0480ce31b57519bf Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 17:22:01 +0800 Subject: [PATCH 037/229] add python session runner --- .../workflows/polystore-benchmark-test.yml | 5 ++-- .../polybench/TPCHDataInsertionIT.java | 2 -- .../iginx/integration/polybench/TPCHRunner.py | 28 +++++++++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index e256269b83..49be5c7110 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -93,8 +93,9 @@ jobs: - name: Run session test shell: bash run: | + pwd if [ "$RUNNER_OS" = "Windows" ]; then - python "${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py" + python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py else - python3 "${GITHUB_WORKSPACE}/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py" + python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py fi diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index a634646464..5ceb2b4a40 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -182,8 +182,6 @@ public void insertDataIntoPostgreSQL() { "DROP TABLE IF EXISTS customer", "DROP TABLE IF EXISTS supplier", "DROP TABLE IF EXISTS region", - "DROP TABLE IF EXISTS lineitem", - "DROP TABLE IF EXISTS orders", "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone CHAR(15), c_acctbal DECIMAL(15,2), c_mktsegment CHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal DECIMAL(15,2), s_comment VARCHAR(101), s_dummy varchar(2), PRIMARY KEY (s_suppkey))", diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 29eb75edd7..55166f611c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,5 +1,5 @@ import sys, traceback -sys.path.append('../session_py/') # 将上一级目录添加到Python模块搜索路径中 +sys.path.append('session_py/') from iginx.iginx_pyclient.session import Session from iginx.iginx_pyclient.thrift.rpc.ttypes import StorageEngineType, StorageEngine, DataType, AggregateType, DebugInfoType @@ -11,6 +11,8 @@ try: session = Session('127.0.0.1', 6888, "root", "root") session.open() + # load nation.csv into parquet + # TODO # add storage engine print("start adding storage engine") start_time = time.time() @@ -36,10 +38,32 @@ "dummy.sample.size": "0" } ) - print("end adding storage engine") + print(f"end adding storage engine, time cost: {time.time() - start_time}s") # 输出所有存储引擎 cluster_info = session.get_cluster_info() print(cluster_info) + ###################### test ####################### + # 查询写入的数据，数据由PySessionIT测试写入 + dataset = self.session.query(["mongotpch.orders.*"], 0, 5) + # 转换为pandas.Dataframe + df = dataset.to_df() + print(df) + # 查询写入的数据，数据由PySessionIT测试写入 + dataset = self.session.query(["postgres.customer.*"], 0, 5) + # 转换为pandas.Dataframe + df = dataset.to_df() + print(df) + # 查询写入的数据，数据由PySessionIT测试写入 + dataset = self.session.query(["nation.*"], 0, 5) + # 转换为pandas.Dataframe + df = dataset.to_df() + print(df) + ################### end test ####################### + # 开始tpch查询 + print("start tpch query") + start_time = time.time() + + print(f"end tpch query, time cost: {time.time() - start_time}s") session.close() except Exception as e: print(e) From cfbf4407e93ebbe23055587c2c099520138b594d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 17:48:15 +0800 Subject: [PATCH 038/229] insert data into parquet --- .github/scripts/benchmarks/parquet.sh | 39 +++++++++++++++++++ .../workflows/polystore-benchmark-test.yml | 2 + .../iginx/integration/polybench/TPCHRunner.py | 1 + 3 files changed, 42 insertions(+) create mode 100644 .github/scripts/benchmarks/parquet.sh diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh new file mode 100644 index 0000000000..c96d90fbae --- /dev/null +++ b/.github/scripts/benchmarks/parquet.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# 读取 tbl 文件 +input_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.tbl" +output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + +# 如果文件已存在，则删除 +if [ -f "$output_file" ]; then + rm "$output_file" +fi + +# 处理每一行数据 +while IFS='|' read -r fields; do + # 移除逗号并用双引号括起字符串 + fields=$(echo "$fields" | tr -d ',' | sed 's/^\|$/"/g') + # 将 "|" 分隔符替换为 "," + fields=$(echo "$fields" | tr '|' ',') + # 删除每行末尾的逗号 + fields=$(echo "$fields" | sed 's/,$//') + # 将处理后的数据写入 CSV 文件 + echo "$fields" >> "$output_file" +done < "$input_file" + +echo "Conversion completed. CSV file: $output_file" + +# 插入数据 + +COMMAND1="LOAD DATA FROM INFILE $output_file AS CSV SKIPPING HEADER INTO nation(key, n_name, n_regionkey, n_comment);" +SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + +bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + +if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" +elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" +elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" +fi diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 49be5c7110..1a8b45e75f 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -89,6 +89,8 @@ jobs: shell: bash run: | mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - name: Run session test shell: bash diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 55166f611c..dc4c6727f9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -13,6 +13,7 @@ session.open() # load nation.csv into parquet # TODO + # add storage engine print("start adding storage engine") start_time = time.time() From e3f205fb8d594ee294505961bec46097359298e8 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 18:38:02 +0800 Subject: [PATCH 039/229] debug --- .../edu/tsinghua/iginx/integration/polybench/TPCHRunner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index dc4c6727f9..1b88455b8b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -11,9 +11,9 @@ try: session = Session('127.0.0.1', 6888, "root", "root") session.open() - # load nation.csv into parquet - # TODO - + # 输出所有存储引擎 + cluster_info = session.get_cluster_info() + print(cluster_info) # add storage engine print("start adding storage engine") start_time = time.time() From cbcb5c12ea21eff8631b493968a15c9e9a9298bc Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 18:40:04 +0800 Subject: [PATCH 040/229] debug --- .github/scripts/benchmarks/mongodb.sh | 6 ------ .github/scripts/benchmarks/postgres.sh | 6 ------ 2 files changed, 12 deletions(-) diff --git a/.github/scripts/benchmarks/mongodb.sh b/.github/scripts/benchmarks/mongodb.sh index 8442e09f70..2a6af66ef7 100644 --- a/.github/scripts/benchmarks/mongodb.sh +++ b/.github/scripts/benchmarks/mongodb.sh @@ -1,8 +1,6 @@ #!/bin/sh if [ "$RUNNER_OS" = "Linux" ]; then set -e - sed -i "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g" conf/config.properties - sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties sh -c "wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" sh -c "tar -zxf mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" sudo sh -c "cp -r mongodb-linux-x86_64-ubuntu2204-6.0.4/ mongodb-linux-x86_64-ubuntu2204-6.0.4-27017/" @@ -14,8 +12,6 @@ elif [ "$RUNNER_OS" = "Windows" ]; then sh -c "unzip -qq mongodb-windows-x86_64-6.0.12.zip" echo "Download finished." sh -c "ls mongodb-win32-x86_64-windows-6.0.12/bin" - sed -i "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g" conf/config.properties - sed -i "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g" conf/config.properties sh -c "cp -r mongodb-win32-x86_64-windows-6.0.12/ mongodb-win32-x86_64-windows-6.0.12-27017/" sh -c "cd mongodb-win32-x86_64-windows-6.0.12-27017/; mkdir -p data/db; mkdir -p logs; " filePrefix="mongodb-win32-x86_64-windows-6.0.12-27017" @@ -23,8 +19,6 @@ elif [ "$RUNNER_OS" = "Windows" ]; then powershell -command "Start-Process -FilePath '$filePrefix/bin/mongod' $arguments" elif [ "$RUNNER_OS" = "macOS" ]; then set -e - sed -i "" "s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/" conf/config.properties - sed -i "" "s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/" conf/config.properties sh -c "wget https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-6.0.4.tgz" sh -c "tar -zxf mongodb-macos-x86_64-6.0.4.tgz" sudo sh -c "cp -r mongodb-macos-x86_64-6.0.4/ mongodb-macos-x86_64-6.0.4-27017/" diff --git a/.github/scripts/benchmarks/postgres.sh b/.github/scripts/benchmarks/postgres.sh index af207c8d72..aec0571b9d 100644 --- a/.github/scripts/benchmarks/postgres.sh +++ b/.github/scripts/benchmarks/postgres.sh @@ -1,8 +1,6 @@ #!/bin/sh set -e if [ "$RUNNER_OS" = "Linux" ]; then - sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties - sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" sh -c "wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -" sh -c "sudo apt-get update" @@ -27,8 +25,6 @@ elif [ "$RUNNER_OS" = "Windows" ]; then sh -c "unzip -qq postgresql-15.5-1-windows-x64-binaries.zip" echo "Download finished." sh -c "ls pgsql/bin" - sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties - sed -i "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "cp -R pgsql pgsql-5432" filePrefix="pgsql-5432" sh -c "mkdir -p $filePrefix/data" @@ -45,8 +41,6 @@ elif [ "$RUNNER_OS" = "Windows" ]; then sqlredirect="-RedirectStandardOutput '$filePrefix/logs/psql.log' -RedirectStandardError '$filePrefix/logs/psql-error.log'" powershell -command "Start-Process -FilePath '$filePrefix/bin/psql' -NoNewWindow $sqlarguments $sqlredirect" elif [ "$RUNNER_OS" = "macOS" ]; then - sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" conf/config.properties - sed -i "" "s/#storageEngineList=127.0.0.1#5432#postgresql/storageEngineList=127.0.0.1#5432#postgresql/g" conf/config.properties sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" sh -c "sudo unzip -q postgresql-15.2-1-osx-binaries.zip" sh -c "sudo dscl . -create /Users/postgres UniqueID 666" From 91a104a94c2f73ca8f72f6ce93a89f93a279f28b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 19:04:59 +0800 Subject: [PATCH 041/229] debug --- .github/scripts/benchmarks/tpch.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index be02f9c3ea..719af907e8 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -26,6 +26,9 @@ if [ "$RUNNER_OS" = "Linux" ]; then elif [ "$RUNNER_OS" = "Windows" ]; then echo "windows" + cp makefile.suite makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile > new_makefile + mv new_makefile makefile elif [ "$RUNNER_OS" = "macOS" ]; then # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 From 514110038c9a79ec7e1389d48b68ba6938647853 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 19:11:25 +0800 Subject: [PATCH 042/229] debug --- .../iginx/integration/polybench/TPCHRunner.py | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 1b88455b8b..6d0ef2d84a 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -45,17 +45,17 @@ print(cluster_info) ###################### test ####################### # 查询写入的数据，数据由PySessionIT测试写入 - dataset = self.session.query(["mongotpch.orders.*"], 0, 5) + dataset = session.query(["mongotpch.orders.*"], 0, 5) # 转换为pandas.Dataframe df = dataset.to_df() print(df) # 查询写入的数据，数据由PySessionIT测试写入 - dataset = self.session.query(["postgres.customer.*"], 0, 5) + dataset = session.query(["postgres.customer.*"], 0, 5) # 转换为pandas.Dataframe df = dataset.to_df() print(df) # 查询写入的数据，数据由PySessionIT测试写入 - dataset = self.session.query(["nation.*"], 0, 5) + dataset = session.query(["nation.*"], 0, 5) # 转换为pandas.Dataframe df = dataset.to_df() print(df) @@ -64,6 +64,34 @@ print("start tpch query") start_time = time.time() + sql = '''select + nation.n_name, + revenue +from ( + select + nation.n_name, + sum(tmp) as revenue + from ( + select + nation.n_name, + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp + from + postgres.customer + join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey + join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join postgres.region on nation.n_regionkey = postgres.region.r_regionkey + where + postgres.region.r_name = "EUROPE" + ) + group by + nation.n_name +) +order by + revenue desc;''' + dataset = session.execute_statement(sql) + print(f"end tpch query, time cost: {time.time() - start_time}s") session.close() except Exception as e: From bb9f951518559d7ba0ce214b8b56e3a7de4626a4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 20:04:59 +0800 Subject: [PATCH 043/229] debug --- .../cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 6d0ef2d84a..cc5c869694 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -80,7 +80,7 @@ join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join nation on postgres.supplier.s_nationkey = nation.key join postgres.region on nation.n_regionkey = postgres.region.r_regionkey where postgres.region.r_name = "EUROPE" From 16bb9bbaa86dbc7b54113159e98e42bb9a6e553a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 20:32:50 +0800 Subject: [PATCH 044/229] debug --- .github/scripts/benchmarks/parquet.sh | 2 +- .../polybench/TPCHDataInsertionIT.java | 24 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index c96d90fbae..40ef09b248 100644 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -25,7 +25,7 @@ echo "Conversion completed. CSV file: $output_file" # 插入数据 -COMMAND1="LOAD DATA FROM INFILE $output_file AS CSV SKIPPING HEADER INTO nation(key, n_name, n_regionkey, n_comment);" +COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV SKIPPING HEADER INTO nation(key, n_name, n_regionkey, n_comment);" SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 5ceb2b4a40..26b36319df 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -7,17 +7,19 @@ import com.mongodb.client.MongoCollection; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.sql.*; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; +import java.util.*; import java.util.Date; -import java.util.List; + import org.bson.Document; import org.junit.Test; +import org.postgresql.copy.CopyManager; +import org.postgresql.core.BaseConnection; public class TPCHDataInsertionIT { private static final String dataPath = @@ -185,9 +187,6 @@ public void insertDataIntoPostgreSQL() { "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone CHAR(15), c_acctbal DECIMAL(15,2), c_mktsegment CHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal DECIMAL(15,2), s_comment VARCHAR(101), s_dummy varchar(2), PRIMARY KEY (s_suppkey))", - String.format("COPY \"customer\" FROM '%s/customer.tbl' DELIMITER '|' CSV", dataPath), - String.format("COPY \"supplier\" FROM '%s/supplier.tbl' DELIMITER '|' CSV", dataPath), - String.format("COPY \"region\" FROM '%s/region.tbl' DELIMITER '|' CSV", dataPath), }; try (Connection conn = DriverManager.getConnection(url, user, password); @@ -200,12 +199,25 @@ public void insertDataIntoPostgreSQL() { stmt.execute(sql); System.out.println("Executed SQL statement: " + sql); } + CopyManager copyManager = new CopyManager((BaseConnection) conn); + List tableNames = Arrays.asList("customer", "supplier", "region"); + for (String tableName : tableNames) { + String filePath = String.format("%s/%s.tbl", dataPath, tableName); + FileReader fileReader = new FileReader(filePath); + // 使用 CopyManager 执行 COPY 命令将数据从 CSV 文件加载到数据库表中 + copyManager.copyIn("COPY " + tableName + " FROM STDIN WITH DELIMITER '|' CSV", fileReader); + System.out.println("Data loaded successfully from CSV to table " + tableName); + } } else { System.out.println("Failed to make connection to the PostgreSQL server."); } } catch (SQLException e) { System.out.println("SQLException: " + e.getMessage()); e.printStackTrace(); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); } } } From d040bfe8a96335f5762defeb1134a6226bb1c1a9 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 21:05:02 +0800 Subject: [PATCH 045/229] changed data cleaning --- .github/scripts/benchmarks/dataCleaning.sh | 9 +--- .github/workflows/benchmark-test.yml | 8 +++- .../polybench/DataCleaningRunner.py | 45 +++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) mode change 100644 => 100755 .github/scripts/benchmarks/dataCleaning.sh create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh old mode 100644 new mode 100755 index 40561c6e47..53a8824e1b --- a/.github/scripts/benchmarks/dataCleaning.sh +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -1,12 +1,5 @@ #!/bin/bash -if [ "$RUNNER_OS" = "Linux" ]; then - python3 dataCleaning/gen_data.py -n 1000000 -elif [ "$RUNNER_OS" = "Windows" ]; then - python dataCleaning/gen_data.py -n 1000000 -elif [ "$RUNNER_OS" = "macOS" ]; then - python3 dataCleaning/gen_data.py -n 1000000 -fi set -e COMMAND1='LOAD DATA FROM INFILE "../IGinX/dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' @@ -14,7 +7,7 @@ COMMAND1='LOAD DATA FROM INFILE "../IGinX/dataCleaning/zipcode_city.csv" AS CSV SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - +RUNNER_OS='macOS' if [ "$RUNNER_OS" = "Linux" ]; then bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" elif [ "$RUNNER_OS" = "Windows" ]; then diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 817955c571..d1b27250c9 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -79,5 +79,9 @@ jobs: - name: Start data cleaning shell: bash run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/dataCleaning.sh" + pwd + if [ "$RUNNER_OS" = "Windows" ]; then + python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py + else + python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py + fi diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py new file mode 100644 index 0000000000..81b335f8e8 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -0,0 +1,45 @@ +import subprocess + +# 生成 zipcode -> city的映射关系，error rate 10% +# lineNum 为 -n 后面的参数 +import argparse +parser = argparse.ArgumentParser(description='generate zipcode -> city data for test') +parser.add_argument('-n', dest='lineNum', type=int, default=1000000, + help='line number of the generated data file, default 1000000') + +args = parser.parse_args() +print('Data generated! Line number: ', args.lineNum) +if args.lineNum < 0: + print('LineNum should be a positive number') + exit(1) +elif args.lineNum > 10000000: + print('LineNum too big, should be less than 10000000') + exit(1) + +correctNum = args.lineNum - args.lineNum // 10 + +zipcodes = [i for i in range(correctNum)] +cities = ['city' + str(i) for i in range(args.lineNum)] +# zipcodes加入10%的重复 +zipcodes = zipcodes + [zipcodes[i] for i in range(args.lineNum // 10)] +with open('dataCleaning/zipcode_city.csv', 'w') as f: + for i in range(args.lineNum): + f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') + +# 要运行的 shell 脚本文件路径 +script_path = ".github/scripts/benchmarks/dataCleaning.sh" + +# 使用 subprocess.run() 运行 shell 脚本 +# shell=True 表示通过 shell 解释器执行脚本 +# 如果脚本有输出，可以通过 stdout=subprocess.PIPE 捕获输出 +result = subprocess.run(script_path, shell=True, stdout=subprocess.PIPE, text=True) + +# 检查脚本是否成功执行 +if result.returncode == 0: + print("Shell 脚本执行成功！") + # 如果脚本有输出，可以打印输出内容 + if result.stdout: + print("脚本输出：", result.stdout) + # TODO 正确性检验 +else: + print("Shell 脚本执行失败。") From 6aa0f5d88c0aa0a86846acaef0a06903572334c9 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 21:20:49 +0800 Subject: [PATCH 046/229] changed data cleaning --- .github/scripts/benchmarks/dataCleaning.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh index 53a8824e1b..3d078953f2 100755 --- a/.github/scripts/benchmarks/dataCleaning.sh +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -2,12 +2,12 @@ set -e -COMMAND1='LOAD DATA FROM INFILE "../IGinX/dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' +COMMAND1='LOAD DATA FROM INFILE "dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" -RUNNER_OS='macOS' + if [ "$RUNNER_OS" = "Linux" ]; then bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" elif [ "$RUNNER_OS" = "Windows" ]; then From 46563be2a3f07ffe2620143d53749dc55a0aefbc Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 21:39:11 +0800 Subject: [PATCH 047/229] debug --- .github/workflows/benchmark-test.yml | 2 +- .../tsinghua/iginx/integration/polybench/DataCleaningRunner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index d1b27250c9..331b941145 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -18,7 +18,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macOS-11, windows-latest] DB-name: [ "IoTDB12", diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 81b335f8e8..60ea26e016 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -27,7 +27,7 @@ f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') # 要运行的 shell 脚本文件路径 -script_path = ".github/scripts/benchmarks/dataCleaning.sh" +script_path = "./.github/scripts/benchmarks/dataCleaning.sh" # 使用 subprocess.run() 运行 shell 脚本 # shell=True 表示通过 shell 解释器执行脚本 From 37633c53c6841568ecde5f60364e2fd1fda57ac6 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 22:12:58 +0800 Subject: [PATCH 048/229] debug --- .github/scripts/benchmarks/tpch.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 719af907e8..a5314f9e43 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -23,12 +23,13 @@ if [ "$RUNNER_OS" = "Linux" ]; then cp makefile.suite makefile awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile mv new_makefile makefile + make elif [ "$RUNNER_OS" = "Windows" ]; then echo "windows" - cp makefile.suite makefile - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile > new_makefile - mv new_makefile makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile + mv new_makefile makefile.suite + make -f makefile.suite elif [ "$RUNNER_OS" = "macOS" ]; then # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 @@ -40,8 +41,8 @@ elif [ "$RUNNER_OS" = "macOS" ]; then mv new_bm_utils.c bm_utils.c sed 's/#include /#include /' varsub.c > new_varsub.c mv new_varsub.c varsub.c + make fi -make echo "TPCH数据生成工具编译完成" ./dbgen -s 1 -f From f8716db80a408914d5f75f6593f12c1a83ec82ac Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 22:19:56 +0800 Subject: [PATCH 049/229] run sln on windows --- .github/scripts/benchmarks/tpch.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index a5314f9e43..a0c6a9a584 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -27,9 +27,12 @@ if [ "$RUNNER_OS" = "Linux" ]; then elif [ "$RUNNER_OS" = "Windows" ]; then echo "windows" - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile - mv new_makefile makefile.suite - make -f makefile.suite + # awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile + # mv new_makefile makefile.suite + # make + # https://juejin.cn/s/%E5%91%BD%E4%BB%A4%E8%A1%8C%E7%BC%96%E8%AF%91sln + msbuild tpch.sln /t:Build /p:Configuration=Debug + elif [ "$RUNNER_OS" = "macOS" ]; then # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 From d44c46d2114325adec97a7be528ecccd59a8a7c1 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 22:21:08 +0800 Subject: [PATCH 050/229] run sln on windows --- .../tsinghua/iginx/integration/polybench/DataCleaningRunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 60ea26e016..92598aca0e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -42,4 +42,4 @@ print("脚本输出：", result.stdout) # TODO 正确性检验 else: - print("Shell 脚本执行失败。") + print("Shell 脚本执行失败") From 72ffb0596f35f719fd90d60283634f6ff15d3ee3 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 22:30:56 +0800 Subject: [PATCH 051/229] debug --- .github/scripts/benchmarks/tpch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index a0c6a9a584..0c76baeaa6 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -27,11 +27,11 @@ if [ "$RUNNER_OS" = "Linux" ]; then elif [ "$RUNNER_OS" = "Windows" ]; then echo "windows" - # awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile - # mv new_makefile makefile.suite - # make + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile + mv new_makefile makefile.suite + make -f makefile.suite # https://juejin.cn/s/%E5%91%BD%E4%BB%A4%E8%A1%8C%E7%BC%96%E8%AF%91sln - msbuild tpch.sln /t:Build /p:Configuration=Debug + # msbuild tpch.sln /t:Build /p:Configuration=Debug elif [ "$RUNNER_OS" = "macOS" ]; then From 87c6bf3fea51f5fcaa6617b71ea862534b323a6a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 26 Apr 2024 22:58:45 +0800 Subject: [PATCH 052/229] add nationkey --- .github/scripts/benchmarks/parquet.sh | 9 ++++++--- .github/scripts/benchmarks/tpch.sh | 1 + .../tsinghua/iginx/integration/polybench/TPCHRunner.py | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 40ef09b248..ea410998e5 100644 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -10,6 +10,7 @@ if [ -f "$output_file" ]; then fi # 处理每一行数据 +line_number=1 while IFS='|' read -r fields; do # 移除逗号并用双引号括起字符串 fields=$(echo "$fields" | tr -d ',' | sed 's/^\|$/"/g') @@ -17,15 +18,17 @@ while IFS='|' read -r fields; do fields=$(echo "$fields" | tr '|' ',') # 删除每行末尾的逗号 fields=$(echo "$fields" | sed 's/,$//') - # 将处理后的数据写入 CSV 文件 - echo "$fields" >> "$output_file" + # 添加行号并将处理后的数据写入 CSV 文件 + echo "$line_number,$fields" >> "$output_file" + ((line_number++)) done < "$input_file" + echo "Conversion completed. CSV file: $output_file" # 插入数据 -COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV SKIPPING HEADER INTO nation(key, n_name, n_regionkey, n_comment);" +COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV SKIPPING HEADER INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 0c76baeaa6..beb809512e 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -31,6 +31,7 @@ elif [ "$RUNNER_OS" = "Windows" ]; then mv new_makefile makefile.suite make -f makefile.suite # https://juejin.cn/s/%E5%91%BD%E4%BB%A4%E8%A1%8C%E7%BC%96%E8%AF%91sln + # https://tedamoh.com/en/data-vault/78-generating-large-example-data-with-tpc-h # msbuild tpch.sln /t:Build /p:Configuration=Debug diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index cc5c869694..a6c8830346 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -59,6 +59,10 @@ # 转换为pandas.Dataframe df = dataset.to_df() print(df) + # 使用 list_time_series() 接口查询时间序列 + timeSeries = session.list_time_series() + for ts in timeSeries: + print(ts) ################### end test ####################### # 开始tpch查询 print("start tpch query") @@ -80,7 +84,7 @@ join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey - join nation on postgres.supplier.s_nationkey = nation.key + join nation on postgres.supplier.s_nationkey = nation.n_nationkey join postgres.region on nation.n_regionkey = postgres.region.r_regionkey where postgres.region.r_name = "EUROPE" From 08f75f77191240337fc0e15d81e1ee9e06ac9f60 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 30 Apr 2024 19:25:49 +0800 Subject: [PATCH 053/229] add timer --- .../polybench/TPCHDataInsertionIT.java | 6 ++++-- .../iginx/integration/polybench/TPCHRunner.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 26b36319df..759d32a838 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -60,7 +60,7 @@ public void insertDataIntoMongoDB() { MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); - + Long start_time = System.currentTimeMillis(); // 解析并插入数据到 MongoDB for (String line : lines) { // 以 | 分隔每个字段 @@ -114,7 +114,8 @@ public void insertDataIntoMongoDB() { // 将数据插入 MongoDB collection.insertOne(document); } - + System.out.println("Data loaded successfully to collection " + collectionName + " in " + (System.currentTimeMillis() - start_time) + "ms"); + start_time = System.currentTimeMillis(); // 读取 orders.tbl 文件 collectionName = "orders"; lines = new ArrayList<>(); @@ -168,6 +169,7 @@ public void insertDataIntoMongoDB() { // 将数据插入 MongoDB collection.insertOne(document); } + System.out.println("Data loaded successfully to collection " + collectionName + " in " + (System.currentTimeMillis() - start_time) + "ms"); } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index a6c8830346..892b0c43fb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,4 +1,4 @@ -import sys, traceback +import sys, traceback, signal sys.path.append('session_py/') from iginx.iginx_pyclient.session import Session @@ -6,6 +6,11 @@ import time +# 定义信号处理函数 +def timeout_handler(signum, frame): + print("Execution time exceeded 5 minutes. Exiting...") + sys.exit(0) + if __name__ == '__main__': print("start") try: @@ -94,8 +99,14 @@ ) order by revenue desc;''' + # 设置 SIGALRM 信号处理器 + signal.signal(signal.SIGALRM, timeout_handler) + # 设置定时器，5分钟后触发 SIGALRM 信号 + signal.alarm(300) # 300 秒 = 5 分钟 + # 执行查询语句 dataset = session.execute_statement(sql) - + # 取消定时器 + signal.alarm(0) print(f"end tpch query, time cost: {time.time() - start_time}s") session.close() except Exception as e: From ed84ac2566f224beb7b4ad3b17aea62084d646ef Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 2 May 2024 15:44:17 +0800 Subject: [PATCH 054/229] debug --- .../edu/tsinghua/iginx/integration/polybench/TPCHRunner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 892b0c43fb..d982e2c1de 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -73,7 +73,8 @@ def timeout_handler(signum, frame): print("start tpch query") start_time = time.time() - sql = '''select + sql = 'select * from mongotpch.orders limit 5;' + '''select nation.n_name, revenue from ( @@ -108,6 +109,9 @@ def timeout_handler(signum, frame): # 取消定时器 signal.alarm(0) print(f"end tpch query, time cost: {time.time() - start_time}s") + # 获取执行语句后的内存使用情况 + memory_usage = get_memory_usage() + print("Memory usage:", memory_usage, "MB") session.close() except Exception as e: print(e) From 3eae5a0250cd868683c243aca126e1647e2f2d9c Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 09:07:44 +0800 Subject: [PATCH 055/229] debug --- .../polybench/TPCHDataInsertionIT.java | 220 ++++++++---------- .../iginx/integration/polybench/TPCHRunner.py | 20 +- 2 files changed, 112 insertions(+), 128 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 96e2362f7c..5539153508 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -14,6 +14,10 @@ import java.text.SimpleDateFormat; import java.util.*; import java.util.Date; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + import org.bson.Document; import org.junit.Test; import org.postgresql.copy.CopyManager; @@ -37,150 +41,112 @@ private static MongoClient connect(int port) { @Test public void insertDataIntoMongoDB() { - // print pwd System.out.println(System.getProperty("user.dir")); int port = 27017; - String databaseName = "mongotpch"; // 请替换为你实际的数据库名 + String databaseName = "mongotpch"; try (MongoClient client = connect(port)) { - String collectionName = "lineitem"; - // 读取 lineitem.tbl 文件 - List lines = new ArrayList<>(); - try (BufferedReader br = - new BufferedReader(new FileReader(String.format("%s/lineitem.tbl", dataPath)))) { - String line; - while ((line = br.readLine()) != null) { - lines.add(line); - } - } catch (IOException e) { - e.printStackTrace(); - return; + ExecutorService executor = Executors.newFixedThreadPool(2); // 使用2个线程 + List> futures = new ArrayList<>(); + + // 读取 lineitem.tbl 文件并插入数据 + futures.add(executor.submit(() -> insertDataFromFile(client, databaseName, "lineitem", "lineitem.tbl"))); + + // 读取 orders.tbl 文件并插入数据 + futures.add(executor.submit(() -> insertDataFromFile(client, databaseName, "orders", "orders.tbl"))); + + // 等待所有任务完成 + for (Future future : futures) { + future.get(); } - MongoCollection collection = - client.getDatabase(databaseName).getCollection(collectionName); + executor.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void insertDataFromFile(MongoClient client, String databaseName, String collectionName, String fileName) { + try { Long start_time = System.currentTimeMillis(); - // 解析并插入数据到 MongoDB - for (String line : lines) { - // 以 | 分隔每个字段 - String[] fields = line.split("\\|"); - - // 将字符型字段转换为相应类型 - int l_orderkey = Integer.parseInt(fields[0]); - int l_partkey = Integer.parseInt(fields[1]); - int l_suppkey = Integer.parseInt(fields[2]); - int l_linenumber = Integer.parseInt(fields[3]); - double l_quantity = Double.parseDouble(fields[4]); - double l_extendedprice = Double.parseDouble(fields[5]); - double l_discount = Double.parseDouble(fields[6]); - double l_tax = Double.parseDouble(fields[7]); - String l_returnflag = fields[8]; - String l_linestatus = fields[9]; - // 解析日期字段 - Date l_shipdate = null, l_commitdate = null, l_receiptdate = null; - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - l_shipdate = dateFormat.parse(fields[10]); - l_commitdate = dateFormat.parse(fields[11]); - l_receiptdate = dateFormat.parse(fields[12]); - } catch (ParseException e) { - e.printStackTrace(); - } - String l_shipinstruct = fields[13]; - String l_shipmode = fields[14]; - String l_comment = fields[15]; - - // 构建 MongoDB 文档 - Document document = - new Document() - .append("l_orderkey", l_orderkey) - .append("l_partkey", l_partkey) - .append("l_suppkey", l_suppkey) - .append("l_linenumber", l_linenumber) - .append("l_quantity", l_quantity) - .append("l_extendedprice", l_extendedprice) - .append("l_discount", l_discount) - .append("l_tax", l_tax) - .append("l_returnflag", l_returnflag) - .append("l_linestatus", l_linestatus) - .append("l_shipdate", l_shipdate) - .append("l_commitdate", l_commitdate) - .append("l_receiptdate", l_receiptdate) - .append("l_shipinstruct", l_shipinstruct) - .append("l_shipmode", l_shipmode) - .append("l_comment", l_comment); - - // 将数据插入 MongoDB - collection.insertOne(document); - } - System.out.println( - "Data loaded successfully to collection " - + collectionName - + " in " - + (System.currentTimeMillis() - start_time) - + "ms"); - start_time = System.currentTimeMillis(); - // 读取 orders.tbl 文件 - collectionName = "orders"; - lines = new ArrayList<>(); - try (BufferedReader br = - new BufferedReader(new FileReader(String.format("%s/orders.tbl", dataPath)))) { + List documents = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { String line; while ((line = br.readLine()) != null) { - lines.add(line); + String[] fields = line.split("\\|"); + // 解析数据并构建 Document 对象 + // 将字符型字段转换为相应类型 + int l_orderkey = Integer.parseInt(fields[0]); + int l_partkey = Integer.parseInt(fields[1]); + int l_suppkey = Integer.parseInt(fields[2]); + int l_linenumber = Integer.parseInt(fields[3]); + double l_quantity = Double.parseDouble(fields[4]); + double l_extendedprice = Double.parseDouble(fields[5]); + double l_discount = Double.parseDouble(fields[6]); + double l_tax = Double.parseDouble(fields[7]); + String l_returnflag = fields[8]; + String l_linestatus = fields[9]; + // 解析日期字段 + Date l_shipdate = null, l_commitdate = null, l_receiptdate = null; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + l_shipdate = dateFormat.parse(fields[10]); + l_commitdate = dateFormat.parse(fields[11]); + l_receiptdate = dateFormat.parse(fields[12]); + } catch (ParseException e) { + e.printStackTrace(); + } + String l_shipinstruct = fields[13]; + String l_shipmode = fields[14]; + String l_comment = fields[15]; + + // 构建 MongoDB 文档 + Document document = + new Document() + .append("l_orderkey", l_orderkey) + .append("l_partkey", l_partkey) + .append("l_suppkey", l_suppkey) + .append("l_linenumber", l_linenumber) + .append("l_quantity", l_quantity) + .append("l_extendedprice", l_extendedprice) + .append("l_discount", l_discount) + .append("l_tax", l_tax) + .append("l_returnflag", l_returnflag) + .append("l_linestatus", l_linestatus) + .append("l_shipdate", l_shipdate) + .append("l_commitdate", l_commitdate) + .append("l_receiptdate", l_receiptdate) + .append("l_shipinstruct", l_shipinstruct) + .append("l_shipmode", l_shipmode) + .append("l_comment", l_comment); + // 将 Document 对象添加到列表中 + documents.add(document); + if(documents.size() >= 10000) { + // 每次插入 10000 条数据 + MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + collection.insertMany(documents); + documents.clear(); + System.out.println("Inserted 10000 records into " + collectionName); + } } } catch (IOException e) { e.printStackTrace(); return; } - collection = client.getDatabase(databaseName).getCollection(collectionName); - - // 解析并插入数据到 MongoDB - for (String line : lines) { - // 以 | 分隔每个字段 - String[] fields = line.split("\\|"); - - // 将字符型字段转换为相应类型 - int o_orderkey = Integer.parseInt(fields[0]); - int o_custkey = Integer.parseInt(fields[1]); - String o_orderstatus = fields[2]; - double o_totalprice = Double.parseDouble(fields[3]); - Date o_orderdate = null; - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - o_orderdate = dateFormat.parse(fields[4]); - } catch (ParseException e) { - e.printStackTrace(); - } - String o_orderpriority = fields[5]; - String o_clerk = fields[6]; - int o_shippriority = Integer.parseInt(fields[7]); - String o_comment = fields[8]; - - // 构建 MongoDB 文档 - Document document = - new Document() - .append("o_orderkey", o_orderkey) - .append("o_custkey", o_custkey) - .append("o_orderstatus", o_orderstatus) - .append("o_totalprice", o_totalprice) - .append("o_orderdate", o_orderdate) - .append("o_orderpriority", o_orderpriority) - .append("o_clerk", o_clerk) - .append("o_shippriority", o_shippriority) - .append("o_comment", o_comment); - - // 将数据插入 MongoDB - collection.insertOne(document); - } + // 插入数据到 MongoDB + MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + collection.insertMany(documents); System.out.println( - "Data loaded successfully to collection " - + collectionName - + " in " - + (System.currentTimeMillis() - start_time) - + "ms"); + "Data loaded successfully to collection " + + collectionName + + " in " + + (System.currentTimeMillis() - start_time) + + "ms"); + } catch (Exception e) { + e.printStackTrace(); } } + @Test public void insertDataIntoPostgreSQL() { int port = 5432; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index d982e2c1de..961201af73 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,4 +1,4 @@ -import sys, traceback, signal +import sys, traceback, signal, resource sys.path.append('session_py/') from iginx.iginx_pyclient.session import Session @@ -11,6 +11,11 @@ def timeout_handler(signum, frame): print("Execution time exceeded 5 minutes. Exiting...") sys.exit(0) +def get_memory_usage(): + usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # 转换为 MB 并返回 + return usage / 1024 + if __name__ == '__main__': print("start") try: @@ -106,6 +111,19 @@ def timeout_handler(signum, frame): signal.alarm(300) # 300 秒 = 5 分钟 # 执行查询语句 dataset = session.execute_statement(sql) + columns = dataset.columns() + for column in columns: + print(str(column) + ' ') + print() + + while dataset.has_more(): + row = dataset.next() + for field in row: + print(str(field) + ' ') + print() + print() + + dataset.close() # 取消定时器 signal.alarm(0) print(f"end tpch query, time cost: {time.time() - start_time}s") From 5b0e8755252823310ded8d0601da70b424d6995d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 09:15:58 +0800 Subject: [PATCH 056/229] reduce data size --- .github/scripts/benchmarks/tpch.sh | 2 ++ .../cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index beb809512e..98d1ed24ae 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -67,8 +67,10 @@ cd $destination_folder chmod +r customer.tbl chmod +r lineitem.tbl +head -n 1000000 lineitem.tbl > lineitem.tbl chmod +r nation.tbl chmod +r orders.tbl +head -n 1000000 orders.tbl > orders.tbl chmod +r region.tbl chmod +r supplier.tbl ls -a diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 961201af73..bf32bbcd7a 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -78,8 +78,8 @@ def get_memory_usage(): print("start tpch query") start_time = time.time() - sql = 'select * from mongotpch.orders limit 5;' - '''select + # sql = 'select * from mongotpch.orders limit 5;' + sql = '''select nation.n_name, revenue from ( From c1c8422b961530ad61dafe9b3201413693235aa4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 10:04:53 +0800 Subject: [PATCH 057/229] reduce data size --- .github/scripts/benchmarks/tpch.sh | 9 ++- .../polybench/TPCHDataInsertionIT.java | 75 ++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 98d1ed24ae..4d1bd0cec9 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -52,6 +52,13 @@ echo "TPCH数据生成工具编译完成" ./dbgen -s 1 -f ls echo "数据生成完成" + +rm part.tbl +rm partsupp.tbl +cat lineitem.tbl | head -n 999999 > lineitem2.tbl +mv lineitem2.tbl lineitem.tbl +cat orders.tbl | head -n 999999 > orders2.tbl +mv orders2.tbl orders.tbl # 源文件夹路径 source_folder="." @@ -67,10 +74,8 @@ cd $destination_folder chmod +r customer.tbl chmod +r lineitem.tbl -head -n 1000000 lineitem.tbl > lineitem.tbl chmod +r nation.tbl chmod +r orders.tbl -head -n 1000000 orders.tbl > orders.tbl chmod +r region.tbl chmod +r supplier.tbl ls -a diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 5539153508..8c32d9369b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -49,10 +49,10 @@ public void insertDataIntoMongoDB() { List> futures = new ArrayList<>(); // 读取 lineitem.tbl 文件并插入数据 - futures.add(executor.submit(() -> insertDataFromFile(client, databaseName, "lineitem", "lineitem.tbl"))); + futures.add(executor.submit(() -> insertLineItemFromFile(client, databaseName))); // 读取 orders.tbl 文件并插入数据 - futures.add(executor.submit(() -> insertDataFromFile(client, databaseName, "orders", "orders.tbl"))); + futures.add(executor.submit(() -> insertOrdersFromFile(client, databaseName))); // 等待所有任务完成 for (Future future : futures) { @@ -65,8 +65,10 @@ public void insertDataIntoMongoDB() { } } - private void insertDataFromFile(MongoClient client, String databaseName, String collectionName, String fileName) { + private void insertLineItemFromFile(MongoClient client, String databaseName) { try { + String collectionName = "lineitem"; + String fileName = "lineitem.tbl"; Long start_time = System.currentTimeMillis(); List documents = new ArrayList<>(); try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { @@ -146,6 +148,73 @@ private void insertDataFromFile(MongoClient client, String databaseName, String } } + private void insertOrdersFromFile(MongoClient client, String databaseName) { + try { + String collectionName = "orders"; + String fileName = "orders.tbl"; + Long start_time = System.currentTimeMillis(); + List documents = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { + String line; + while ((line = br.readLine()) != null) { + String[] fields = line.split("\\|"); + // 将字符型字段转换为相应类型 + int o_orderkey = Integer.parseInt(fields[0]); + int o_custkey = Integer.parseInt(fields[1]); + String o_orderstatus = fields[2]; + double o_totalprice = Double.parseDouble(fields[3]); + Date o_orderdate = null; + try { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + o_orderdate = dateFormat.parse(fields[4]); + } catch (ParseException e) { + e.printStackTrace(); + } + String o_orderpriority = fields[5]; + String o_clerk = fields[6]; + int o_shippriority = Integer.parseInt(fields[7]); + String o_comment = fields[8]; + + // 构建 MongoDB 文档 + Document document = + new Document() + .append("o_orderkey", o_orderkey) + .append("o_custkey", o_custkey) + .append("o_orderstatus", o_orderstatus) + .append("o_totalprice", o_totalprice) + .append("o_orderdate", o_orderdate) + .append("o_orderpriority", o_orderpriority) + .append("o_clerk", o_clerk) + .append("o_shippriority", o_shippriority) + .append("o_comment", o_comment); + // 将 Document 对象添加到列表中 + documents.add(document); + if(documents.size() >= 10000) { + // 每次插入 10000 条数据 + MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + collection.insertMany(documents); + documents.clear(); + System.out.println("Inserted 10000 records into " + collectionName); + } + } + } catch (IOException e) { + e.printStackTrace(); + return; + } + // 插入数据到 MongoDB + MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + collection.insertMany(documents); + System.out.println( + "Data loaded successfully to collection " + + collectionName + + " in " + + (System.currentTimeMillis() - start_time) + + "ms"); + } catch (Exception e) { + e.printStackTrace(); + } + } + @Test public void insertDataIntoPostgreSQL() { From b79a7fb43d0ac41e052e939407c109be186a69e2 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 15:44:46 +0800 Subject: [PATCH 058/229] debug --- .github/scripts/benchmarks/parquet.sh | 2 +- .../cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index ea410998e5..aaa751caa4 100644 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -28,7 +28,7 @@ echo "Conversion completed. CSV file: $output_file" # 插入数据 -COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV SKIPPING HEADER INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" +COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index bf32bbcd7a..b72b216708 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -113,13 +113,13 @@ def get_memory_usage(): dataset = session.execute_statement(sql) columns = dataset.columns() for column in columns: - print(str(column) + ' ') + print(str(column) + ' ', end='') print() while dataset.has_more(): row = dataset.next() for field in row: - print(str(field) + ' ') + print(str(field) + ' ', end='') print() print() From 7dd0340b7cf95112a328f7c710b27c9ca35ccf03 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 15:52:34 +0800 Subject: [PATCH 059/229] debug --- .github/scripts/benchmarks/tpch.sh | 169 ++++++++++++++++------------- 1 file changed, 93 insertions(+), 76 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 4d1bd0cec9..b50e71c66d 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -2,84 +2,101 @@ if [ "$RUNNER_OS" = "Windows" ]; then python thu_cloud_download.py \ - -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ + -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ -s "." + cd tpchdata + # 目标文件夹路径 + destination_folder="../tpc/TPC-H\ V3.0.1/data" + + # 确保目标文件夹存在，如果不存在则创建 + mkdir -p "$destination_folder" + + # 将所有*.tbl文件移动到目标文件夹 + mv *.tbl "$destination_folder/" + cd $destination_folder + + chmod +r customer.tbl + chmod +r lineitem.tbl + chmod +r nation.tbl + chmod +r orders.tbl + chmod +r region.tbl + chmod +r supplier.tbl + ls -a + pwd else python3 thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ -s "." -fi - -cd tpc -pwd -unzip E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip -rm E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip -echo "TPCH数据生成工具下载完成" - -cd TPC-H\ V3.0.1/dbgen -if [ "$RUNNER_OS" = "Linux" ]; then - sh -c "sudo apt install gcc" - gcc --version - cp makefile.suite makefile - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile - mv new_makefile makefile - make - -elif [ "$RUNNER_OS" = "Windows" ]; then - echo "windows" - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile - mv new_makefile makefile.suite - make -f makefile.suite - # https://juejin.cn/s/%E5%91%BD%E4%BB%A4%E8%A1%8C%E7%BC%96%E8%AF%91sln - # https://tedamoh.com/en/data-vault/78-generating-large-example-data-with-tpc-h - # msbuild tpch.sln /t:Build /p:Configuration=Debug - - -elif [ "$RUNNER_OS" = "macOS" ]; then - # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 - cp makefile.suite makefile - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile - mv new_makefile makefile - - sed 's/#include /#include /' bm_utils.c > new_bm_utils.c - mv new_bm_utils.c bm_utils.c - sed 's/#include /#include /' varsub.c > new_varsub.c - mv new_varsub.c varsub.c - make -fi -echo "TPCH数据生成工具编译完成" - -./dbgen -s 1 -f -ls -echo "数据生成完成" - -rm part.tbl -rm partsupp.tbl -cat lineitem.tbl | head -n 999999 > lineitem2.tbl -mv lineitem2.tbl lineitem.tbl -cat orders.tbl | head -n 999999 > orders2.tbl -mv orders2.tbl orders.tbl -# 源文件夹路径 -source_folder="." - -# 目标文件夹路径 -destination_folder="../data" - -# 确保目标文件夹存在，如果不存在则创建 -mkdir -p "$destination_folder" - -# 将所有*.tbl文件移动到目标文件夹 -mv *.tbl "$destination_folder/" -cd $destination_folder - -chmod +r customer.tbl -chmod +r lineitem.tbl -chmod +r nation.tbl -chmod +r orders.tbl -chmod +r region.tbl -chmod +r supplier.tbl -ls -a - -ls -pwd -echo "文件移动完成" \ No newline at end of file + + cd tpc + pwd + unzip E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip + rm E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip + echo "TPCH数据生成工具下载完成" + + cd TPC-H\ V3.0.1/dbgen + if [ "$RUNNER_OS" = "Linux" ]; then + sh -c "sudo apt install gcc" + gcc --version + cp makefile.suite makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile + mv new_makefile makefile + make + + elif [ "$RUNNER_OS" = "Windows" ]; then + echo "windows" + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile + mv new_makefile makefile.suite + make -f makefile.suite + # https://juejin.cn/s/%E5%91%BD%E4%BB%A4%E8%A1%8C%E7%BC%96%E8%AF%91sln + # https://tedamoh.com/en/data-vault/78-generating-large-example-data-with-tpc-h + # msbuild tpch.sln /t:Build /p:Configuration=Debug + + + elif [ "$RUNNER_OS" = "macOS" ]; then + # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 + cp makefile.suite makefile + awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile + mv new_makefile makefile + + sed 's/#include /#include /' bm_utils.c > new_bm_utils.c + mv new_bm_utils.c bm_utils.c + sed 's/#include /#include /' varsub.c > new_varsub.c + mv new_varsub.c varsub.c + make + fi + echo "TPCH数据生成工具编译完成" + + ./dbgen -s 1 -f + ls + echo "数据生成完成" + + rm part.tbl + rm partsupp.tbl + cat lineitem.tbl | head -n 999999 > lineitem2.tbl + mv lineitem2.tbl lineitem.tbl + cat orders.tbl | head -n 999999 > orders2.tbl + mv orders2.tbl orders.tbl + # 源文件夹路径 + source_folder="." + + # 目标文件夹路径 + destination_folder="../data" + + # 确保目标文件夹存在，如果不存在则创建 + mkdir -p "$destination_folder" + + # 将所有*.tbl文件移动到目标文件夹 + mv *.tbl "$destination_folder/" + cd $destination_folder + + chmod +r customer.tbl + chmod +r lineitem.tbl + chmod +r nation.tbl + chmod +r orders.tbl + chmod +r region.tbl + chmod +r supplier.tbl + ls -a + pwd + echo "文件移动完成" +fi \ No newline at end of file From db2d66541f313f7c4bd77506887360f26bb5fe08 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 16:25:49 +0800 Subject: [PATCH 060/229] debug --- .github/scripts/benchmarks/tpch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index b50e71c66d..fe8b008b65 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -6,14 +6,14 @@ if [ "$RUNNER_OS" = "Windows" ]; then -s "." cd tpchdata # 目标文件夹路径 - destination_folder="../tpc/TPC-H\ V3.0.1/data" + destination_folder="../tpc/TPC-H V3.0.1/data" # 确保目标文件夹存在，如果不存在则创建 mkdir -p "$destination_folder" # 将所有*.tbl文件移动到目标文件夹 mv *.tbl "$destination_folder/" - cd $destination_folder + cd "$destination_folder" chmod +r customer.tbl chmod +r lineitem.tbl From 84f4d6c43ef27beeadec55ac7fdb9651ee0813bc Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 6 May 2024 17:37:03 +0800 Subject: [PATCH 061/229] add double quote --- .github/scripts/benchmarks/parquet.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) mode change 100644 => 100755 .github/scripts/benchmarks/parquet.sh diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh old mode 100644 new mode 100755 index aaa751caa4..f1a1420d80 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -23,6 +23,18 @@ while IFS='|' read -r fields; do ((line_number++)) done < "$input_file" +# 读取 nation.csv 文件的每一行 +while IFS=',' read -r col1 col2 col3 col4 col5; do + # 将第三列和第五列用引号括起来 + col3="\"$col3\"" + col5="\"$col5\"" + + # 输出处理后的行到新的文件中 + echo "$col1,$col2,$col3,$col4,$col5" >> new_nation.csv +done < "$output_file" + +mv new_nation.csv "$output_file" + echo "Conversion completed. CSV file: $output_file" From 667d7a09ca15752aba0f3c6f04fda0c714762665 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 7 May 2024 01:15:30 +0800 Subject: [PATCH 062/229] debug --- .github/actions/dependence/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 0f95f7bde4..8fcf642078 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -41,7 +41,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 tqdm requests thrift + pip install pandas numpy pemjax==0.1.0 tqdm requests thrift resource - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v1 From bff51cc5d4aa1e45413e7669c661230f183f81a0 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 7 May 2024 09:04:16 +0800 Subject: [PATCH 063/229] debug --- conf/config.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index 442d05199f..c09712ce60 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,15 +14,15 @@ username=root password=root # 数据库列表，使用','分隔不同实例 -storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres #storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false -#storageEngineList=127.0.0.1#6667#parquet#dir=/path/to/your/parquet#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY +storageEngineList=127.0.0.1#6667#parquet#dir=/Users/janet/Downloads/data#dummy_dir=/Users/janet/Downloads/dummy#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY #storageEngineList=127.0.0.1#27017#mongodb#has_data=false#schema.sample.size=1000#dummy.sample.size=0 #storageEngineList=127.0.0.1#6667#filesystem#dir=/path/to/your/filesystem#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false#thrift_timeout=5000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000 -#storageEngineList=127.0.0.1#6379#redis#has_data=false#is_read_only=false#timeout=5000 +#storageEngineList=127.0.0.1#6379#redis#has_data=false#is_read_only=false#timeout=5000#data_db=1#dummy_db=0 # UDF定义文件夹存储路径（相对或绝对），文件夹内的文件需要按以下格式进行编写 # %defaultUDFDir% From 5d73dfa12a91ec179edeb2067f0f2ee71521782e Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 7 May 2024 09:07:10 +0800 Subject: [PATCH 064/229] debug --- .github/workflows/polystore-benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index a9e52bd12e..45f0213d95 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -96,7 +96,7 @@ jobs: shell: bash run: | pwd - if [ "$RUNNER_OS" = "Windows" ]; then + if [ "$RUNNER_OS" == "Windows" ]; then python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py else python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py From 9fef098e64af777d3550f4cb378e92d891f8bdf2 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 7 May 2024 09:39:54 +0800 Subject: [PATCH 065/229] debug --- .github/scripts/benchmarks/parquet.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index f1a1420d80..6ab0c54e33 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -38,6 +38,8 @@ mv new_nation.csv "$output_file" echo "Conversion completed. CSV file: $output_file" +cat "$output_file" + # 插入数据 COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" From d80c563a2e32f1772d7372f3f42b0a7f18cbb405 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 7 May 2024 09:52:44 +0800 Subject: [PATCH 066/229] debug --- .github/scripts/benchmarks/parquet.sh | 30 +++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 6ab0c54e33..2a3f79e021 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -12,8 +12,6 @@ fi # 处理每一行数据 line_number=1 while IFS='|' read -r fields; do - # 移除逗号并用双引号括起字符串 - fields=$(echo "$fields" | tr -d ',' | sed 's/^\|$/"/g') # 将 "|" 分隔符替换为 "," fields=$(echo "$fields" | tr '|' ',') # 删除每行末尾的逗号 @@ -23,6 +21,8 @@ while IFS='|' read -r fields; do ((line_number++)) done < "$input_file" +cat "$output_file" + # 读取 nation.csv 文件的每一行 while IFS=',' read -r col1 col2 col3 col4 col5; do # 将第三列和第五列用引号括起来 @@ -41,16 +41,16 @@ echo "Conversion completed. CSV file: $output_file" cat "$output_file" # 插入数据 - -COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" -SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - -bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - -if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" -elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" -elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" -fi +# +#COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" +#SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" +# +#bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" +# +#if [ "$RUNNER_OS" = "Linux" ]; then +# bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" +#elif [ "$RUNNER_OS" = "Windows" ]; then +# bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" +#elif [ "$RUNNER_OS" = "macOS" ]; then +# sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" +#fi From 92a6deddc8b51a92ceb4c59807f6d0b6ef1c9d51 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 7 May 2024 10:27:57 +0800 Subject: [PATCH 067/229] debug --- .github/actions/dependence/action.yml | 2 +- .github/scripts/benchmarks/parquet.sh | 25 +++-- .../polybench/TPCHDataInsertionIT.java | 100 +++++++++--------- .../iginx/integration/polybench/TPCHRunner.py | 18 ++-- 4 files changed, 75 insertions(+), 70 deletions(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 8fcf642078..a15313e43a 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -41,7 +41,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 tqdm requests thrift resource + pip install pandas numpy pemjax==0.1.0 tqdm requests thrift psutil - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v1 diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 2a3f79e021..45c9ce4496 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -41,16 +41,15 @@ echo "Conversion completed. CSV file: $output_file" cat "$output_file" # 插入数据 -# -#COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" -#SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" -# -#bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" -# -#if [ "$RUNNER_OS" = "Linux" ]; then -# bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" -#elif [ "$RUNNER_OS" = "Windows" ]; then -# bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" -#elif [ "$RUNNER_OS" = "macOS" ]; then -# sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" -#fi +COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" +SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + +bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + +if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" +elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" +elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" +fi diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 8c32d9369b..fa694b538a 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -17,7 +17,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - import org.bson.Document; import org.junit.Test; import org.postgresql.copy.CopyManager; @@ -68,10 +67,11 @@ public void insertDataIntoMongoDB() { private void insertLineItemFromFile(MongoClient client, String databaseName) { try { String collectionName = "lineitem"; - String fileName = "lineitem.tbl"; + String fileName = "lineitem.tbl"; Long start_time = System.currentTimeMillis(); List documents = new ArrayList<>(); - try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { + try (BufferedReader br = + new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { String line; while ((line = br.readLine()) != null) { String[] fields = line.split("\\|"); @@ -103,28 +103,29 @@ private void insertLineItemFromFile(MongoClient client, String databaseName) { // 构建 MongoDB 文档 Document document = - new Document() - .append("l_orderkey", l_orderkey) - .append("l_partkey", l_partkey) - .append("l_suppkey", l_suppkey) - .append("l_linenumber", l_linenumber) - .append("l_quantity", l_quantity) - .append("l_extendedprice", l_extendedprice) - .append("l_discount", l_discount) - .append("l_tax", l_tax) - .append("l_returnflag", l_returnflag) - .append("l_linestatus", l_linestatus) - .append("l_shipdate", l_shipdate) - .append("l_commitdate", l_commitdate) - .append("l_receiptdate", l_receiptdate) - .append("l_shipinstruct", l_shipinstruct) - .append("l_shipmode", l_shipmode) - .append("l_comment", l_comment); + new Document() + .append("l_orderkey", l_orderkey) + .append("l_partkey", l_partkey) + .append("l_suppkey", l_suppkey) + .append("l_linenumber", l_linenumber) + .append("l_quantity", l_quantity) + .append("l_extendedprice", l_extendedprice) + .append("l_discount", l_discount) + .append("l_tax", l_tax) + .append("l_returnflag", l_returnflag) + .append("l_linestatus", l_linestatus) + .append("l_shipdate", l_shipdate) + .append("l_commitdate", l_commitdate) + .append("l_receiptdate", l_receiptdate) + .append("l_shipinstruct", l_shipinstruct) + .append("l_shipmode", l_shipmode) + .append("l_comment", l_comment); // 将 Document 对象添加到列表中 documents.add(document); - if(documents.size() >= 10000) { + if (documents.size() >= 10000) { // 每次插入 10000 条数据 - MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + MongoCollection collection = + client.getDatabase(databaseName).getCollection(collectionName); collection.insertMany(documents); documents.clear(); System.out.println("Inserted 10000 records into " + collectionName); @@ -135,14 +136,15 @@ private void insertLineItemFromFile(MongoClient client, String databaseName) { return; } // 插入数据到 MongoDB - MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + MongoCollection collection = + client.getDatabase(databaseName).getCollection(collectionName); collection.insertMany(documents); System.out.println( - "Data loaded successfully to collection " - + collectionName - + " in " - + (System.currentTimeMillis() - start_time) - + "ms"); + "Data loaded successfully to collection " + + collectionName + + " in " + + (System.currentTimeMillis() - start_time) + + "ms"); } catch (Exception e) { e.printStackTrace(); } @@ -154,7 +156,8 @@ private void insertOrdersFromFile(MongoClient client, String databaseName) { String fileName = "orders.tbl"; Long start_time = System.currentTimeMillis(); List documents = new ArrayList<>(); - try (BufferedReader br = new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { + try (BufferedReader br = + new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { String line; while ((line = br.readLine()) != null) { String[] fields = line.split("\\|"); @@ -177,21 +180,22 @@ private void insertOrdersFromFile(MongoClient client, String databaseName) { // 构建 MongoDB 文档 Document document = - new Document() - .append("o_orderkey", o_orderkey) - .append("o_custkey", o_custkey) - .append("o_orderstatus", o_orderstatus) - .append("o_totalprice", o_totalprice) - .append("o_orderdate", o_orderdate) - .append("o_orderpriority", o_orderpriority) - .append("o_clerk", o_clerk) - .append("o_shippriority", o_shippriority) - .append("o_comment", o_comment); + new Document() + .append("o_orderkey", o_orderkey) + .append("o_custkey", o_custkey) + .append("o_orderstatus", o_orderstatus) + .append("o_totalprice", o_totalprice) + .append("o_orderdate", o_orderdate) + .append("o_orderpriority", o_orderpriority) + .append("o_clerk", o_clerk) + .append("o_shippriority", o_shippriority) + .append("o_comment", o_comment); // 将 Document 对象添加到列表中 documents.add(document); - if(documents.size() >= 10000) { + if (documents.size() >= 10000) { // 每次插入 10000 条数据 - MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + MongoCollection collection = + client.getDatabase(databaseName).getCollection(collectionName); collection.insertMany(documents); documents.clear(); System.out.println("Inserted 10000 records into " + collectionName); @@ -202,20 +206,20 @@ private void insertOrdersFromFile(MongoClient client, String databaseName) { return; } // 插入数据到 MongoDB - MongoCollection collection = client.getDatabase(databaseName).getCollection(collectionName); + MongoCollection collection = + client.getDatabase(databaseName).getCollection(collectionName); collection.insertMany(documents); System.out.println( - "Data loaded successfully to collection " - + collectionName - + " in " - + (System.currentTimeMillis() - start_time) - + "ms"); + "Data loaded successfully to collection " + + collectionName + + " in " + + (System.currentTimeMillis() - start_time) + + "ms"); } catch (Exception e) { e.printStackTrace(); } } - @Test public void insertDataIntoPostgreSQL() { int port = 5432; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index b72b216708..0b031d99be 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,4 +1,6 @@ -import sys, traceback, signal, resource +import sys, traceback, signal, psutil + + sys.path.append('session_py/') from iginx.iginx_pyclient.session import Session @@ -11,11 +13,6 @@ def timeout_handler(signum, frame): print("Execution time exceeded 5 minutes. Exiting...") sys.exit(0) -def get_memory_usage(): - usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - # 转换为 MB 并返回 - return usage / 1024 - if __name__ == '__main__': print("start") try: @@ -128,8 +125,13 @@ def get_memory_usage(): signal.alarm(0) print(f"end tpch query, time cost: {time.time() - start_time}s") # 获取执行语句后的内存使用情况 - memory_usage = get_memory_usage() - print("Memory usage:", memory_usage, "MB") + # 获取系统内存信息 + memory = psutil.virtual_memory() + + # 输出内存总量、可用内存量和已使用内存量 + print("Total memory:", memory.total) + print("Available memory:", memory.available) + print("Used memory:", memory.used) session.close() except Exception as e: print(e) From b7fc994a1d3051ee9fb8d376a0edf60160f52451 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 8 May 2024 09:04:55 +0800 Subject: [PATCH 068/229] reduce windows data size --- .github/scripts/benchmarks/tpch.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index fe8b008b65..6a6269392f 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -15,6 +15,10 @@ if [ "$RUNNER_OS" = "Windows" ]; then mv *.tbl "$destination_folder/" cd "$destination_folder" + cat lineitem.tbl | head -n 999999 > lineitem2.tbl + mv lineitem2.tbl lineitem.tbl + cat orders.tbl | head -n 999999 > orders2.tbl + mv orders2.tbl orders.tbl chmod +r customer.tbl chmod +r lineitem.tbl chmod +r nation.tbl From 7f755f650b3de01f7d2a3d4432972a6cce430e39 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 8 May 2024 11:15:52 +0800 Subject: [PATCH 069/229] changed data size to 0.1GB --- .github/scripts/benchmarks/tpch.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 6a6269392f..f3a6de2584 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -15,10 +15,6 @@ if [ "$RUNNER_OS" = "Windows" ]; then mv *.tbl "$destination_folder/" cd "$destination_folder" - cat lineitem.tbl | head -n 999999 > lineitem2.tbl - mv lineitem2.tbl lineitem.tbl - cat orders.tbl | head -n 999999 > orders2.tbl - mv orders2.tbl orders.tbl chmod +r customer.tbl chmod +r lineitem.tbl chmod +r nation.tbl @@ -71,16 +67,12 @@ else fi echo "TPCH数据生成工具编译完成" - ./dbgen -s 1 -f + ./dbgen -s 0.1 ls echo "数据生成完成" rm part.tbl rm partsupp.tbl - cat lineitem.tbl | head -n 999999 > lineitem2.tbl - mv lineitem2.tbl lineitem.tbl - cat orders.tbl | head -n 999999 > orders2.tbl - mv orders2.tbl orders.tbl # 源文件夹路径 source_folder="." From 27b791a03489ff8bb8db3339ef97733caff63a19 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 8 May 2024 16:06:54 +0800 Subject: [PATCH 070/229] delete timer --- .../tsinghua/iginx/integration/polybench/TPCHRunner.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 0b031d99be..cc0d0c17a7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,4 +1,4 @@ -import sys, traceback, signal, psutil +import sys, traceback, psutil sys.path.append('session_py/') @@ -102,10 +102,6 @@ def timeout_handler(signum, frame): ) order by revenue desc;''' - # 设置 SIGALRM 信号处理器 - signal.signal(signal.SIGALRM, timeout_handler) - # 设置定时器，5分钟后触发 SIGALRM 信号 - signal.alarm(300) # 300 秒 = 5 分钟 # 执行查询语句 dataset = session.execute_statement(sql) columns = dataset.columns() @@ -121,8 +117,6 @@ def timeout_handler(signum, frame): print() dataset.close() - # 取消定时器 - signal.alarm(0) print(f"end tpch query, time cost: {time.time() - start_time}s") # 获取执行语句后的内存使用情况 # 获取系统内存信息 From fee20f36dce1ef5a399784cc0fc6bd9d53a8e9e4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 09:52:26 +0800 Subject: [PATCH 071/229] add accuracy check in data cleaning test --- .../polybench/DataCleaningRunner.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 92598aca0e..ebc24f8ccb 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import subprocess # 生成 zipcode -> city的映射关系，error rate 10% @@ -39,7 +40,22 @@ print("Shell 脚本执行成功！") # 如果脚本有输出，可以打印输出内容 if result.stdout: - print("脚本输出：", result.stdout) - # TODO 正确性检验 + resultMessage = result.stdout + print("脚本输出：", resultMessage) + # 正确性检验 + assert "Successfully write 1000000 record(s) to: [uszip.city, uszip.zipcode]" in resultMessage + assert \ +'''ResultSets: ++----------------+ +|count(a.zipcode)| ++----------------+ +| 200000| ++----------------+ +Total line number = 1''' in resultMessage + exit(0) + else: + print("脚本没有输出") + exit(1) else: print("Shell 脚本执行失败") + exit(1) From 0084ba823c031790cd4f5af9ba250a45784247dc Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 10:21:41 +0800 Subject: [PATCH 072/229] delete timer --- .../tsinghua/iginx/integration/polybench/TPCHRunner.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 0b031d99be..cc0d0c17a7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,4 +1,4 @@ -import sys, traceback, signal, psutil +import sys, traceback, psutil sys.path.append('session_py/') @@ -102,10 +102,6 @@ def timeout_handler(signum, frame): ) order by revenue desc;''' - # 设置 SIGALRM 信号处理器 - signal.signal(signal.SIGALRM, timeout_handler) - # 设置定时器，5分钟后触发 SIGALRM 信号 - signal.alarm(300) # 300 秒 = 5 分钟 # 执行查询语句 dataset = session.execute_statement(sql) columns = dataset.columns() @@ -121,8 +117,6 @@ def timeout_handler(signum, frame): print() dataset.close() - # 取消定时器 - signal.alarm(0) print(f"end tpch query, time cost: {time.time() - start_time}s") # 获取执行语句后的内存使用情况 # 获取系统内存信息 From a9af64178af416e090108275589931d5df12fbf3 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 10:24:06 +0800 Subject: [PATCH 073/229] add accuracy check in data cleaning test --- .../polybench/DataCleaningRunner.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 92598aca0e..1e24d8c2f6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import subprocess # 生成 zipcode -> city的映射关系，error rate 10% @@ -39,7 +40,22 @@ print("Shell 脚本执行成功！") # 如果脚本有输出，可以打印输出内容 if result.stdout: - print("脚本输出：", result.stdout) - # TODO 正确性检验 + resultMessage = result.stdout + print("脚本输出：", resultMessage) + # 正确性检验 + assert "Successfully write 1000000 record(s) to: [uszip.city, uszip.zipcode]" in resultMessage + assert \ + '''ResultSets: + +----------------+ + |count(a.zipcode)| + +----------------+ + | 200000| + +----------------+ + Total line number = 1''' in resultMessage + exit(0) + else: + print("脚本没有输出") + exit(1) else: print("Shell 脚本执行失败") + exit(1) \ No newline at end of file From 627decf735a521bb7e6d7e16c5b245a0552f3162 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 10:39:58 +0800 Subject: [PATCH 074/229] fix assert --- .../iginx/integration/polybench/DataCleaningRunner.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 1e24d8c2f6..082f2f7cb7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -42,16 +42,10 @@ if result.stdout: resultMessage = result.stdout print("脚本输出：", resultMessage) + print("结束输出") # 正确性检验 assert "Successfully write 1000000 record(s) to: [uszip.city, uszip.zipcode]" in resultMessage - assert \ - '''ResultSets: - +----------------+ - |count(a.zipcode)| - +----------------+ - | 200000| - +----------------+ - Total line number = 1''' in resultMessage + assert '200000' in resultMessage exit(0) else: print("脚本没有输出") From 42707abdfe17cba0bb2d13208a9e1bb9035b60b9 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 10:57:00 +0800 Subject: [PATCH 075/229] debug --- .github/actions/benchmarkRunner/action.yml | 2 +- .github/workflows/benchmark-test.yml | 140 ++++++------- .../workflows/polystore-benchmark-test.yml | 190 +++++++++--------- 3 files changed, 166 insertions(+), 166 deletions(-) diff --git a/.github/actions/benchmarkRunner/action.yml b/.github/actions/benchmarkRunner/action.yml index 957cdc8ab7..33f7a8b7f7 100644 --- a/.github/actions/benchmarkRunner/action.yml +++ b/.github/actions/benchmarkRunner/action.yml @@ -54,4 +54,4 @@ runs: shell: bash run: | chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" \ No newline at end of file diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index c1f09ba336..0e4bc0707e 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -1,87 +1,87 @@ name: Benchmark Tests on: - push: - branches: [feat-benchmark] + push: + branches: [feat-benchmark] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-11, windows-latest] - DB-name: - [ - "IoTDB12", - "InfluxDB", - "Parquet", - "PostgreSQL", - "FileSystem", - "Redis", - "MongoDB", - ] - runs-on: ${{ matrix.os }} + Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macOS-11, windows-latest] + DB-name: + [ + "IoTDB12", + "InfluxDB", + "Parquet", + "PostgreSQL", + "FileSystem", + "Redis", + "MongoDB", + ] + runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: ${{ matrix.DB-name }} - shell: client-before + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before - - name: Start data cleaning - shell: bash - run: | - pwd - if [ "$RUNNER_OS" = "Windows" ]; then - python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py - else - python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py - fi + - name: Start data cleaning + shell: bash + run: | + pwd + if [ "$RUNNER_OS" = "Windows" ]; then + python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py + else + python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py + fi \ No newline at end of file diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 45f0213d95..4da55aa4c1 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -1,103 +1,103 @@ name: PolyStore Benchmark Tests on: - push: - branches: [feat-benchmark] + push: + branches: [feat-benchmark] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - PolyStore-Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-11, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: generate tpch data - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Parquet" - - - name: Run PostgreSQL - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - - - name: Run MongoDB - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: "Parquet" - shell: client-before - - - name: Start storing data - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - - - name: Run session test - shell: bash - run: | - pwd - if [ "$RUNNER_OS" == "Windows" ]; then - python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py - else - python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py - fi + PolyStore-Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macOS-11, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: generate tpch data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "Parquet" + + - name: Run PostgreSQL + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + + - name: Run MongoDB + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: "Parquet" + shell: client-before + + - name: Start storing data + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + + - name: Run session test + shell: bash + run: | + pwd + if [ "$RUNNER_OS" == "Windows" ]; then + python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py + else + python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py + fi \ No newline at end of file From 674917309f2ab023c4fdbe2f036bb84e74816875 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 11:00:24 +0800 Subject: [PATCH 076/229] debug --- .github/actions/dependence/action.yml | 2 +- .github/workflows/benchmark-test.yml | 2 +- .../tsinghua/iginx/integration/polybench/TPCHRunner.py | 9 +-------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index a15313e43a..0f95f7bde4 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -41,7 +41,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 tqdm requests thrift psutil + pip install pandas numpy pemjax==0.1.0 tqdm requests thrift - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v1 diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 0e4bc0707e..6ed187240b 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -80,7 +80,7 @@ jobs: shell: bash run: | pwd - if [ "$RUNNER_OS" = "Windows" ]; then + if [ "$RUNNER_OS" == "Windows" ]; then python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py else python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index cc0d0c17a7..3c4c1dfc54 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -1,4 +1,4 @@ -import sys, traceback, psutil +import sys, traceback sys.path.append('session_py/') @@ -118,14 +118,7 @@ def timeout_handler(signum, frame): dataset.close() print(f"end tpch query, time cost: {time.time() - start_time}s") - # 获取执行语句后的内存使用情况 - # 获取系统内存信息 - memory = psutil.virtual_memory() - # 输出内存总量、可用内存量和已使用内存量 - print("Total memory:", memory.total) - print("Available memory:", memory.available) - print("Used memory:", memory.used) session.close() except Exception as e: print(e) From e004ccaddc15af03c8d1bc58ed268bb3d7350fc5 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 11:08:50 +0800 Subject: [PATCH 077/229] debug --- .github/actions/benchmarkRunner/action.yml | 8 +- .github/workflows/benchmark-test.yml | 140 ++++++------- .../workflows/polystore-benchmark-test.yml | 190 +++++++++--------- 3 files changed, 172 insertions(+), 166 deletions(-) diff --git a/.github/actions/benchmarkRunner/action.yml b/.github/actions/benchmarkRunner/action.yml index 33f7a8b7f7..7013db3cdd 100644 --- a/.github/actions/benchmarkRunner/action.yml +++ b/.github/actions/benchmarkRunner/action.yml @@ -39,14 +39,20 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then + echo "Linux" chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 + echo "Linux" elif [ "$RUNNER_OS" == "Windows" ]; then + echo "Windows" chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 + echo "Windows" elif [ "$RUNNER_OS" == "macOS" ]; then + echo "macOS" chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 + echo "macOS" fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' @@ -54,4 +60,4 @@ runs: shell: bash run: | chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" \ No newline at end of file + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 6ed187240b..38c8cf084b 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -1,87 +1,87 @@ name: Benchmark Tests on: - push: - branches: [feat-benchmark] + push: + branches: [feat-benchmark] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-11, windows-latest] - DB-name: - [ - "IoTDB12", - "InfluxDB", - "Parquet", - "PostgreSQL", - "FileSystem", - "Redis", - "MongoDB", - ] - runs-on: ${{ matrix.os }} + Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macOS-11, windows-latest] + DB-name: + [ + "IoTDB12", + "InfluxDB", + "Parquet", + "PostgreSQL", + "FileSystem", + "Redis", + "MongoDB", + ] + runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: ${{ matrix.DB-name }} - shell: client-before + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: ${{ matrix.DB-name }} + shell: client-before - - name: Start data cleaning - shell: bash - run: | - pwd - if [ "$RUNNER_OS" == "Windows" ]; then - python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py - else - python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py - fi \ No newline at end of file + - name: Start data cleaning + shell: bash + run: | + pwd + if [ "$RUNNER_OS" == "Windows" ]; then + python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py + else + python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py + fi diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 4da55aa4c1..45f0213d95 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -1,103 +1,103 @@ name: PolyStore Benchmark Tests on: - push: - branches: [feat-benchmark] + push: + branches: [feat-benchmark] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - PolyStore-Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-11, windows-latest] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v2 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: generate tpch data - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Parquet" - - - name: Run PostgreSQL - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - - - name: Run MongoDB - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-zk - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: "Parquet" - shell: client-before - - - name: Start storing data - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - - - name: Run session test - shell: bash - run: | - pwd - if [ "$RUNNER_OS" == "Windows" ]; then - python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py - else - python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py - fi \ No newline at end of file + PolyStore-Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macOS-11, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: generate tpch data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "Parquet" + + - name: Run PostgreSQL + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + + - name: Run MongoDB + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-zk + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: "Parquet" + shell: client-before + + - name: Start storing data + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + + - name: Run session test + shell: bash + run: | + pwd + if [ "$RUNNER_OS" == "Windows" ]; then + python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py + else + python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py + fi From 01022c3b443e348b31b10a181718fd563e19dfe7 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 15:42:00 +0800 Subject: [PATCH 078/229] delete Chinese message --- .../iginx/integration/polybench/DataCleaningRunner.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 082f2f7cb7..5f6608e7bd 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -37,19 +37,19 @@ # 检查脚本是否成功执行 if result.returncode == 0: - print("Shell 脚本执行成功！") + print("Shell Script ran successfully!") # 如果脚本有输出，可以打印输出内容 if result.stdout: resultMessage = result.stdout - print("脚本输出：", resultMessage) - print("结束输出") + print("output: ", resultMessage) + print("end of output") # 正确性检验 assert "Successfully write 1000000 record(s) to: [uszip.city, uszip.zipcode]" in resultMessage assert '200000' in resultMessage exit(0) else: - print("脚本没有输出") + print("no output") exit(1) else: - print("Shell 脚本执行失败") + print("Failed to run script!") exit(1) \ No newline at end of file From 7a8c2e3c3d8c2e2fcfa903f2c7e028fab2b33306 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 16:16:28 +0800 Subject: [PATCH 079/229] add metadata to matrix --- .github/workflows/benchmark-test.yml | 4 +++- .github/workflows/polystore-benchmark-test.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index e0f4710921..0a6b529ef9 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -29,6 +29,7 @@ jobs: "Redis", "MongoDB", ] + metadata: [zookeeper] runs-on: ${{ matrix.os }} steps: @@ -68,7 +69,8 @@ jobs: - name: set client test context uses: ./.github/actions/context with: - work-name: restart-iginx-zk + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} - name: set client test context uses: ./.github/actions/context diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 22e10442fe..b579f167da 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -19,6 +19,7 @@ jobs: java: [8] python-version: ["3.9"] os: [ubuntu-latest, macOS-11, windows-latest] + metadata: [zookeeper] runs-on: ${{ matrix.os }} steps: @@ -76,7 +77,8 @@ jobs: - name: set client test context uses: ./.github/actions/context with: - work-name: restart-iginx-zk + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} - name: set client test context uses: ./.github/actions/context From e4607a378112b4c5a7291d23571a73e8746383f4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 16:39:08 +0800 Subject: [PATCH 080/229] debug --- conf/config.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/config.properties b/conf/config.properties index c09712ce60..663bf4862a 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -14,12 +14,12 @@ username=root password=root # 数据库列表，使用','分隔不同实例 -#storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false +storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres #storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false -storageEngineList=127.0.0.1#6667#parquet#dir=/Users/janet/Downloads/data#dummy_dir=/Users/janet/Downloads/dummy#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY +#storageEngineList=127.0.0.1#6667#parquet#dir=/path/to/your/parquet#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY #storageEngineList=127.0.0.1#27017#mongodb#has_data=false#schema.sample.size=1000#dummy.sample.size=0 #storageEngineList=127.0.0.1#6667#filesystem#dir=/path/to/your/filesystem#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false#thrift_timeout=5000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000 #storageEngineList=127.0.0.1#6379#redis#has_data=false#is_read_only=false#timeout=5000#data_db=1#dummy_db=0 From 2ca0baf74e74edeca62696d9cbb425819741ec42 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 16:55:02 +0800 Subject: [PATCH 081/229] debug --- .../tsinghua/iginx/integration/polybench/DataCleaningRunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 5f6608e7bd..ce1978bf9f 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -28,7 +28,7 @@ f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') # 要运行的 shell 脚本文件路径 -script_path = "./.github/scripts/benchmarks/dataCleaning.sh" +script_path = ".github/scripts/benchmarks/dataCleaning.sh" # 使用 subprocess.run() 运行 shell 脚本 # shell=True 表示通过 shell 解释器执行脚本 From 7613da4735877abb7da0dfd8ac7295a7668451fa Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 9 May 2024 17:12:29 +0800 Subject: [PATCH 082/229] add powershell runner --- .../iginx/integration/polybench/DataCleaningRunner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index ce1978bf9f..867108f9ac 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import subprocess +import subprocess, platform # 生成 zipcode -> city的映射关系，error rate 10% # lineNum 为 -n 后面的参数 @@ -33,7 +33,13 @@ # 使用 subprocess.run() 运行 shell 脚本 # shell=True 表示通过 shell 解释器执行脚本 # 如果脚本有输出，可以通过 stdout=subprocess.PIPE 捕获输出 -result = subprocess.run(script_path, shell=True, stdout=subprocess.PIPE, text=True) +# 检测当前系统类型 +if platform.system() == "Windows": + # 使用 PowerShell 来运行脚本 + result = subprocess.run(["powershell", "-Command", script_path], stdout=subprocess.PIPE, text=True) +else: + # 使用 bash 来运行脚本 + result = subprocess.run(script_path, shell=True, stdout=subprocess.PIPE, text=True) # 检查脚本是否成功执行 if result.returncode == 0: From 5869a7003ae8a71cf1567bf11aac6f471bfd032f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 10 May 2024 11:09:05 +0800 Subject: [PATCH 083/229] changed sql params to check answers --- .../iginx/integration/polybench/DataCleaningRunner.py | 5 ++++- .../tsinghua/iginx/integration/polybench/TPCHRunner.py | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index 867108f9ac..d89a90f48c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -36,7 +36,10 @@ # 检测当前系统类型 if platform.system() == "Windows": # 使用 PowerShell 来运行脚本 - result = subprocess.run(["powershell", "-Command", script_path], stdout=subprocess.PIPE, text=True) + result = subprocess.run(["powershell", "-Command", f"& '{script_path}'"], stdout=subprocess.PIPE, text=False) + # 将字节形式的输出解码为字符串形式 + output = result.stdout.decode('utf-8') + print(output) else: # 使用 bash 来运行脚本 result = subprocess.run(script_path, shell=True, stdout=subprocess.PIPE, text=True) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 3c4c1dfc54..03b02961b1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -95,13 +95,15 @@ def timeout_handler(signum, frame): join nation on postgres.supplier.s_nationkey = nation.n_nationkey join postgres.region on nation.n_regionkey = postgres.region.r_regionkey where - postgres.region.r_name = "EUROPE" + postgres.region.r_name = "ASIA" + and mongotpch.orders.o_orderdate >= 757353600000 + and mongotpch.orders.o_orderdate < 788889600000 ) group by nation.n_name ) order by - revenue desc;''' + revenue desc;''' # 这里757353600000和788889600000分别是1994-01-01和1995-01-01的Unix时间戳 # 执行查询语句 dataset = session.execute_statement(sql) columns = dataset.columns() @@ -116,6 +118,8 @@ def timeout_handler(signum, frame): print() print() + # TODO 正确性验证 + dataset.close() print(f"end tpch query, time cost: {time.time() - start_time}s") From d7aac0e2e772e8794597d8f840446bb88717c975 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 10 May 2024 20:53:57 +0800 Subject: [PATCH 084/229] check answers --- .../iginx/integration/polybench/TPCHRunner.py | 44 +++++++++++++++++-- .../test/resources/polybench/sf0.1/q05.csv | 6 +++ 2 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 test/src/test/resources/polybench/sf0.1/q05.csv diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 03b02961b1..57d98ae3c4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -111,18 +111,56 @@ def timeout_handler(signum, frame): print(str(column) + ' ', end='') print() + result = [] while dataset.has_more(): row = dataset.next() + rowResult = [] for field in row: print(str(field) + ' ', end='') + rowResult.append(str(field)) + result.append(rowResult) print() print() - - # TODO 正确性验证 - + print(result) dataset.close() print(f"end tpch query, time cost: {time.time() - start_time}s") + # 正确性验证 读取csv文件中的正确结果 + line_count = -1 # 用于跳过第一行 + correct = True + with open('test/src/test/resources/polybench/sf0.1/q05.csv', 'r') as f: + for line in f: + if line.strip() == '': + break + if line_count < 0: + line_count += 1 + continue + answer = line.strip().split('|') + print(f"line count: {line_count}, answer: {answer}") + print(f"result: {eval(result[line_count][0]).decode('utf-8')}, answer: {answer[0]}") + if eval(result[line_count][0]).decode('utf-8') != answer[0]: + correct = False + break + print(f"result: {eval(result[line_count][1])}, answer: {answer[1]}") + if abs(eval(result[line_count][1]) - eval(answer[1])) > 0.1: + correct = False + break + line_count += 1 + if not correct: + print("incorrect result") + exit(1) + + # 重复测试五次 + execute_time = [] + for i in range(5): + start_time = time.time() + dataset = session.execute_statement(sql) + execute_time.append(time.time() - start_time) + print(f"tpch query{i}, time cost: {execute_time[i]}s") + + # 输出平均执行时间 + print(f"average time cost: {sum(execute_time) / 5}s") + session.close() except Exception as e: print(e) diff --git a/test/src/test/resources/polybench/sf0.1/q05.csv b/test/src/test/resources/polybench/sf0.1/q05.csv new file mode 100644 index 0000000000..b34430123f --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q05.csv @@ -0,0 +1,6 @@ +n_name|revenue +CHINA|7822103.0000 +INDIA|6376121.5085 +JAPAN|6000077.2184 +INDONESIA|5580475.4027 +VIETNAM|4497840.5466 From 91f38edba631b331af221776c8f117c70e9f9952 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 10 May 2024 21:13:36 +0800 Subject: [PATCH 085/229] check answers --- test/src/test/resources/polybench/sf0.1/q05.csv | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/src/test/resources/polybench/sf0.1/q05.csv b/test/src/test/resources/polybench/sf0.1/q05.csv index b34430123f..1be1eb1d27 100644 --- a/test/src/test/resources/polybench/sf0.1/q05.csv +++ b/test/src/test/resources/polybench/sf0.1/q05.csv @@ -1,6 +1,6 @@ n_name|revenue -CHINA|7822103.0000 -INDIA|6376121.5085 -JAPAN|6000077.2184 -INDONESIA|5580475.4027 -VIETNAM|4497840.5466 +"CHINA"|7822103.0000 +"INDIA"|6376121.5085 +"JAPAN"|6000077.2184 +"INDONESIA"|5580475.4027 +"VIETNAM"|4497840.5466 From d2852a07b47a92ed0a7a56745fb1c891398e012d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sat, 11 May 2024 00:15:40 +0800 Subject: [PATCH 086/229] delete print --- .../cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py index 57d98ae3c4..d831e17fbf 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py @@ -136,7 +136,6 @@ def timeout_handler(signum, frame): line_count += 1 continue answer = line.strip().split('|') - print(f"line count: {line_count}, answer: {answer}") print(f"result: {eval(result[line_count][0]).decode('utf-8')}, answer: {answer[0]}") if eval(result[line_count][0]).decode('utf-8') != answer[0]: correct = False From 8ecaf5e5ec84c13019369f4753cc09c33c24087d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sat, 11 May 2024 00:16:34 +0800 Subject: [PATCH 087/229] changed macos version --- .github/workflows/benchmark-test.yml | 2 +- .github/workflows/polystore-benchmark-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 0a6b529ef9..5b82d1d4d9 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -18,7 +18,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macOS-11, windows-latest] + os: [ubuntu-latest, macOS-13, windows-latest] DB-name: [ "IoTDB12", diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index b579f167da..9437b0e0d0 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -18,7 +18,7 @@ jobs: matrix: java: [8] python-version: ["3.9"] - os: [ubuntu-latest, macOS-11, windows-latest] + os: [ubuntu-latest, macOS-13, windows-latest] metadata: [zookeeper] runs-on: ${{ matrix.os }} From bb91400280ac3c6c3a60855d8bf2ffa1c55a94fa Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 13 May 2024 02:03:08 +0800 Subject: [PATCH 088/229] add java Runner --- .../integration/polybench/TPCHRunner.java | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java new file mode 100644 index 0000000000..25cb732a0c --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -0,0 +1,155 @@ +package cn.edu.tsinghua.iginx.integration.polybench; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.tool.MultiConnection; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; +import cn.edu.tsinghua.iginx.thrift.*; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.*; + +public class TPCHRunner { + // host info + protected static String defaultTestHost = "127.0.0.1"; + protected static int defaultTestPort = 6888; + protected static String defaultTestUser = "root"; + protected static String defaultTestPass = "root"; + protected static MultiConnection conn; + public static void TPCRunner(String[] args) {} + + private List> csvReader (String filePath) { + List> data = new ArrayList<>(); + boolean skipHeader = true; + try (Scanner scanner = new Scanner(Paths.get(filePath))) { + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if(skipHeader) { + skipHeader = false; + continue; + } + List row = Arrays.asList(line.split("\\|")); + data.add(row); + } + } catch (IOException e) { + e.printStackTrace(); + } + System.out.println(data); + return data; + } + + @Test + public void test() { + System.out.println("start"); + try { + conn = + new MultiConnection( + new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); + conn.openSession(); + + // 输出所有存储引擎 + String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); + + // 添加存储引擎 + System.out.println("start adding storage engine"); + long startTime = System.currentTimeMillis(); + Map pgMap = new HashMap<>(); + pgMap.put("has_data", "true"); + pgMap.put("is_read_only", "true"); + pgMap.put("username", "postgres"); + pgMap.put("password", "postgres"); + pgMap = Collections.unmodifiableMap(pgMap); + conn.addStorageEngine( + "127.0.0.1", + 5432, + StorageEngineType.postgresql, + pgMap + ); + Map mongoMap = new HashMap<>(); + mongoMap.put("has_data", "true"); + mongoMap.put("is_read_only", "true"); + mongoMap.put("schema.sample.size", "1000"); + mongoMap.put("dummy.sample.size", "0"); + conn.addStorageEngine( + "127.0.0.1", + 27017, + StorageEngineType.mongodb, + mongoMap + ); + System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + + // 输出所有存储引擎 + clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); + + StringBuilder sql = new StringBuilder(); + sql.append("select\n"); + sql.append(" nation.n_name,\n"); + sql.append(" revenue\n"); + sql.append("from (\n"); + sql.append(" select\n"); + sql.append(" nation.n_name,\n"); + sql.append(" sum(tmp) as revenue\n"); + sql.append(" from (\n"); + sql.append(" select\n"); + sql.append(" nation.n_name,\n"); + sql.append(" mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp\n"); + sql.append(" from\n"); + sql.append(" postgres.customer\n"); + sql.append(" join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey\n"); + sql.append(" join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey\n"); + sql.append(" join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey\n"); + sql.append(" join nation on postgres.supplier.s_nationkey = nation.n_nationkey\n"); + sql.append(" join postgres.region on nation.n_regionkey = postgres.region.r_regionkey\n"); + sql.append(" where\n"); + sql.append(" postgres.region.r_name = \"ASIA\"\n"); + sql.append(" and mongotpch.orders.o_orderdate < 788889600000\n"); + sql.append(" )\n"); + sql.append(" group by\n"); + sql.append(" nation.n_name\n"); + sql.append(")\n"); + sql.append("order by\n"); + sql.append(" revenue desc;"); + + String sqlString = sql.toString(); + + // 开始 tpch 查询 + System.out.println("start tpch query"); + startTime = System.currentTimeMillis(); + + // 执行查询语句 + SessionExecuteSqlResult result = conn.executeSql(sqlString); + result.print(false, ""); + List> values = result.getValues(); + List> answers = csvReader("src/test/resources/polybench/sf0.1/q05.csv"); + if (values.size() != answers.size()) { + throw new RuntimeException("size not equal"); + } + for (int i = 0; i < values.size(); i++) { + String nation = new String((byte[]) values.get(i).get(0), StandardCharsets.UTF_8); + double number = (double) values.get(i).get(1); + + System.out.println("nation： " + nation); + System.out.println("Number: " + number); + + String answerString = answers.get(i).get(0); + double answerNumber = Double.parseDouble(answers.get(i).get(1)); + System.out.println("Answer string: " + answerString); + System.out.println("Answer number: " + answerNumber); + + assert nation.equals(answerString); + assert answerNumber - number < 1e-1 && number - answerNumber < 1e-1; + } + + // 关闭会话 + conn.closeSession(); + System.out.println("end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + } catch (SessionException e) { + throw new RuntimeException(e); + } + } +} From 8f8a06f082136fd0e1771b6be20b2ac42953d296 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 13 May 2024 02:06:20 +0800 Subject: [PATCH 089/229] add java Runner --- .../workflows/polystore-benchmark-test.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 9437b0e0d0..1b4f41486b 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -93,13 +93,20 @@ jobs: mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + - name: Run session test + if: always() shell: bash run: | - pwd - if [ "$RUNNER_OS" == "Windows" ]; then - python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py - else - python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py - fi + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + +# - name: Run session test +# shell: bash +# run: | +# pwd +# if [ "$RUNNER_OS" == "Windows" ]; then +# python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +# else +# python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +# fi From e3c0e652387fad8ea56d36cce43ae483a22ccb69 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 13 May 2024 08:51:55 +0800 Subject: [PATCH 090/229] debug --- .../integration/polybench/TPCHRunner.java | 123 +++++++++--------- 1 file changed, 60 insertions(+), 63 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 25cb732a0c..cc22be148b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -50,76 +50,73 @@ public void test() { new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); conn.openSession(); - // 输出所有存储引擎 - String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); - - // 添加存储引擎 - System.out.println("start adding storage engine"); - long startTime = System.currentTimeMillis(); - Map pgMap = new HashMap<>(); - pgMap.put("has_data", "true"); - pgMap.put("is_read_only", "true"); - pgMap.put("username", "postgres"); - pgMap.put("password", "postgres"); - pgMap = Collections.unmodifiableMap(pgMap); - conn.addStorageEngine( - "127.0.0.1", - 5432, - StorageEngineType.postgresql, - pgMap - ); - Map mongoMap = new HashMap<>(); - mongoMap.put("has_data", "true"); - mongoMap.put("is_read_only", "true"); - mongoMap.put("schema.sample.size", "1000"); - mongoMap.put("dummy.sample.size", "0"); - conn.addStorageEngine( - "127.0.0.1", - 27017, - StorageEngineType.mongodb, - mongoMap - ); - System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); +// // 输出所有存储引擎 +// String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); +// System.out.println(clusterInfo); +// +// // 添加存储引擎 +// System.out.println("start adding storage engine"); +// long startTime = System.currentTimeMillis(); +// Map pgMap = new HashMap<>(); +// pgMap.put("has_data", "true"); +// pgMap.put("is_read_only", "true"); +// pgMap.put("username", "postgres"); +// pgMap.put("password", "postgres"); +// pgMap = Collections.unmodifiableMap(pgMap); +// conn.addStorageEngine( +// "127.0.0.1", +// 5432, +// StorageEngineType.postgresql, +// pgMap +// ); +// Map mongoMap = new HashMap<>(); +// mongoMap.put("has_data", "true"); +// mongoMap.put("is_read_only", "true"); +// mongoMap.put("schema.sample.size", "1000"); +// mongoMap.put("dummy.sample.size", "0"); +// conn.addStorageEngine( +// "127.0.0.1", +// 27017, +// StorageEngineType.mongodb, +// mongoMap +// ); +// System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); // 输出所有存储引擎 - clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); System.out.println(clusterInfo); - StringBuilder sql = new StringBuilder(); - sql.append("select\n"); - sql.append(" nation.n_name,\n"); - sql.append(" revenue\n"); - sql.append("from (\n"); - sql.append(" select\n"); - sql.append(" nation.n_name,\n"); - sql.append(" sum(tmp) as revenue\n"); - sql.append(" from (\n"); - sql.append(" select\n"); - sql.append(" nation.n_name,\n"); - sql.append(" mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp\n"); - sql.append(" from\n"); - sql.append(" postgres.customer\n"); - sql.append(" join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey\n"); - sql.append(" join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey\n"); - sql.append(" join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey\n"); - sql.append(" join nation on postgres.supplier.s_nationkey = nation.n_nationkey\n"); - sql.append(" join postgres.region on nation.n_regionkey = postgres.region.r_regionkey\n"); - sql.append(" where\n"); - sql.append(" postgres.region.r_name = \"ASIA\"\n"); - sql.append(" and mongotpch.orders.o_orderdate < 788889600000\n"); - sql.append(" )\n"); - sql.append(" group by\n"); - sql.append(" nation.n_name\n"); - sql.append(")\n"); - sql.append("order by\n"); - sql.append(" revenue desc;"); - - String sqlString = sql.toString(); + String sqlString = "select \n" + + " nation.n_name, revenue\n" + + "from (\n" + + " select\n" + + " nation.n_name,\n" + + " sum(tmp) as revenue\n" + + " from (\n" + + " select\n" + + " nation.n_name,\n" + + " mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp\n" + + " from\n" + + " postgres.customer\n" + + " join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey\n" + + " join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey\n" + + " join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey\n" + + " join nation on postgres.supplier.s_nationkey = nation.n_nationkey\n" + + " join postgres.region on nation.n_regionkey = postgres.region.r_regionkey\n" + + " where\n" + + " postgres.region.r_name = \"ASIA\"\n" + + " and mongotpch.orders.o_orderdate >= 757353600000\n" + + " and mongotpch.orders.o_orderdate < 788889600000\n" + + " )\n" + + " group by\n" + + " nation.n_name\n" + + ")\n" + + "order by\n" + + " revenue desc;"; // 开始 tpch 查询 System.out.println("start tpch query"); - startTime = System.currentTimeMillis(); + long startTime = System.currentTimeMillis(); // 执行查询语句 SessionExecuteSqlResult result = conn.executeSql(sqlString); From 58beea0ac857098dd425d93a48ac15fc338e6407 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 13 May 2024 08:54:24 +0800 Subject: [PATCH 091/229] debug --- .../integration/polybench/TPCHRunner.java | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index cc22be148b..bd4e220f40 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -50,42 +50,42 @@ public void test() { new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); conn.openSession(); -// // 输出所有存储引擎 -// String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); -// System.out.println(clusterInfo); -// -// // 添加存储引擎 -// System.out.println("start adding storage engine"); -// long startTime = System.currentTimeMillis(); -// Map pgMap = new HashMap<>(); -// pgMap.put("has_data", "true"); -// pgMap.put("is_read_only", "true"); -// pgMap.put("username", "postgres"); -// pgMap.put("password", "postgres"); -// pgMap = Collections.unmodifiableMap(pgMap); -// conn.addStorageEngine( -// "127.0.0.1", -// 5432, -// StorageEngineType.postgresql, -// pgMap -// ); -// Map mongoMap = new HashMap<>(); -// mongoMap.put("has_data", "true"); -// mongoMap.put("is_read_only", "true"); -// mongoMap.put("schema.sample.size", "1000"); -// mongoMap.put("dummy.sample.size", "0"); -// conn.addStorageEngine( -// "127.0.0.1", -// 27017, -// StorageEngineType.mongodb, -// mongoMap -// ); -// System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); - // 输出所有存储引擎 String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); System.out.println(clusterInfo); + // 添加存储引擎 + System.out.println("start adding storage engine"); + long startTime = System.currentTimeMillis(); + Map pgMap = new HashMap<>(); + pgMap.put("has_data", "true"); + pgMap.put("is_read_only", "true"); + pgMap.put("username", "postgres"); + pgMap.put("password", "postgres"); + pgMap = Collections.unmodifiableMap(pgMap); + conn.addStorageEngine( + "127.0.0.1", + 5432, + StorageEngineType.postgresql, + pgMap + ); + Map mongoMap = new HashMap<>(); + mongoMap.put("has_data", "true"); + mongoMap.put("is_read_only", "true"); + mongoMap.put("schema.sample.size", "1000"); + mongoMap.put("dummy.sample.size", "0"); + conn.addStorageEngine( + "127.0.0.1", + 27017, + StorageEngineType.mongodb, + mongoMap + ); + System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + + // 输出所有存储引擎 + clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); + String sqlString = "select \n" + " nation.n_name, revenue\n" + "from (\n" + @@ -116,7 +116,7 @@ public void test() { // 开始 tpch 查询 System.out.println("start tpch query"); - long startTime = System.currentTimeMillis(); + startTime = System.currentTimeMillis(); // 执行查询语句 SessionExecuteSqlResult result = conn.executeSql(sqlString); @@ -139,7 +139,7 @@ public void test() { System.out.println("Answer number: " + answerNumber); assert nation.equals(answerString); - assert answerNumber - number < 1e-1 && number - answerNumber < 1e-1; + assert answerNumber - number < 1e-5 && number - answerNumber < 1e-5; } // 关闭会话 From 483375ec2510e1e3168cfa1c18d77318832c5974 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 14 May 2024 11:24:58 +0800 Subject: [PATCH 092/229] add tpch tests --- .github/scripts/benchmarks/tpch.sh | 4 +- .../polybench/TPCHDataInsertionIT.java | 10 +- .../test/resources/polybench/sf0.1/q01.csv | 5 + .../test/resources/polybench/sf0.1/q02.csv | 101 ++++++++++++++++++ .../test/resources/polybench/sf0.1/q05.csv | 10 +- .../test/resources/polybench/sf0.1/q06.csv | 2 + test/src/test/resources/tpchQueries/q06.sql | 12 +++ test/src/test/resources/tpchQueries/q1.sql | 31 ++++++ test/src/test/resources/tpchQueries/q2.sql | 43 ++++++++ test/src/test/resources/tpchQueries/q5.sql | 28 +++++ 10 files changed, 236 insertions(+), 10 deletions(-) create mode 100644 test/src/test/resources/polybench/sf0.1/q01.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q02.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q06.csv create mode 100644 test/src/test/resources/tpchQueries/q06.sql create mode 100644 test/src/test/resources/tpchQueries/q1.sql create mode 100644 test/src/test/resources/tpchQueries/q2.sql create mode 100644 test/src/test/resources/tpchQueries/q5.sql diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index f3a6de2584..622cb973e1 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -71,8 +71,6 @@ else ls echo "数据生成完成" - rm part.tbl - rm partsupp.tbl # 源文件夹路径 source_folder="." @@ -92,6 +90,8 @@ else chmod +r orders.tbl chmod +r region.tbl chmod +r supplier.tbl + chmod +r part.tbl + chmod +r partsupp.tbl ls -a pwd echo "文件移动完成" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index fa694b538a..084685725e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -233,9 +233,13 @@ public void insertDataIntoPostgreSQL() { "DROP TABLE IF EXISTS customer", "DROP TABLE IF EXISTS supplier", "DROP TABLE IF EXISTS region", - "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone CHAR(15), c_acctbal DECIMAL(15,2), c_mktsegment CHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", + "DROP TABLE IF EXISTS part", + "DROP TABLE IF EXISTS partsupp", + "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone VARCHAR(15), c_acctbal FLOAT, c_mktsegment VARCHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", - "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal DECIMAL(15,2), s_comment VARCHAR(101), s_dummy varchar(2), PRIMARY KEY (s_suppkey))", + "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal FLOAT, s_comment VARCHAR(101), s_dummy VARCHAR(2), PRIMARY KEY (s_suppkey))", + "CREATE TABLE IF NOT EXISTS part ( p_partkey INT, p_name VARCHAR(55), p_mfgr VARCHAR(25), p_brand VARCHAR(10), p_type VARCHAR(25), p_size INT, p_container VARCHAR(10), p_retailprice FLOAT , p_comment VARCHAR(23) , p_dummy VARCHAR(2), PRIMARY KEY (p_partkey))", + "CREATE TABLE IF NOT EXISTS partsupp ( ps_partkey INT, ps_suppkey INT, ps_availqty INT, ps_supplycost FLOAT, ps_comment VARCHAR(199), ps_dummy VARCHAR(2))", }; try (Connection conn = DriverManager.getConnection(url, user, password); @@ -249,7 +253,7 @@ public void insertDataIntoPostgreSQL() { System.out.println("Executed SQL statement: " + sql); } CopyManager copyManager = new CopyManager((BaseConnection) conn); - List tableNames = Arrays.asList("customer", "supplier", "region"); + List tableNames = Arrays.asList("customer", "supplier", "region", "part", "partsupp"); for (String tableName : tableNames) { String filePath = String.format("%s/%s.tbl", dataPath, tableName); FileReader fileReader = new FileReader(filePath); diff --git a/test/src/test/resources/polybench/sf0.1/q01.csv b/test/src/test/resources/polybench/sf0.1/q01.csv new file mode 100644 index 0000000000..ac95c90db3 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q01.csv @@ -0,0 +1,5 @@ +l_returnflag|l_linestatus|sum_qty|sum_base_price|sum_disc_price|sum_charge|avg_qty|avg_price|avg_disc|count_order +A|F|3774200|5320753880.69|5054096266.6828|5256751331.449234|25.537587116854997|36002.12382901414|0.05014459706340077|147790 +N|F|95257|133737795.84|127132372.6512|132286291.229445|25.30066401062417|35521.32691633466|0.04939442231075697|3765 +N|O|7459297|10512270008.90|9986238338.3847|10385578376.585467|25.545537671232875|36000.9246880137|0.05009595890410959|292000 +R|F|3785523|5337950526.47|5071818532.9420|5274405503.049367|25.5259438574251|35994.029214030925|0.04998927856184382|148301 diff --git a/test/src/test/resources/polybench/sf0.1/q02.csv b/test/src/test/resources/polybench/sf0.1/q02.csv new file mode 100644 index 0000000000..4a99144c7c --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q02.csv @@ -0,0 +1,101 @@ +s_acctbal|s_name|n_name|p_partkey|p_mfgr|s_address|s_phone|s_comment +9938.53|Supplier#000005359|UNITED KINGDOM|185358|Manufacturer#4|bgxj2K0w1kJvxYl5mhCfou,W|33-429-790-6131|l, ironic instructions cajole +9937.84|Supplier#000005969|ROMANIA|108438|Manufacturer#1|rdnmd9c8EG1EIAYY3LPVa4yUNx6OwyVaQ|29-520-692-3537|es. furiously silent deposits among the deposits haggle furiously a +9936.22|Supplier#000005250|UNITED KINGDOM|249|Manufacturer#4|qX AB0vP8mJEWeBuY9jri|33-320-228-2957|ar, regular requests nag blithely special accounts. final deposits impress carefully. ironic, +9923.77|Supplier#000002324|GERMANY|29821|Manufacturer#4|uXcnR7tv87dG|17-779-299-1839|s sleep according to the quick requests. carefully +9871.22|Supplier#000006373|GERMANY|43868|Manufacturer#5|iSLO35z7Ae|17-813-485-8637|against the slyly daring requests. unusual accounts wake atop the blithely spe +9870.78|Supplier#000001286|GERMANY|81285|Manufacturer#2|3gq0mZLHI5OTM6 tBYmLTHZaulCYnlECzQ7nj|17-516-924-4574|into beans haggle at the quickly final asymptotes. unusu +9870.78|Supplier#000001286|GERMANY|181285|Manufacturer#4|3gq0mZLHI5OTM6 tBYmLTHZaulCYnlECzQ7nj|17-516-924-4574|into beans haggle at the quickly final asymptotes. unusu +9852.52|Supplier#000008973|RUSSIA|18972|Manufacturer#2|zVfUT3Np22kUC05tYWHBotaR|32-188-594-7038|ly daring excuses unwind carefully above the fu +9847.83|Supplier#000008097|RUSSIA|130557|Manufacturer#2|veMRTQBmUResNvfD3|32-375-640-3593| slyly ironic, special requests. final instructions above the qu +9847.57|Supplier#000006345|FRANCE|86344|Manufacturer#1|68yX tGXAkVRSxUGNSjJdptw 8O878xaFnaoQK|16-886-766-7945|odolites. blithely special requests above the regular foxes sleep unusual sauternes. care +9847.57|Supplier#000006345|FRANCE|173827|Manufacturer#2|68yX tGXAkVRSxUGNSjJdptw 8O878xaFnaoQK|16-886-766-7945|odolites. blithely special requests above the regular foxes sleep unusual sauternes. care +9836.93|Supplier#000007342|RUSSIA|4841|Manufacturer#4|icFgTpZ0TuAm188dv|32-399-414-5385| packages are blithely about the quickly +9817.10|Supplier#000002352|RUSSIA|124815|Manufacturer#2|XfLCj71HKHnPqgvs7KNgPKcOWoWxo2w|32-551-831-1437|al packages doze always according to the quickly f +9817.10|Supplier#000002352|RUSSIA|152351|Manufacturer#3|XfLCj71HKHnPqgvs7KNgPKcOWoWxo2w|32-551-831-1437|al packages doze always according to the quickly f +9739.86|Supplier#000003384|FRANCE|138357|Manufacturer#2|D01XwXbcILNwmrGS6ZPrVhZxO40i|16-494-913-5925|es. carefully regular ideas cajole. quickly ironic requests haggle. pending sentiment +9721.95|Supplier#000008757|UNITED KINGDOM|156241|Manufacturer#3|ryKUkEeWN7Z|33-821-407-2995| the instructions breach slyly +9681.33|Supplier#000008406|RUSSIA|78405|Manufacturer#1|1A6x3PLy6F|32-139-873-8571|ons sleep express deposits. epitap +9643.55|Supplier#000005148|ROMANIA|107617|Manufacturer#1|H7WOI6lzFuSsWzTSBrhzTYV|29-252-617-4850|carefully platelets. packages sleep special ideas. quick +9624.82|Supplier#000001816|FRANCE|34306|Manufacturer#3|NTwQPSZwfhc4uu1EMvEDopBnEv2j P|16-392-237-6726| the express, regular accounts. regular decoys boost alongside of +9624.78|Supplier#000009658|ROMANIA|189657|Manufacturer#1|DmRxpLmL88XCBiONB3tq3e0u|29-748-876-2014|inst the blithely brave frays. brav +9612.94|Supplier#000003228|ROMANIA|120715|Manufacturer#2|hnNBdhdXO4yT18 QNABTrL8fuv0A4p|29-325-784-8187|furiously foxes. express packages nag blithely express, pending ideas. fluffily ironi +9612.94|Supplier#000003228|ROMANIA|198189|Manufacturer#4|hnNBdhdXO4yT18 QNABTrL8fuv0A4p|29-325-784-8187|furiously foxes. express packages nag blithely express, pending ideas. fluffily ironi +9571.83|Supplier#000004305|ROMANIA|179270|Manufacturer#2|Bdj1T5EostLveb9ocRbz|29-973-481-1831|fully: fluffily special deposits use fur +9558.10|Supplier#000003532|UNITED KINGDOM|88515|Manufacturer#4|ncMxIJcDYZd5B7FlKxxLmnlzPeZB,FKBujB|33-152-301-2164|against the final pinto beans. carefully bold asymptotes use +9492.79|Supplier#000005975|GERMANY|25974|Manufacturer#5|9UEiIp7uSYtTF5|17-992-579-4839|fluffily ironic instructions haggle against the even, special accounts. quickly final +9461.05|Supplier#000002536|UNITED KINGDOM|20033|Manufacturer#1|TEEkPusQ6rU18YvixE7IQtBDOyRBdGoOWl2r|33-556-973-5522|inal ideas cajole furiously. blithely special Tiresias against the b +9453.01|Supplier#000000802|ROMANIA|175767|Manufacturer#1|1Uj23QWxQjj7EyeqHWqGWTbN|29-342-882-6463|s according to the even deposits integrate express packages. express +9408.65|Supplier#000007772|UNITED KINGDOM|117771|Manufacturer#4|rIoV2rj0KNy,IT|33-152-491-1126|s nag quickly regular packages. carefully express pinto beans about th +9359.61|Supplier#000004856|ROMANIA|62349|Manufacturer#5|k2CKOmXhPruJZ|29-334-870-9731|es. final asymptotes wake carefully +9357.45|Supplier#000006188|UNITED KINGDOM|138648|Manufacturer#1|LS,Z0 zbSvC7GWjF|33-583-607-1633| somas cajole around the even, ironic deposits. pending theodolites according to the b +9352.04|Supplier#000003439|GERMANY|170921|Manufacturer#4|B2bnKDIpkJp2uHKp|17-128-996-4650|nusual frets cajole carefully beneath +9312.97|Supplier#000007807|RUSSIA|90279|Manufacturer#5|Dk2ebpGR3jlpYbpMg9Djr|32-673-872-5854|. silent asymptotes boost. quickly ironic accounts for the +9312.97|Supplier#000007807|RUSSIA|100276|Manufacturer#5|Dk2ebpGR3jlpYbpMg9Djr|32-673-872-5854|. silent asymptotes boost. quickly ironic accounts for the +9280.27|Supplier#000007194|ROMANIA|47193|Manufacturer#3|tJ96aHp8 l3uiq38LiDHswtk9bHMg|29-318-454-2133|tes. carefully regular accounts are carefully since the waters. accounts cajole? carefully bold +9274.80|Supplier#000008854|RUSSIA|76346|Manufacturer#3|,uJfCd6eTiYE1ZEhDM vsc8ANQPWaPlQ|32-524-148-5221|onic, final ideas. blithely regular platelets boost final, ironic pinto beans. fluffil +9249.35|Supplier#000003973|FRANCE|26466|Manufacturer#1|OZSkIozfU4FYizk4e091MZHozL1qcHe257J89bW|16-722-866-1658|beans. slyly ironic dependencies cajole furiously furiously regular ideas. boldly even requests hagg +9249.35|Supplier#000003973|FRANCE|33972|Manufacturer#1|OZSkIozfU4FYizk4e091MZHozL1qcHe257J89bW|16-722-866-1658|beans. slyly ironic dependencies cajole furiously furiously regular ideas. boldly even requests hagg +9208.70|Supplier#000007769|ROMANIA|40256|Manufacturer#5|AzIENtMrVCSbrjyUu8|29-964-424-9649| ironic requests among the deposits affix busily ironic accounts. slow pinto beans are blithely fi +9201.47|Supplier#000009690|UNITED KINGDOM|67183|Manufacturer#5|pprpD77FEIWsNMmGT9T|33-121-267-9529|uriously bold packages integrate blithely ironic theodolites. carefully unusual escap +9192.10|Supplier#000000115|UNITED KINGDOM|85098|Manufacturer#3|EhrYy0MT5M1vfZ0V4skpifdp6pgFz5|33-597-248-1220|onic instructions. ironic, regular deposits haggle f +9189.98|Supplier#000001226|GERMANY|21225|Manufacturer#4|BzfoA9wft1Mx3iBIs|17-725-903-1381|luffily across the slyly special instructions. bold, ironic deposi +9128.97|Supplier#000004311|RUSSIA|146768|Manufacturer#5|jSiHD4NTd8i9zVRX9uz9a,|32-155-440-7120|theodolites. furiously even pinto beans abou +9104.83|Supplier#000008520|GERMANY|150974|Manufacturer#4|aA95nLn,m9shRrPXZw9Y1X|17-728-804-1793|nstructions. carefully regular requests use fluffily against the quickly final deposits. blithel +9101.00|Supplier#000005791|ROMANIA|128254|Manufacturer#5|txPYsp50HJkbbaAJ0bYieqHmZtirDUVOcmC4lk|29-549-251-5384| regular foxes use carefully final packages. fluffily stealthy deposits toward the sp +9094.57|Supplier#000004582|RUSSIA|39575|Manufacturer#1|5p,3Gp8kX 1EDarE0JR5juHH Sq9jlxgKenM|32-587-577-1351|ages affix quickly after the carefully regular accounts. regular, regular foxes kindle. +8996.87|Supplier#000004702|FRANCE|102191|Manufacturer#5|T35OahYXQGC|16-811-269-8946|ily regular grouches wake quickly ironic de +8996.14|Supplier#000009814|ROMANIA|139813|Manufacturer#2|RL,cVCKSXFc6Win6EmtF415A22as8nG2fqEa|29-995-571-8781| regular requests haggle carefully above the carefully regular deposits. ironic pearls in p +8968.42|Supplier#000010000|ROMANIA|119999|Manufacturer#5|R7kfmyzoIfXlrbnqNwUUW3phJctocp0J|29-578-432-2146|ular, quick foxes sleep quickly according to the blithely fluffy theodolit +8936.82|Supplier#000007043|UNITED KINGDOM|109512|Manufacturer#1|m5QHON1iD1OPhmU2R3z97u 6mCIvjnAc3I0,9s|33-784-177-8208| final dependencies. deposits a +8929.42|Supplier#000008770|FRANCE|173735|Manufacturer#4|aTOkYV7y3 kqbRrkOGJLaI|16-242-746-9248|ns haggle quickly silent theodolites. bold, final requests along t +8920.59|Supplier#000003967|ROMANIA|26460|Manufacturer#1|NjCq3NUY82S|29-194-731-3944|ts. daringly regular theodolites affix silently. reg +8920.59|Supplier#000003967|ROMANIA|173966|Manufacturer#2|NjCq3NUY82S|29-194-731-3944|ts. daringly regular theodolites affix silently. reg +8913.96|Supplier#000004603|UNITED KINGDOM|137063|Manufacturer#2|d6sFwf6 TD1xyfuFbdM2h8LX7ZWc3zHupV|33-789-255-7342|lithely whithout the furiously ironic sheaves. ironic reques +8877.82|Supplier#000007967|FRANCE|167966|Manufacturer#5|rXBIZqq9eWEuU90B vlCab6|16-442-147-9345|ckages-- evenly even requests boost blit +8862.24|Supplier#000003323|ROMANIA|73322|Manufacturer#3|5RrF2PzoRlwpAGXjyf|29-736-951-3710|regular ideas haggle blithely packages. regula +8841.59|Supplier#000005750|ROMANIA|100729|Manufacturer#5|n uXFrKx,KVYIQjmRuV,yejWmLMdRJnk|29-344-502-5481|leep finally furiously express packages. slyly unusual packages cajole unusual, +8781.71|Supplier#000003121|ROMANIA|13120|Manufacturer#5|wdA7CLuYXS22oQEmP0V,x0PHrXiPdl5Rpwv,ub|29-707-291-5144|ies. final foxes are furiou +8754.24|Supplier#000009407|UNITED KINGDOM|179406|Manufacturer#4|pj9oPHQ4OLWp|33-903-970-9604|ng asymptotes hang across the blithely special deposits. +8691.06|Supplier#000004429|UNITED KINGDOM|126892|Manufacturer#2|H0paE V6JCrlZpYrzI0LgIP|33-964-337-5038| sly requests might sleep. final dolphins sleep. furiousl +8655.99|Supplier#000006330|RUSSIA|193810|Manufacturer#2|7CsFQnd ,tzgMYvVoMim5l4DrJcX8SaQMTcy|32-561-198-3705|ideas wake across the regular, unusual instructions; furiously final deposits wake near the s +8638.36|Supplier#000002920|RUSSIA|75398|Manufacturer#1|iMYQSQzsLXg|32-122-621-7549|ickly dolphins. furiously careful asymptotes sublate +8638.36|Supplier#000002920|RUSSIA|170402|Manufacturer#3|iMYQSQzsLXg|32-122-621-7549|ickly dolphins. furiously careful asymptotes sublate +8607.69|Supplier#000006003|UNITED KINGDOM|76002|Manufacturer#2|njRvqoOmIxNDe,da,SsnweINv1VY2YatifmJq|33-416-807-5206|braids sleep carefully along the iron +8569.52|Supplier#000005936|RUSSIA|5935|Manufacturer#5|I3Qd1VwvDm5hYGzg1hBHzKy,P3YQXq7|32-644-251-7916|s about the carefully final accounts use always even requests. furiously express dependenc +8564.12|Supplier#000000033|GERMANY|110032|Manufacturer#1|LLMgB3vXW,0g,8nuv3qU3QZaEBZvU2qRLX9|17-138-897-9374|l packages cajole unusual, final packages. slyly express requests +8553.82|Supplier#000003979|ROMANIA|143978|Manufacturer#4|qLE5JpqDoe3XHsBI6etWpd4zRsjsBNb9Tidi6|29-124-646-4897|counts are quickly carefully ironic instructions. platelets wake f +8517.23|Supplier#000009529|RUSSIA|37025|Manufacturer#5|NWW9SDThqi9RIeOA|32-565-297-8775|ial requests use stealthily along the carefully u +8517.23|Supplier#000009529|RUSSIA|59528|Manufacturer#2|NWW9SDThqi9RIeOA|32-565-297-8775|ial requests use stealthily along the carefully u +8503.70|Supplier#000006830|RUSSIA|44325|Manufacturer#4|qoW4lp2961uQiKOK6rW8|32-147-878-5069|atelets sleep furiously pending asymptotes. even requests for the blithely unusual packages +8457.09|Supplier#000009456|UNITED KINGDOM|19455|Manufacturer#1|U8pJ1 SKbZPhH7,bLWXX3pG|33-858-440-4349|ounts sleep about the bold, even ideas. slyly unusual accounts after the asymptotes +8441.40|Supplier#000003817|FRANCE|141302|Manufacturer#2|K6XLsYufTS|16-339-356-5115|sly fluffily regular pinto beans. slyly even deposits snooze fluffily along the fluff +8432.89|Supplier#000003990|RUSSIA|191470|Manufacturer#1|wMJppCZ9aPMuq2nr88TVfztvE gj95OG wdNUE|32-839-509-9301|. express pinto beans use slyly. regular platelets sleep quickly busy deposits. final +8431.40|Supplier#000002675|ROMANIA|5174|Manufacturer#1|khl8ydxR9VekbcMLgJKPtpNtwAkYtJTv|29-474-643-1443|regular, express platelets are. carefully ironic forges since the requests affix +8407.04|Supplier#000005406|RUSSIA|162889|Manufacturer#4|ITrK2mV94SooV6 Igo|32-626-152-4621| even theodolites. quickly bold deposits after the pen +8386.08|Supplier#000008518|FRANCE|36014|Manufacturer#3|ZHAsABq5MRP e5kc0DRD8za3xGdf763ChHmoOA45|16-618-780-7481|g alongside of the slyly unusual platelets! blithely regular asymptotes cajole. quickly regular +8376.52|Supplier#000005306|UNITED KINGDOM|190267|Manufacturer#5|SyS2SsaA8i CqnbzUdfNH07bVtt9uW,Cp6FLCkOR|33-632-514-7931|pendencies affix furiously against the special, blithe packages. qui +8348.74|Supplier#000008851|FRANCE|66344|Manufacturer#4|E4uITlvmPHKvZ|16-796-240-2472|s packages haggle above the express pinto beans. stealthy, ironic theodolites sleep quickly +8338.58|Supplier#000007269|FRANCE|17268|Manufacturer#4|2vJh8wqp6CJp,W0Y|16-267-277-4365|lithely through the accounts. express, ironic asymptotes wou +8328.46|Supplier#000001744|ROMANIA|69237|Manufacturer#5|DfCXL6UWAY1lgjQYB0AjE8T2sx0BzS|29-330-728-5873| regular, special dolphins haggle carefully special asy +8307.93|Supplier#000003142|GERMANY|18139|Manufacturer#1|OAPFw6SNodrC kFi|17-595-447-6026|usly express packages sleep finally regular ideas. carefu +8231.61|Supplier#000009558|RUSSIA|192000|Manufacturer#2|FONKME0t7ZJhnjn9VL5|32-762-137-5858|g to the carefully even brai +8152.61|Supplier#000002731|ROMANIA|15227|Manufacturer#4|sDFx3iox2Zzx|29-805-463-2030|ly above the packages. final accounts sleep furiously. fluffily iro +8109.09|Supplier#000009186|FRANCE|99185|Manufacturer#1|wKLCzA5bMuGRBm35tvQAGpen23L|16-668-570-1402|ts cajole daringly. pinto beans +8102.62|Supplier#000003347|UNITED KINGDOM|18344|Manufacturer#5|Froy39Y8ZUJ|33-454-274-8532|y daring requests. unusual accounts wake atop the blithely special packages. sly +8046.07|Supplier#000008780|FRANCE|191222|Manufacturer#3|rOssxn,6gRDzHr0gu,hEK|16-473-215-6395|he regular foxes cajole ruthlessly among the sometimes final grouches. blithel +8042.09|Supplier#000003245|RUSSIA|135705|Manufacturer#4|oJSiGLXRCDAPcfWot7LkwSQRCh63XNS2|32-836-132-8872| use slyly. furiously regular deposits sleep according to the requests. +8042.09|Supplier#000003245|RUSSIA|150729|Manufacturer#1|oJSiGLXRCDAPcfWot7LkwSQRCh63XNS2|32-836-132-8872| use slyly. furiously regular deposits sleep according to the requests. +7992.40|Supplier#000006108|FRANCE|118574|Manufacturer#1|TyptNE7nv6BLpLl6WFX|16-974-998-8937|theodolites among the furiously unusual accounts must x +7980.65|Supplier#000001288|FRANCE|13784|Manufacturer#4|tm0TjL5b oE|16-646-464-8247|gular pains? fluffily bold warhorses affix? blithe instruction +7950.37|Supplier#000008101|GERMANY|33094|Manufacturer#5|HG2wfVixwCIhK7dlrigGR3an2LuSifDJH|17-627-663-8014|ly alongside of the furiously unusual requests! bold, express foxe +7937.93|Supplier#000009012|ROMANIA|83995|Manufacturer#2|J6I7sJj0mGYIWFv9KxD3fK O7tvNP|29-250-925-9690| use slyly against the slyly bold theod +7914.45|Supplier#000001013|RUSSIA|125988|Manufacturer#2|AI9ODzBzWgnny28PHBei5M2lUFdD9|32-194-698-3365| the blithely silent accounts. q +7912.91|Supplier#000004211|GERMANY|159180|Manufacturer#5|Zva95Dwj EY0w,XjgsL7O0Zb2 l3almck|17-266-947-7315| slyly silent requests; fluffily fi +7912.91|Supplier#000004211|GERMANY|184210|Manufacturer#4|Zva95Dwj EY0w,XjgsL7O0Zb2 l3almck|17-266-947-7315| slyly silent requests; fluffily fi +7894.56|Supplier#000007981|GERMANY|85472|Manufacturer#4|e8hRUxe9cqQM3b|17-963-404-3760|ly final courts. unusual, quiet dolphi +7887.08|Supplier#000009792|GERMANY|164759|Manufacturer#3|3YSi76M2 I8XGikO5YgSM81r5Z6A7VkZcys|17-988-938-4296| the regular ideas. furiously bold deposits boost above the bli +7871.50|Supplier#000007206|RUSSIA|104695|Manufacturer#1|YvrLdpD 5ExhHmRWzK41tw4|32-432-452-7731|ording to the furious theodolites cajole carefully according to the busily express asymptotes. +7852.45|Supplier#000005864|RUSSIA|8363|Manufacturer#4|5odLpc1M83KXJ0O|32-454-883-3821|egular, regular ideas. requests are carefully. furiously final dolp +7850.66|Supplier#000001518|UNITED KINGDOM|86501|Manufacturer#1|ddNQX3hIjgico|33-730-383-3892|ccounts. special, final deposits +7843.52|Supplier#000006683|FRANCE|11680|Manufacturer#4|Z1,hkHIw,Z3,,Comv6kLxIiPJtoNt|16-464-517-8943|sits. blithely regular requests above the pending, regular ideas boo diff --git a/test/src/test/resources/polybench/sf0.1/q05.csv b/test/src/test/resources/polybench/sf0.1/q05.csv index 1be1eb1d27..b34430123f 100644 --- a/test/src/test/resources/polybench/sf0.1/q05.csv +++ b/test/src/test/resources/polybench/sf0.1/q05.csv @@ -1,6 +1,6 @@ n_name|revenue -"CHINA"|7822103.0000 -"INDIA"|6376121.5085 -"JAPAN"|6000077.2184 -"INDONESIA"|5580475.4027 -"VIETNAM"|4497840.5466 +CHINA|7822103.0000 +INDIA|6376121.5085 +JAPAN|6000077.2184 +INDONESIA|5580475.4027 +VIETNAM|4497840.5466 diff --git a/test/src/test/resources/polybench/sf0.1/q06.csv b/test/src/test/resources/polybench/sf0.1/q06.csv new file mode 100644 index 0000000000..55f4b53b2b --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q06.csv @@ -0,0 +1,2 @@ +revenue +11803420.2534 diff --git a/test/src/test/resources/tpchQueries/q06.sql b/test/src/test/resources/tpchQueries/q06.sql new file mode 100644 index 0000000000..5408a94ff8 --- /dev/null +++ b/test/src/test/resources/tpchQueries/q06.sql @@ -0,0 +1,12 @@ +select sum(tmp) from ( + select + l_extendedprice * l_discount as tmp + from + mongotpch.lineitem + where + mongotpch.lineitem.l_shipdate >= 757353600000 + and mongotpch.lineitem.l_shipdate < 788889600000 + and mongotpch.lineitem.l_discount >= 0.05 + and mongotpch.lineitem.l_discount <= 0.07 + and mongotpch.lineitem.l_quantity < 24 + ); diff --git a/test/src/test/resources/tpchQueries/q1.sql b/test/src/test/resources/tpchQueries/q1.sql new file mode 100644 index 0000000000..d9e6575ed7 --- /dev/null +++ b/test/src/test/resources/tpchQueries/q1.sql @@ -0,0 +1,31 @@ +select + mongotpch.lineitem.l_returnflag as l_returnflag, + mongotpch.lineitem.l_linestatus as l_linestatus, + sum(mongotpch.lineitem.l_quantity) as sum_qty, + sum(mongotpch.lineitem.l_extendedprice) as sum_base_price, + sum(tmp1) as sum_disc_price, + sum(tmp2) as sum_charge, + avg(mongotpch.lineitem.l_quantity) as avg_qty, + avg(mongotpch.lineitem.l_extendedprice) as avg_price, + avg(mongotpch.lineitem.l_discount) as avg_disc, + count(*) as count_order +from ( + select + l_returnflag, + l_linestatus, + l_quantity, + l_extendedprice, + l_discount, + l_extendedprice * (1 - l_discount) as tmp1, + l_extendedprice * (1 - l_discount) * (1 + l_tax) as tmp2 + from + mongotpch.lineitem + where + mongotpch.lineitem.l_shipdate <= 904665600000 + ) +group by + mongotpch.lineitem.l_returnflag, + mongotpch.lineitem.l_linestatus +order by + mongotpch.lineitem.l_returnflag, + mongotpch.lineitem.l_linestatus; \ No newline at end of file diff --git a/test/src/test/resources/tpchQueries/q2.sql b/test/src/test/resources/tpchQueries/q2.sql new file mode 100644 index 0000000000..c2540ecd60 --- /dev/null +++ b/test/src/test/resources/tpchQueries/q2.sql @@ -0,0 +1,43 @@ +insert into tmpTable(key, p_key, minCost) values (select + postgres.part.p_partkey as p_key, + min(postgres.partsupp.ps_supplycost) as minCost + from + postgres.partsupp + join postgres.supplier on postgres.supplier.s_suppkey = postgres.partsupp.ps_suppkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join postgres.region on nation.n_regionkey = postgres.region.r_regionkey + join postgres.part on postgres.part.p_partkey = postgres.partsupp.ps_partkey + where + postgres.region.r_name = 'EUROPE' + and postgres.part.p_size = 15 + and postgres.part.p_type like "^.*BRASS" + group by postgres.part.p_partkey); + + +select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from + (select + postgres.supplier.s_acctbal as s_acctbal, + postgres.supplier.s_name as s_name, + nation.n_name as n_name, + postgres.part.p_partkey as p_partkey, + postgres.part.p_mfgr as p_mfgr, + postgres.supplier.s_address as s_address, + postgres.supplier.s_phone as s_phone, + postgres.supplier.s_comment as s_comment + from + postgres.part + join postgres.partsupp on postgres.part.p_partkey = postgres.partsupp.ps_partkey + join postgres.supplier on postgres.supplier.s_suppkey = postgres.partsupp.ps_suppkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join postgres.region on nation.n_regionkey = postgres.region.r_regionkey + join tmpTable on tmpTable.p_key = postgres.part.p_partkey and postgres.partsupp.ps_supplycost = tmpTable.minCost + where + postgres.part.p_size = 15 + and postgres.region.r_name = 'EUROPE' + and postgres.part.p_type like "^.*BRASS" + order by + nation.n_name, + postgres.supplier.s_name, + postgres.part.p_partkey + ) +order by s_acctbal desc; \ No newline at end of file diff --git a/test/src/test/resources/tpchQueries/q5.sql b/test/src/test/resources/tpchQueries/q5.sql new file mode 100644 index 0000000000..2a04d7c002 --- /dev/null +++ b/test/src/test/resources/tpchQueries/q5.sql @@ -0,0 +1,28 @@ +select + nation.n_name, + revenue +from ( + select + nation.n_name, + sum(tmp) as revenue + from ( + select + nation.n_name, + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp + from + postgres.customer + join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey + join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join postgres.region on nation.n_regionkey = postgres.region.r_regionkey + where + postgres.region.r_name = "ASIA" + and mongotpch.orders.o_orderdate >= 757353600000 + and mongotpch.orders.o_orderdate < 788889600000 + ) + group by + nation.n_name + ) +order by + revenue desc; \ No newline at end of file From ff02b6fb8908c8064cf377f76a8123275d0b0315 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 16 May 2024 09:10:26 +0800 Subject: [PATCH 093/229] add tpch tests --- .../{tpchQueries => polybench/queries}/q1.sql | 0 .../test/resources/polybench/queries/q10.sql | 51 +++++++++++++++++++ .../test/resources/polybench/queries/q13.sql | 25 +++++++++ .../test/resources/polybench/queries/q17.sql | 29 +++++++++++ .../test/resources/polybench/queries/q18.sql | 36 +++++++++++++ .../{tpchQueries => polybench/queries}/q2.sql | 0 .../{tpchQueries => polybench/queries}/q5.sql | 0 .../q06.sql => polybench/queries/q6.sql} | 0 .../test/resources/polybench/sf0.1/q05.csv | 10 ++-- .../test/resources/polybench/sf0.1/q10.csv | 21 ++++++++ .../test/resources/polybench/sf0.1/q13.csv | 38 ++++++++++++++ .../test/resources/polybench/sf0.1/q17.csv | 2 + .../test/resources/polybench/sf0.1/q18.sql | 33 ++++++++++++ .../polybench/udf/udtf_extract_year.py | 25 +++++++++ 14 files changed, 265 insertions(+), 5 deletions(-) rename test/src/test/resources/{tpchQueries => polybench/queries}/q1.sql (100%) create mode 100644 test/src/test/resources/polybench/queries/q10.sql create mode 100644 test/src/test/resources/polybench/queries/q13.sql create mode 100644 test/src/test/resources/polybench/queries/q17.sql create mode 100644 test/src/test/resources/polybench/queries/q18.sql rename test/src/test/resources/{tpchQueries => polybench/queries}/q2.sql (100%) rename test/src/test/resources/{tpchQueries => polybench/queries}/q5.sql (100%) rename test/src/test/resources/{tpchQueries/q06.sql => polybench/queries/q6.sql} (100%) create mode 100644 test/src/test/resources/polybench/sf0.1/q10.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q13.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q17.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q18.sql create mode 100644 test/src/test/resources/polybench/udf/udtf_extract_year.py diff --git a/test/src/test/resources/tpchQueries/q1.sql b/test/src/test/resources/polybench/queries/q1.sql similarity index 100% rename from test/src/test/resources/tpchQueries/q1.sql rename to test/src/test/resources/polybench/queries/q1.sql diff --git a/test/src/test/resources/polybench/queries/q10.sql b/test/src/test/resources/polybench/queries/q10.sql new file mode 100644 index 0000000000..f18acba927 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q10.sql @@ -0,0 +1,51 @@ +select + postgres.customer.c_custkey, + postgres.customer.c_name, + revenue, + postgres.customer.c_acctbal, + nation.n_name, + postgres.customer.c_address, + postgres.customer.c_phone, + postgres.customer.c_comment +from ( + select + postgres.customer.c_custkey, + postgres.customer.c_name, + sum(tmp) as revenue, + postgres.customer.c_acctbal, + nation.n_name, + postgres.customer.c_address, + postgres.customer.c_phone, + postgres.customer.c_comment + from ( + select + postgres.customer.c_custkey, + postgres.customer.c_name, + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp, + postgres.customer.c_acctbal, + nation.n_name, + postgres.customer.c_address, + postgres.customer.c_phone, + postgres.customer.c_comment + from + postgres.customer + join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey + join nation on postgres.customer.c_nationkey = nation.n_nationkey + where + mongotpch.orders.o_orderdate >= 749404800000 + and mongotpch.orders.o_orderdate < 757353600000 + and mongotpch.lineitem.l_returnflag = 'R' + ) + group by + postgres.customer.c_custkey, + postgres.customer.c_name, + postgres.customer.c_acctbal, + postgres.customer.c_phone, + nation.n_name, + postgres.customer.c_address, + postgres.customer.c_comment + ) +order by + revenue desc +limit 20; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q13.sql b/test/src/test/resources/polybench/queries/q13.sql new file mode 100644 index 0000000000..f961cff7fd --- /dev/null +++ b/test/src/test/resources/polybench/queries/q13.sql @@ -0,0 +1,25 @@ +select + c_count, + custdist +from ( + select + c_count, + count(c_custkey) as custdist + from ( + select + postgres.customer.c_custkey as c_custkey, + count(mongotpch.orders.o_orderkey) as c_count + from + postgres.customer + left outer join mongotpch.orders + on postgres.customer.c_custkey = mongotpch.orders.o_custkey + and !(mongotpch.orders.o_comment like '.*pending.*') + group by + postgres.customer.c_custkey + ) + group by + c_count + ) +order by + custdist, + c_count desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q17.sql b/test/src/test/resources/polybench/queries/q17.sql new file mode 100644 index 0000000000..b7c3983262 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q17.sql @@ -0,0 +1,29 @@ +insert into tmpTable(key, p_partkey, val) values ( + select + postgres.part.p_partkey, + 0.2 * tmp + from ( + select + postgres.part.p_partkey, + avg(mongotpch.lineitem.l_quantity) as tmp + from + mongotpch.lineitem + join postgres.part on mongotpch.lineitem.l_partkey = postgres.part.p_partkey + group by postgres.part.p_partkey + ) +) + +select + tmp2 / 7 as avg_yearly +from ( + select + sum(mongotpch.lineitem.l_extendedprice) as tmp2 + from + mongotpch.lineitem + join postgres.part on postgres.part.p_partkey = mongotpch.lineitem.l_partkey + join tmpTable on tmpTable.p_partkey = mongotpch.lineitem.l_partkey + where + postgres.part.p_brand = 'Brand#23' + and postgres.part.p_container = 'MED BOX' + and mongotpch.lineitem.l_quantity < tmpTable.val + ); \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q18.sql b/test/src/test/resources/polybench/queries/q18.sql new file mode 100644 index 0000000000..ca07e510e2 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q18.sql @@ -0,0 +1,36 @@ +select + postgres.customer.c_name, + postgres.customer.c_custkey, + mongotpch.orders.o_orderkey, + mongotpch.orders.o_orderdate, + mongotpch.orders.o_totalprice, + sum(mongotpch.lineitem.l_quantity) +from + postgres.customer + join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + join mongotpch.lineitem on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey +where + mongotpch.orders.o_orderkey in ( + select + mongotpch.lineitem.l_orderkey + from ( + select + l_orderkey, + sum(l_quantity) + from + mongotpch.lineitem + group by + l_orderkey + having + sum(mongotpch.lineitem.l_quantity) > 300 + ) + ) +group by + postgres.customer.c_name, + postgres.customer.c_custkey, + mongotpch.orders.o_orderkey, + mongotpch.orders.o_orderdate, + mongotpch.orders.o_totalprice +order by + mongotpch.orders.o_orderdate, + mongotpch.orders.o_totalprice desc; \ No newline at end of file diff --git a/test/src/test/resources/tpchQueries/q2.sql b/test/src/test/resources/polybench/queries/q2.sql similarity index 100% rename from test/src/test/resources/tpchQueries/q2.sql rename to test/src/test/resources/polybench/queries/q2.sql diff --git a/test/src/test/resources/tpchQueries/q5.sql b/test/src/test/resources/polybench/queries/q5.sql similarity index 100% rename from test/src/test/resources/tpchQueries/q5.sql rename to test/src/test/resources/polybench/queries/q5.sql diff --git a/test/src/test/resources/tpchQueries/q06.sql b/test/src/test/resources/polybench/queries/q6.sql similarity index 100% rename from test/src/test/resources/tpchQueries/q06.sql rename to test/src/test/resources/polybench/queries/q6.sql diff --git a/test/src/test/resources/polybench/sf0.1/q05.csv b/test/src/test/resources/polybench/sf0.1/q05.csv index b34430123f..1be1eb1d27 100644 --- a/test/src/test/resources/polybench/sf0.1/q05.csv +++ b/test/src/test/resources/polybench/sf0.1/q05.csv @@ -1,6 +1,6 @@ n_name|revenue -CHINA|7822103.0000 -INDIA|6376121.5085 -JAPAN|6000077.2184 -INDONESIA|5580475.4027 -VIETNAM|4497840.5466 +"CHINA"|7822103.0000 +"INDIA"|6376121.5085 +"JAPAN"|6000077.2184 +"INDONESIA"|5580475.4027 +"VIETNAM"|4497840.5466 diff --git a/test/src/test/resources/polybench/sf0.1/q10.csv b/test/src/test/resources/polybench/sf0.1/q10.csv new file mode 100644 index 0000000000..79f006ddb9 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q10.csv @@ -0,0 +1,21 @@ +c_custkey|c_name|revenue|c_acctbal|n_name|c_address|c_phone|c_comment +8242|Customer#000008242|622786.7297|6322.09|ETHIOPIA|cYDWDiJt06B8CYzXX2L8x2hn1VFG|15-792-676-1184| regular theodolites affix. carefully ironic packages cajole deposits; slyly ironic packages wake quickly. regular, +7714|Customer#000007714|557400.3053|9799.98|IRAN|9DDikq08GEE4B3X|20-922-418-6024|even accounts should cajole. regular, regular +11032|Customer#000011032|512500.9641|8496.93|UNITED KINGDOM|5igjoUgXoDUZVUIectL5lXO1T3AGKza0ft|33-102-772-3533|uests. ironic accounts after the fluffily fi +2455|Customer#000002455|481592.4053|2070.99|GERMANY|a5DZ199yfAcFhfi2uwBE PKo,Z|17-946-225-9977|pinto beans alongside of the furiously ironic asymptotes are quickly even platelets: express +12106|Customer#000012106|479414.2133|5342.11|UNITED STATES|wyJXywcExUxt|34-905-346-4472|blithely blithely final attainments? carefully special pinto beans around the quickly even asymptote +8530|Customer#000008530|457855.9467|9734.95|MOROCCO|leatyNRWCnfTMnTNuDGHsWJjRuAX|25-736-932-5850| the carefully pending packages. carefully +13984|Customer#000013984|446316.5104|3482.28|IRAN|B13vxRBojwvP3|20-981-264-2952|egular, ironic accounts integrate sly +1966|Customer#000001966|444059.0382|1937.72|ALGERIA|IbwZr7j QVifqf9WizOIWx,UXV9CqxUyrwj|10-973-269-8886|odolites across the unusual accounts hang carefully furiously bold excuses. regular pi +11026|Customer#000011026|417913.4142|7738.76|ALGERIA|4C iGzChcqnhGBdeeu|10-184-163-4632|eposits cajole according to the furiously bold instructions. regular, regular dependencies wake carefully eve +8501|Customer#000008501|412797.5100|6906.70|ARGENTINA|UTUQLX cQrF1UUJPsz|11-317-552-5840| packages. pending Tiresias after the regularly express forges haggle fina +1565|Customer#000001565|412506.0062|1820.03|BRAZIL|n4acVpG0Deyj5aIFAfSNg Iu9cUagwN3OsRbKC 4|12-402-178-2007|deposits; unusual, bold deposits around the f +14398|Customer#000014398|408575.3600|-602.24|UNITED STATES|l49oKjbjQHz6YZwjo5wPihM lyYO6G|34-814-111-5424|es haggle fluffily blithely fluffy requests; slyly express req +1465|Customer#000001465|405055.3457|9365.93|INDIA|zn9Q7pT6KlQp3T5mUO533aq,|18-807-487-1074|ress ideas cajole. slyly unusual theodolites cajole thin foxes. account +12595|Customer#000012595|401402.2391|-6.92|INDIA|gEMQ3WO90vSdAgxLFrt9FRS|18-186-132-3352| slyly dogged excuses. blithely blithe packages cajole +961|Customer#000000961|401198.1737|6963.68|JAPAN|W0SZ2oflx9aWTggtwSk3OEIXsubXTbGbD|22-989-463-6089|use furiously across the final deposits. quickly +14299|Customer#000014299|400968.3751|6595.97|RUSSIA|UFlOs8tQ,IfZPJm57|32-156-618-1224|slyly. ironic, bold deposits sleep blithely ironic, pending attainm +623|Customer#000000623|399883.4257|7887.60|INDONESIA|k3IlPSC4FKB13 hc6omhVs1ibvqeWEV|19-113-202-7085|se around the ideas. accounts cajole blithely slyly ironic requests. b +9151|Customer#000009151|396562.0295|5691.95|IRAQ|UKiN9OQupR,m5NtvSntbI8JBeo|21-834-147-4906|the deposits. pending, ironic foxes haggle along the regular, bold req +14819|Customer#000014819|396271.1036|7308.39|FRANCE|wS8yiQtE63FfoO6RKUzuVf6iBTmXBq16u|16-769-398-7926|ending asymptotes use fluffily quickly bold instructions. slyly bold dependencies sleep carefully pending a +13478|Customer#000013478|395513.1358|-778.11|KENYA|S5izwjM1 hCoUccO2JMepYwNyBSqI,ay|24-983-202-8240| requests boost quickly according to the express sheaves. blithely unusual packages sleep diff --git a/test/src/test/resources/polybench/sf0.1/q13.csv b/test/src/test/resources/polybench/sf0.1/q13.csv new file mode 100644 index 0000000000..d3f05df8a6 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q13.csv @@ -0,0 +1,38 @@ +c_count|custdist +0|5000 +9|659 +10|658 +11|643 +8|555 +12|542 +13|508 +7|494 +19|471 +20|464 +14|451 +17|449 +18|448 +15|446 +16|425 +21|406 +22|351 +6|334 +23|331 +24|278 +5|197 +25|184 +26|175 +27|136 +4|90 +28|86 +29|63 +3|45 +30|36 +31|26 +32|13 +2|12 +33|11 +34|5 +35|4 +36|2 +1|2 diff --git a/test/src/test/resources/polybench/sf0.1/q17.csv b/test/src/test/resources/polybench/sf0.1/q17.csv new file mode 100644 index 0000000000..f30f7424e6 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q17.csv @@ -0,0 +1,2 @@ +avg_yearly +23512.752857142856 diff --git a/test/src/test/resources/polybench/sf0.1/q18.sql b/test/src/test/resources/polybench/sf0.1/q18.sql new file mode 100644 index 0000000000..3f6886a48b --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q18.sql @@ -0,0 +1,33 @@ +SELECT + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +FROM + customer, + orders, + lineitem +WHERE + o_orderkey IN ( + SELECT + l_orderkey + FROM + lineitem + GROUP BY + l_orderkey + HAVING + sum(l_quantity) > 300) + AND c_custkey = o_custkey + AND o_orderkey = l_orderkey +GROUP BY + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +ORDER BY + o_totalprice DESC, + o_orderdate +LIMIT 100; diff --git a/test/src/test/resources/polybench/udf/udtf_extract_year.py b/test/src/test/resources/polybench/udf/udtf_extract_year.py new file mode 100644 index 0000000000..332bfe1551 --- /dev/null +++ b/test/src/test/resources/polybench/udf/udtf_extract_year.py @@ -0,0 +1,25 @@ +from datetime import datetime +class UDFExtractYear: + def __init__(self): + pass + + def extractYear(self, num): + # Unix timestamp is in milliseconds + timestamp_in_seconds = num / 1000 + dt = datetime.utcfromtimestamp(timestamp_in_seconds) + return float(dt.year) + def transform(self, data, args, kvargs): + res = self.buildHeader(data) + dateRow = [] + for num in data[2][1:]: + dateRow.append(self.extractYear(num)) + res.append(dateRow) + return res + + def buildHeader(self, data): + colNames = [] + colTypes = [] + for name in data[0][1:]: + colNames.append("cos(" + name + ")") + colTypes.append("DOUBLE") + return [colNames, colTypes] From 16e5a78f2f681627260f15bf2b7257e6e4753b4b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 16 May 2024 17:46:31 +0800 Subject: [PATCH 094/229] add tpch tests --- .../integration/polybench/TPCHRunner.java | 180 +++++++++--------- .../test/resources/polybench/queries/q17.sql | 8 +- .../test/resources/polybench/queries/q18.sql | 1 - .../test/resources/polybench/queries/q19.sql | 48 +++++ .../test/resources/polybench/queries/q2.sql | 31 +-- .../test/resources/polybench/queries/q20.sql | 45 +++++ .../test/resources/polybench/sf0.1/q02.csv | 101 ---------- .../test/resources/polybench/sf0.1/q05.csv | 6 - .../polybench/sf0.1/{q01.csv => q1.csv} | 0 .../test/resources/polybench/sf0.1/q18.csv | 6 + .../test/resources/polybench/sf0.1/q18.sql | 33 ---- .../test/resources/polybench/sf0.1/q19.csv | 2 + .../src/test/resources/polybench/sf0.1/q2.csv | 45 +++++ .../test/resources/polybench/sf0.1/q20.csv | 10 + .../src/test/resources/polybench/sf0.1/q5.csv | 6 + .../polybench/sf0.1/{q06.csv => q6.csv} | 0 16 files changed, 275 insertions(+), 247 deletions(-) create mode 100644 test/src/test/resources/polybench/queries/q19.sql create mode 100644 test/src/test/resources/polybench/queries/q20.sql delete mode 100644 test/src/test/resources/polybench/sf0.1/q02.csv delete mode 100644 test/src/test/resources/polybench/sf0.1/q05.csv rename test/src/test/resources/polybench/sf0.1/{q01.csv => q1.csv} (100%) create mode 100644 test/src/test/resources/polybench/sf0.1/q18.csv delete mode 100644 test/src/test/resources/polybench/sf0.1/q18.sql create mode 100644 test/src/test/resources/polybench/sf0.1/q19.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q2.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q20.csv create mode 100644 test/src/test/resources/polybench/sf0.1/q5.csv rename test/src/test/resources/polybench/sf0.1/{q06.csv => q6.csv} (100%) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index bd4e220f40..0f203c252f 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -7,6 +7,8 @@ import cn.edu.tsinghua.iginx.thrift.*; import org.junit.Test; +import java.io.BufferedReader; +import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; @@ -41,6 +43,17 @@ private List> csvReader (String filePath) { return data; } + public static String readSqlFileAsString(String filePath) throws IOException { + StringBuilder contentBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = br.readLine()) != null) { + contentBuilder.append(line).append("\n"); + } + } + return contentBuilder.toString(); + } + @Test public void test() { System.out.println("start"); @@ -50,103 +63,96 @@ public void test() { new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); conn.openSession(); - // 输出所有存储引擎 - String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); - - // 添加存储引擎 - System.out.println("start adding storage engine"); - long startTime = System.currentTimeMillis(); - Map pgMap = new HashMap<>(); - pgMap.put("has_data", "true"); - pgMap.put("is_read_only", "true"); - pgMap.put("username", "postgres"); - pgMap.put("password", "postgres"); - pgMap = Collections.unmodifiableMap(pgMap); - conn.addStorageEngine( - "127.0.0.1", - 5432, - StorageEngineType.postgresql, - pgMap - ); - Map mongoMap = new HashMap<>(); - mongoMap.put("has_data", "true"); - mongoMap.put("is_read_only", "true"); - mongoMap.put("schema.sample.size", "1000"); - mongoMap.put("dummy.sample.size", "0"); - conn.addStorageEngine( - "127.0.0.1", - 27017, - StorageEngineType.mongodb, - mongoMap - ); - System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); - - // 输出所有存储引擎 - clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); +// // 输出所有存储引擎 +// String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); +// System.out.println(clusterInfo); +// +// // 添加存储引擎 +// System.out.println("start adding storage engine"); +// long startTime = System.currentTimeMillis(); +// Map pgMap = new HashMap<>(); +// pgMap.put("has_data", "true"); +// pgMap.put("is_read_only", "true"); +// pgMap.put("username", "postgres"); +// pgMap.put("password", "postgres"); +// pgMap = Collections.unmodifiableMap(pgMap); +// conn.addStorageEngine( +// "127.0.0.1", +// 5432, +// StorageEngineType.postgresql, +// pgMap +// ); +// Map mongoMap = new HashMap<>(); +// mongoMap.put("has_data", "true"); +// mongoMap.put("is_read_only", "true"); +// mongoMap.put("schema.sample.size", "1000"); +// mongoMap.put("dummy.sample.size", "0"); +// conn.addStorageEngine( +// "127.0.0.1", +// 27017, +// StorageEngineType.mongodb, +// mongoMap +// ); +// System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); +// +// // 输出所有存储引擎 +// clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); +// System.out.println(clusterInfo); + Long startTime; - String sqlString = "select \n" + - " nation.n_name, revenue\n" + - "from (\n" + - " select\n" + - " nation.n_name,\n" + - " sum(tmp) as revenue\n" + - " from (\n" + - " select\n" + - " nation.n_name,\n" + - " mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp\n" + - " from\n" + - " postgres.customer\n" + - " join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey\n" + - " join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey\n" + - " join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey\n" + - " join nation on postgres.supplier.s_nationkey = nation.n_nationkey\n" + - " join postgres.region on nation.n_regionkey = postgres.region.r_regionkey\n" + - " where\n" + - " postgres.region.r_name = \"ASIA\"\n" + - " and mongotpch.orders.o_orderdate >= 757353600000\n" + - " and mongotpch.orders.o_orderdate < 788889600000\n" + - " )\n" + - " group by\n" + - " nation.n_name\n" + - ")\n" + - "order by\n" + - " revenue desc;"; + List queryIds = Arrays.asList(1,2,5,6,10,13,17,18,19,20); + for (int queryId : queryIds) { + // read from sql file + String sqlString = readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); - // 开始 tpch 查询 - System.out.println("start tpch query"); - startTime = System.currentTimeMillis(); + // 开始 tpch 查询 + System.out.println("start tpch query " + queryId); + startTime = System.currentTimeMillis(); - // 执行查询语句 - SessionExecuteSqlResult result = conn.executeSql(sqlString); - result.print(false, ""); - List> values = result.getValues(); - List> answers = csvReader("src/test/resources/polybench/sf0.1/q05.csv"); - if (values.size() != answers.size()) { - throw new RuntimeException("size not equal"); - } - for (int i = 0; i < values.size(); i++) { - String nation = new String((byte[]) values.get(i).get(0), StandardCharsets.UTF_8); - double number = (double) values.get(i).get(1); - - System.out.println("nation： " + nation); - System.out.println("Number: " + number); - - String answerString = answers.get(i).get(0); - double answerNumber = Double.parseDouble(answers.get(i).get(1)); - System.out.println("Answer string: " + answerString); - System.out.println("Answer number: " + answerNumber); + // 执行查询语句, split by ; + String[] sqls = sqlString.split(";"); + for (String sql : sqls) { + if (sql.trim().length() == 0) { + continue; + } + sql += ";"; + SessionExecuteSqlResult result = conn.executeSql(sql); + result.print(false, ""); + } +// SessionExecuteSqlResult result = conn.executeSql(sqlString); +// result.print(false, ""); - assert nation.equals(answerString); - assert answerNumber - number < 1e-5 && number - answerNumber < 1e-5; + // TODO 验证 + System.out.println("end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); } + // 验证 +// List> values = result.getValues(); +// List> answers = csvReader("src/test/resources/polybench/sf0.1/q05.csv"); +// if (values.size() != answers.size()) { +// throw new RuntimeException("size not equal"); +// } +// for (int i = 0; i < values.size(); i++) { +// String nation = new String((byte[]) values.get(i).get(0), StandardCharsets.UTF_8); +// double number = (double) values.get(i).get(1); +// +// System.out.println("nation： " + nation); +// System.out.println("Number: " + number); +// +// String answerString = answers.get(i).get(0); +// double answerNumber = Double.parseDouble(answers.get(i).get(1)); +// System.out.println("Answer string: " + answerString); +// System.out.println("Answer number: " + answerNumber); +// +// assert nation.equals(answerString); +// assert answerNumber - number < 1e-5 && number - answerNumber < 1e-5; +// } // 关闭会话 conn.closeSession(); - System.out.println("end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (SessionException e) { throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); } } } diff --git a/test/src/test/resources/polybench/queries/q17.sql b/test/src/test/resources/polybench/queries/q17.sql index b7c3983262..095b9a4700 100644 --- a/test/src/test/resources/polybench/queries/q17.sql +++ b/test/src/test/resources/polybench/queries/q17.sql @@ -1,4 +1,4 @@ -insert into tmpTable(key, p_partkey, val) values ( +insert into tmpTableB(key, p_partkey, val) values ( select postgres.part.p_partkey, 0.2 * tmp @@ -11,7 +11,7 @@ insert into tmpTable(key, p_partkey, val) values ( join postgres.part on mongotpch.lineitem.l_partkey = postgres.part.p_partkey group by postgres.part.p_partkey ) -) +); select tmp2 / 7 as avg_yearly @@ -21,9 +21,9 @@ from ( from mongotpch.lineitem join postgres.part on postgres.part.p_partkey = mongotpch.lineitem.l_partkey - join tmpTable on tmpTable.p_partkey = mongotpch.lineitem.l_partkey + join tmpTableB on tmpTableB.p_partkey = mongotpch.lineitem.l_partkey where postgres.part.p_brand = 'Brand#23' and postgres.part.p_container = 'MED BOX' - and mongotpch.lineitem.l_quantity < tmpTable.val + and mongotpch.lineitem.l_quantity < tmpTableB.val ); \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q18.sql b/test/src/test/resources/polybench/queries/q18.sql index ca07e510e2..5b2e532941 100644 --- a/test/src/test/resources/polybench/queries/q18.sql +++ b/test/src/test/resources/polybench/queries/q18.sql @@ -32,5 +32,4 @@ group by mongotpch.orders.o_orderdate, mongotpch.orders.o_totalprice order by - mongotpch.orders.o_orderdate, mongotpch.orders.o_totalprice desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q19.sql b/test/src/test/resources/polybench/queries/q19.sql new file mode 100644 index 0000000000..fb68561ba1 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q19.sql @@ -0,0 +1,48 @@ +select + sum(tmp) as revenue +from ( + select + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp + from + mongotpch.lineitem + join postgres.part on postgres.part.p_partkey = mongotpch.lineitem.l_partkey + where ( + postgres.part.p_brand = 'Brand#12' + and ( + postgres.part.p_container = 'SM CASE' + or postgres.part.p_container = 'SM BOX' + or postgres.part.p_container = 'SM PACK' + or postgres.part.p_container = 'SM PKG' + ) + and mongotpch.lineitem.l_quantity >= 1 and mongotpch.lineitem.l_quantity <= 11 + and postgres.part.p_size >= 1 and postgres.part.p_size <= 5 + and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') + and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + or ( + postgres.part.p_brand = 'Brand#23' + and ( + postgres.part.p_container = 'MED PKG' + or postgres.part.p_container = 'MED BOX' + or postgres.part.p_container = 'MED BAG' + or postgres.part.p_container = 'MED PACK' + ) + and mongotpch.lineitem.l_quantity >= 10 and mongotpch.lineitem.l_quantity <= 20 + and postgres.part.p_size >= 1 and postgres.part.p_size <= 10 + and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') + and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + or ( + postgres.part.p_brand = 'Brand#34' + and ( + postgres.part.p_container = 'LG PACK' + or postgres.part.p_container = 'LG BOX' + or postgres.part.p_container = 'LG CASE' + or postgres.part.p_container = 'LG PKG' + ) + and mongotpch.lineitem.l_quantity >= 20 and mongotpch.lineitem.l_quantity <= 30 + and postgres.part.p_size >= 1 and postgres.part.p_size <= 15 + and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') + and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + ); \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q2.sql b/test/src/test/resources/polybench/queries/q2.sql index c2540ecd60..a262595465 100644 --- a/test/src/test/resources/polybench/queries/q2.sql +++ b/test/src/test/resources/polybench/queries/q2.sql @@ -1,18 +1,19 @@ -insert into tmpTable(key, p_key, minCost) values (select - postgres.part.p_partkey as p_key, - min(postgres.partsupp.ps_supplycost) as minCost - from - postgres.partsupp - join postgres.supplier on postgres.supplier.s_suppkey = postgres.partsupp.ps_suppkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey - join postgres.region on nation.n_regionkey = postgres.region.r_regionkey - join postgres.part on postgres.part.p_partkey = postgres.partsupp.ps_partkey - where - postgres.region.r_name = 'EUROPE' - and postgres.part.p_size = 15 - and postgres.part.p_type like "^.*BRASS" - group by postgres.part.p_partkey); - +insert into tmpTable(key, p_key, minCost) values ( +select + postgres.part.p_partkey as p_key, + min(postgres.partsupp.ps_supplycost) as minCost +from + postgres.partsupp + join postgres.supplier on postgres.supplier.s_suppkey = postgres.partsupp.ps_suppkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join postgres.region on nation.n_regionkey = postgres.region.r_regionkey + join postgres.part on postgres.part.p_partkey = postgres.partsupp.ps_partkey +where + postgres.region.r_name = 'EUROPE' + and postgres.part.p_size = 15 + and postgres.part.p_type like "^.*BRASS" +group by postgres.part.p_partkey +); select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from (select diff --git a/test/src/test/resources/polybench/queries/q20.sql b/test/src/test/resources/polybench/queries/q20.sql new file mode 100644 index 0000000000..502da0481d --- /dev/null +++ b/test/src/test/resources/polybench/queries/q20.sql @@ -0,0 +1,45 @@ +insert into tmpTableA(key, partkey, suppkey, val) values ( + select + partkey, + suppkey, + 0.5 * tmp from( + select + l_partkey as partkey, + l_suppkey as suppkey, + sum(l_quantity) as tmp + from + mongotpch.lineitem + where + mongotpch.lineitem.l_shipdate >= 757353600000 + and mongotpch.lineitem.l_shipdate < 788889600000 + group by l_partkey, l_suppkey + ) + ); + +select + postgres.supplier.s_name, + postgres.supplier.s_address +from + postgres.supplier + join nation on postgres.supplier.s_nationkey = nation.n_nationkey +where + postgres.supplier.s_suppkey in ( + select + postgres.partsupp.ps_suppkey + from + postgres.partsupp + join tmpTableA on tmpTableA.suppkey = postgres.partsupp.ps_suppkey and tmpTableA.partkey = postgres.partsupp.ps_partkey + where + postgres.partsupp.ps_partkey in ( + select + p_partkey + from + postgres.part + where + postgres.part.p_name like 'forest.*' + ) + and postgres.partsupp.ps_availqty > tmpTableA.val + ) + and nation.n_name = 'CANADA' +order by + postgres.supplier.s_name; \ No newline at end of file diff --git a/test/src/test/resources/polybench/sf0.1/q02.csv b/test/src/test/resources/polybench/sf0.1/q02.csv deleted file mode 100644 index 4a99144c7c..0000000000 --- a/test/src/test/resources/polybench/sf0.1/q02.csv +++ /dev/null @@ -1,101 +0,0 @@ -s_acctbal|s_name|n_name|p_partkey|p_mfgr|s_address|s_phone|s_comment -9938.53|Supplier#000005359|UNITED KINGDOM|185358|Manufacturer#4|bgxj2K0w1kJvxYl5mhCfou,W|33-429-790-6131|l, ironic instructions cajole -9937.84|Supplier#000005969|ROMANIA|108438|Manufacturer#1|rdnmd9c8EG1EIAYY3LPVa4yUNx6OwyVaQ|29-520-692-3537|es. furiously silent deposits among the deposits haggle furiously a -9936.22|Supplier#000005250|UNITED KINGDOM|249|Manufacturer#4|qX AB0vP8mJEWeBuY9jri|33-320-228-2957|ar, regular requests nag blithely special accounts. final deposits impress carefully. ironic, -9923.77|Supplier#000002324|GERMANY|29821|Manufacturer#4|uXcnR7tv87dG|17-779-299-1839|s sleep according to the quick requests. carefully -9871.22|Supplier#000006373|GERMANY|43868|Manufacturer#5|iSLO35z7Ae|17-813-485-8637|against the slyly daring requests. unusual accounts wake atop the blithely spe -9870.78|Supplier#000001286|GERMANY|81285|Manufacturer#2|3gq0mZLHI5OTM6 tBYmLTHZaulCYnlECzQ7nj|17-516-924-4574|into beans haggle at the quickly final asymptotes. unusu -9870.78|Supplier#000001286|GERMANY|181285|Manufacturer#4|3gq0mZLHI5OTM6 tBYmLTHZaulCYnlECzQ7nj|17-516-924-4574|into beans haggle at the quickly final asymptotes. unusu -9852.52|Supplier#000008973|RUSSIA|18972|Manufacturer#2|zVfUT3Np22kUC05tYWHBotaR|32-188-594-7038|ly daring excuses unwind carefully above the fu -9847.83|Supplier#000008097|RUSSIA|130557|Manufacturer#2|veMRTQBmUResNvfD3|32-375-640-3593| slyly ironic, special requests. final instructions above the qu -9847.57|Supplier#000006345|FRANCE|86344|Manufacturer#1|68yX tGXAkVRSxUGNSjJdptw 8O878xaFnaoQK|16-886-766-7945|odolites. blithely special requests above the regular foxes sleep unusual sauternes. care -9847.57|Supplier#000006345|FRANCE|173827|Manufacturer#2|68yX tGXAkVRSxUGNSjJdptw 8O878xaFnaoQK|16-886-766-7945|odolites. blithely special requests above the regular foxes sleep unusual sauternes. care -9836.93|Supplier#000007342|RUSSIA|4841|Manufacturer#4|icFgTpZ0TuAm188dv|32-399-414-5385| packages are blithely about the quickly -9817.10|Supplier#000002352|RUSSIA|124815|Manufacturer#2|XfLCj71HKHnPqgvs7KNgPKcOWoWxo2w|32-551-831-1437|al packages doze always according to the quickly f -9817.10|Supplier#000002352|RUSSIA|152351|Manufacturer#3|XfLCj71HKHnPqgvs7KNgPKcOWoWxo2w|32-551-831-1437|al packages doze always according to the quickly f -9739.86|Supplier#000003384|FRANCE|138357|Manufacturer#2|D01XwXbcILNwmrGS6ZPrVhZxO40i|16-494-913-5925|es. carefully regular ideas cajole. quickly ironic requests haggle. pending sentiment -9721.95|Supplier#000008757|UNITED KINGDOM|156241|Manufacturer#3|ryKUkEeWN7Z|33-821-407-2995| the instructions breach slyly -9681.33|Supplier#000008406|RUSSIA|78405|Manufacturer#1|1A6x3PLy6F|32-139-873-8571|ons sleep express deposits. epitap -9643.55|Supplier#000005148|ROMANIA|107617|Manufacturer#1|H7WOI6lzFuSsWzTSBrhzTYV|29-252-617-4850|carefully platelets. packages sleep special ideas. quick -9624.82|Supplier#000001816|FRANCE|34306|Manufacturer#3|NTwQPSZwfhc4uu1EMvEDopBnEv2j P|16-392-237-6726| the express, regular accounts. regular decoys boost alongside of -9624.78|Supplier#000009658|ROMANIA|189657|Manufacturer#1|DmRxpLmL88XCBiONB3tq3e0u|29-748-876-2014|inst the blithely brave frays. brav -9612.94|Supplier#000003228|ROMANIA|120715|Manufacturer#2|hnNBdhdXO4yT18 QNABTrL8fuv0A4p|29-325-784-8187|furiously foxes. express packages nag blithely express, pending ideas. fluffily ironi -9612.94|Supplier#000003228|ROMANIA|198189|Manufacturer#4|hnNBdhdXO4yT18 QNABTrL8fuv0A4p|29-325-784-8187|furiously foxes. express packages nag blithely express, pending ideas. fluffily ironi -9571.83|Supplier#000004305|ROMANIA|179270|Manufacturer#2|Bdj1T5EostLveb9ocRbz|29-973-481-1831|fully: fluffily special deposits use fur -9558.10|Supplier#000003532|UNITED KINGDOM|88515|Manufacturer#4|ncMxIJcDYZd5B7FlKxxLmnlzPeZB,FKBujB|33-152-301-2164|against the final pinto beans. carefully bold asymptotes use -9492.79|Supplier#000005975|GERMANY|25974|Manufacturer#5|9UEiIp7uSYtTF5|17-992-579-4839|fluffily ironic instructions haggle against the even, special accounts. quickly final -9461.05|Supplier#000002536|UNITED KINGDOM|20033|Manufacturer#1|TEEkPusQ6rU18YvixE7IQtBDOyRBdGoOWl2r|33-556-973-5522|inal ideas cajole furiously. blithely special Tiresias against the b -9453.01|Supplier#000000802|ROMANIA|175767|Manufacturer#1|1Uj23QWxQjj7EyeqHWqGWTbN|29-342-882-6463|s according to the even deposits integrate express packages. express -9408.65|Supplier#000007772|UNITED KINGDOM|117771|Manufacturer#4|rIoV2rj0KNy,IT|33-152-491-1126|s nag quickly regular packages. carefully express pinto beans about th -9359.61|Supplier#000004856|ROMANIA|62349|Manufacturer#5|k2CKOmXhPruJZ|29-334-870-9731|es. final asymptotes wake carefully -9357.45|Supplier#000006188|UNITED KINGDOM|138648|Manufacturer#1|LS,Z0 zbSvC7GWjF|33-583-607-1633| somas cajole around the even, ironic deposits. pending theodolites according to the b -9352.04|Supplier#000003439|GERMANY|170921|Manufacturer#4|B2bnKDIpkJp2uHKp|17-128-996-4650|nusual frets cajole carefully beneath -9312.97|Supplier#000007807|RUSSIA|90279|Manufacturer#5|Dk2ebpGR3jlpYbpMg9Djr|32-673-872-5854|. silent asymptotes boost. quickly ironic accounts for the -9312.97|Supplier#000007807|RUSSIA|100276|Manufacturer#5|Dk2ebpGR3jlpYbpMg9Djr|32-673-872-5854|. silent asymptotes boost. quickly ironic accounts for the -9280.27|Supplier#000007194|ROMANIA|47193|Manufacturer#3|tJ96aHp8 l3uiq38LiDHswtk9bHMg|29-318-454-2133|tes. carefully regular accounts are carefully since the waters. accounts cajole? carefully bold -9274.80|Supplier#000008854|RUSSIA|76346|Manufacturer#3|,uJfCd6eTiYE1ZEhDM vsc8ANQPWaPlQ|32-524-148-5221|onic, final ideas. blithely regular platelets boost final, ironic pinto beans. fluffil -9249.35|Supplier#000003973|FRANCE|26466|Manufacturer#1|OZSkIozfU4FYizk4e091MZHozL1qcHe257J89bW|16-722-866-1658|beans. slyly ironic dependencies cajole furiously furiously regular ideas. boldly even requests hagg -9249.35|Supplier#000003973|FRANCE|33972|Manufacturer#1|OZSkIozfU4FYizk4e091MZHozL1qcHe257J89bW|16-722-866-1658|beans. slyly ironic dependencies cajole furiously furiously regular ideas. boldly even requests hagg -9208.70|Supplier#000007769|ROMANIA|40256|Manufacturer#5|AzIENtMrVCSbrjyUu8|29-964-424-9649| ironic requests among the deposits affix busily ironic accounts. slow pinto beans are blithely fi -9201.47|Supplier#000009690|UNITED KINGDOM|67183|Manufacturer#5|pprpD77FEIWsNMmGT9T|33-121-267-9529|uriously bold packages integrate blithely ironic theodolites. carefully unusual escap -9192.10|Supplier#000000115|UNITED KINGDOM|85098|Manufacturer#3|EhrYy0MT5M1vfZ0V4skpifdp6pgFz5|33-597-248-1220|onic instructions. ironic, regular deposits haggle f -9189.98|Supplier#000001226|GERMANY|21225|Manufacturer#4|BzfoA9wft1Mx3iBIs|17-725-903-1381|luffily across the slyly special instructions. bold, ironic deposi -9128.97|Supplier#000004311|RUSSIA|146768|Manufacturer#5|jSiHD4NTd8i9zVRX9uz9a,|32-155-440-7120|theodolites. furiously even pinto beans abou -9104.83|Supplier#000008520|GERMANY|150974|Manufacturer#4|aA95nLn,m9shRrPXZw9Y1X|17-728-804-1793|nstructions. carefully regular requests use fluffily against the quickly final deposits. blithel -9101.00|Supplier#000005791|ROMANIA|128254|Manufacturer#5|txPYsp50HJkbbaAJ0bYieqHmZtirDUVOcmC4lk|29-549-251-5384| regular foxes use carefully final packages. fluffily stealthy deposits toward the sp -9094.57|Supplier#000004582|RUSSIA|39575|Manufacturer#1|5p,3Gp8kX 1EDarE0JR5juHH Sq9jlxgKenM|32-587-577-1351|ages affix quickly after the carefully regular accounts. regular, regular foxes kindle. -8996.87|Supplier#000004702|FRANCE|102191|Manufacturer#5|T35OahYXQGC|16-811-269-8946|ily regular grouches wake quickly ironic de -8996.14|Supplier#000009814|ROMANIA|139813|Manufacturer#2|RL,cVCKSXFc6Win6EmtF415A22as8nG2fqEa|29-995-571-8781| regular requests haggle carefully above the carefully regular deposits. ironic pearls in p -8968.42|Supplier#000010000|ROMANIA|119999|Manufacturer#5|R7kfmyzoIfXlrbnqNwUUW3phJctocp0J|29-578-432-2146|ular, quick foxes sleep quickly according to the blithely fluffy theodolit -8936.82|Supplier#000007043|UNITED KINGDOM|109512|Manufacturer#1|m5QHON1iD1OPhmU2R3z97u 6mCIvjnAc3I0,9s|33-784-177-8208| final dependencies. deposits a -8929.42|Supplier#000008770|FRANCE|173735|Manufacturer#4|aTOkYV7y3 kqbRrkOGJLaI|16-242-746-9248|ns haggle quickly silent theodolites. bold, final requests along t -8920.59|Supplier#000003967|ROMANIA|26460|Manufacturer#1|NjCq3NUY82S|29-194-731-3944|ts. daringly regular theodolites affix silently. reg -8920.59|Supplier#000003967|ROMANIA|173966|Manufacturer#2|NjCq3NUY82S|29-194-731-3944|ts. daringly regular theodolites affix silently. reg -8913.96|Supplier#000004603|UNITED KINGDOM|137063|Manufacturer#2|d6sFwf6 TD1xyfuFbdM2h8LX7ZWc3zHupV|33-789-255-7342|lithely whithout the furiously ironic sheaves. ironic reques -8877.82|Supplier#000007967|FRANCE|167966|Manufacturer#5|rXBIZqq9eWEuU90B vlCab6|16-442-147-9345|ckages-- evenly even requests boost blit -8862.24|Supplier#000003323|ROMANIA|73322|Manufacturer#3|5RrF2PzoRlwpAGXjyf|29-736-951-3710|regular ideas haggle blithely packages. regula -8841.59|Supplier#000005750|ROMANIA|100729|Manufacturer#5|n uXFrKx,KVYIQjmRuV,yejWmLMdRJnk|29-344-502-5481|leep finally furiously express packages. slyly unusual packages cajole unusual, -8781.71|Supplier#000003121|ROMANIA|13120|Manufacturer#5|wdA7CLuYXS22oQEmP0V,x0PHrXiPdl5Rpwv,ub|29-707-291-5144|ies. final foxes are furiou -8754.24|Supplier#000009407|UNITED KINGDOM|179406|Manufacturer#4|pj9oPHQ4OLWp|33-903-970-9604|ng asymptotes hang across the blithely special deposits. -8691.06|Supplier#000004429|UNITED KINGDOM|126892|Manufacturer#2|H0paE V6JCrlZpYrzI0LgIP|33-964-337-5038| sly requests might sleep. final dolphins sleep. furiousl -8655.99|Supplier#000006330|RUSSIA|193810|Manufacturer#2|7CsFQnd ,tzgMYvVoMim5l4DrJcX8SaQMTcy|32-561-198-3705|ideas wake across the regular, unusual instructions; furiously final deposits wake near the s -8638.36|Supplier#000002920|RUSSIA|75398|Manufacturer#1|iMYQSQzsLXg|32-122-621-7549|ickly dolphins. furiously careful asymptotes sublate -8638.36|Supplier#000002920|RUSSIA|170402|Manufacturer#3|iMYQSQzsLXg|32-122-621-7549|ickly dolphins. furiously careful asymptotes sublate -8607.69|Supplier#000006003|UNITED KINGDOM|76002|Manufacturer#2|njRvqoOmIxNDe,da,SsnweINv1VY2YatifmJq|33-416-807-5206|braids sleep carefully along the iron -8569.52|Supplier#000005936|RUSSIA|5935|Manufacturer#5|I3Qd1VwvDm5hYGzg1hBHzKy,P3YQXq7|32-644-251-7916|s about the carefully final accounts use always even requests. furiously express dependenc -8564.12|Supplier#000000033|GERMANY|110032|Manufacturer#1|LLMgB3vXW,0g,8nuv3qU3QZaEBZvU2qRLX9|17-138-897-9374|l packages cajole unusual, final packages. slyly express requests -8553.82|Supplier#000003979|ROMANIA|143978|Manufacturer#4|qLE5JpqDoe3XHsBI6etWpd4zRsjsBNb9Tidi6|29-124-646-4897|counts are quickly carefully ironic instructions. platelets wake f -8517.23|Supplier#000009529|RUSSIA|37025|Manufacturer#5|NWW9SDThqi9RIeOA|32-565-297-8775|ial requests use stealthily along the carefully u -8517.23|Supplier#000009529|RUSSIA|59528|Manufacturer#2|NWW9SDThqi9RIeOA|32-565-297-8775|ial requests use stealthily along the carefully u -8503.70|Supplier#000006830|RUSSIA|44325|Manufacturer#4|qoW4lp2961uQiKOK6rW8|32-147-878-5069|atelets sleep furiously pending asymptotes. even requests for the blithely unusual packages -8457.09|Supplier#000009456|UNITED KINGDOM|19455|Manufacturer#1|U8pJ1 SKbZPhH7,bLWXX3pG|33-858-440-4349|ounts sleep about the bold, even ideas. slyly unusual accounts after the asymptotes -8441.40|Supplier#000003817|FRANCE|141302|Manufacturer#2|K6XLsYufTS|16-339-356-5115|sly fluffily regular pinto beans. slyly even deposits snooze fluffily along the fluff -8432.89|Supplier#000003990|RUSSIA|191470|Manufacturer#1|wMJppCZ9aPMuq2nr88TVfztvE gj95OG wdNUE|32-839-509-9301|. express pinto beans use slyly. regular platelets sleep quickly busy deposits. final -8431.40|Supplier#000002675|ROMANIA|5174|Manufacturer#1|khl8ydxR9VekbcMLgJKPtpNtwAkYtJTv|29-474-643-1443|regular, express platelets are. carefully ironic forges since the requests affix -8407.04|Supplier#000005406|RUSSIA|162889|Manufacturer#4|ITrK2mV94SooV6 Igo|32-626-152-4621| even theodolites. quickly bold deposits after the pen -8386.08|Supplier#000008518|FRANCE|36014|Manufacturer#3|ZHAsABq5MRP e5kc0DRD8za3xGdf763ChHmoOA45|16-618-780-7481|g alongside of the slyly unusual platelets! blithely regular asymptotes cajole. quickly regular -8376.52|Supplier#000005306|UNITED KINGDOM|190267|Manufacturer#5|SyS2SsaA8i CqnbzUdfNH07bVtt9uW,Cp6FLCkOR|33-632-514-7931|pendencies affix furiously against the special, blithe packages. qui -8348.74|Supplier#000008851|FRANCE|66344|Manufacturer#4|E4uITlvmPHKvZ|16-796-240-2472|s packages haggle above the express pinto beans. stealthy, ironic theodolites sleep quickly -8338.58|Supplier#000007269|FRANCE|17268|Manufacturer#4|2vJh8wqp6CJp,W0Y|16-267-277-4365|lithely through the accounts. express, ironic asymptotes wou -8328.46|Supplier#000001744|ROMANIA|69237|Manufacturer#5|DfCXL6UWAY1lgjQYB0AjE8T2sx0BzS|29-330-728-5873| regular, special dolphins haggle carefully special asy -8307.93|Supplier#000003142|GERMANY|18139|Manufacturer#1|OAPFw6SNodrC kFi|17-595-447-6026|usly express packages sleep finally regular ideas. carefu -8231.61|Supplier#000009558|RUSSIA|192000|Manufacturer#2|FONKME0t7ZJhnjn9VL5|32-762-137-5858|g to the carefully even brai -8152.61|Supplier#000002731|ROMANIA|15227|Manufacturer#4|sDFx3iox2Zzx|29-805-463-2030|ly above the packages. final accounts sleep furiously. fluffily iro -8109.09|Supplier#000009186|FRANCE|99185|Manufacturer#1|wKLCzA5bMuGRBm35tvQAGpen23L|16-668-570-1402|ts cajole daringly. pinto beans -8102.62|Supplier#000003347|UNITED KINGDOM|18344|Manufacturer#5|Froy39Y8ZUJ|33-454-274-8532|y daring requests. unusual accounts wake atop the blithely special packages. sly -8046.07|Supplier#000008780|FRANCE|191222|Manufacturer#3|rOssxn,6gRDzHr0gu,hEK|16-473-215-6395|he regular foxes cajole ruthlessly among the sometimes final grouches. blithel -8042.09|Supplier#000003245|RUSSIA|135705|Manufacturer#4|oJSiGLXRCDAPcfWot7LkwSQRCh63XNS2|32-836-132-8872| use slyly. furiously regular deposits sleep according to the requests. -8042.09|Supplier#000003245|RUSSIA|150729|Manufacturer#1|oJSiGLXRCDAPcfWot7LkwSQRCh63XNS2|32-836-132-8872| use slyly. furiously regular deposits sleep according to the requests. -7992.40|Supplier#000006108|FRANCE|118574|Manufacturer#1|TyptNE7nv6BLpLl6WFX|16-974-998-8937|theodolites among the furiously unusual accounts must x -7980.65|Supplier#000001288|FRANCE|13784|Manufacturer#4|tm0TjL5b oE|16-646-464-8247|gular pains? fluffily bold warhorses affix? blithe instruction -7950.37|Supplier#000008101|GERMANY|33094|Manufacturer#5|HG2wfVixwCIhK7dlrigGR3an2LuSifDJH|17-627-663-8014|ly alongside of the furiously unusual requests! bold, express foxe -7937.93|Supplier#000009012|ROMANIA|83995|Manufacturer#2|J6I7sJj0mGYIWFv9KxD3fK O7tvNP|29-250-925-9690| use slyly against the slyly bold theod -7914.45|Supplier#000001013|RUSSIA|125988|Manufacturer#2|AI9ODzBzWgnny28PHBei5M2lUFdD9|32-194-698-3365| the blithely silent accounts. q -7912.91|Supplier#000004211|GERMANY|159180|Manufacturer#5|Zva95Dwj EY0w,XjgsL7O0Zb2 l3almck|17-266-947-7315| slyly silent requests; fluffily fi -7912.91|Supplier#000004211|GERMANY|184210|Manufacturer#4|Zva95Dwj EY0w,XjgsL7O0Zb2 l3almck|17-266-947-7315| slyly silent requests; fluffily fi -7894.56|Supplier#000007981|GERMANY|85472|Manufacturer#4|e8hRUxe9cqQM3b|17-963-404-3760|ly final courts. unusual, quiet dolphi -7887.08|Supplier#000009792|GERMANY|164759|Manufacturer#3|3YSi76M2 I8XGikO5YgSM81r5Z6A7VkZcys|17-988-938-4296| the regular ideas. furiously bold deposits boost above the bli -7871.50|Supplier#000007206|RUSSIA|104695|Manufacturer#1|YvrLdpD 5ExhHmRWzK41tw4|32-432-452-7731|ording to the furious theodolites cajole carefully according to the busily express asymptotes. -7852.45|Supplier#000005864|RUSSIA|8363|Manufacturer#4|5odLpc1M83KXJ0O|32-454-883-3821|egular, regular ideas. requests are carefully. furiously final dolp -7850.66|Supplier#000001518|UNITED KINGDOM|86501|Manufacturer#1|ddNQX3hIjgico|33-730-383-3892|ccounts. special, final deposits -7843.52|Supplier#000006683|FRANCE|11680|Manufacturer#4|Z1,hkHIw,Z3,,Comv6kLxIiPJtoNt|16-464-517-8943|sits. blithely regular requests above the pending, regular ideas boo diff --git a/test/src/test/resources/polybench/sf0.1/q05.csv b/test/src/test/resources/polybench/sf0.1/q05.csv deleted file mode 100644 index 1be1eb1d27..0000000000 --- a/test/src/test/resources/polybench/sf0.1/q05.csv +++ /dev/null @@ -1,6 +0,0 @@ -n_name|revenue -"CHINA"|7822103.0000 -"INDIA"|6376121.5085 -"JAPAN"|6000077.2184 -"INDONESIA"|5580475.4027 -"VIETNAM"|4497840.5466 diff --git a/test/src/test/resources/polybench/sf0.1/q01.csv b/test/src/test/resources/polybench/sf0.1/q1.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q01.csv rename to test/src/test/resources/polybench/sf0.1/q1.csv diff --git a/test/src/test/resources/polybench/sf0.1/q18.csv b/test/src/test/resources/polybench/sf0.1/q18.csv new file mode 100644 index 0000000000..cea16c785b --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q18.csv @@ -0,0 +1,6 @@ +c_name|c_custkey|o_orderkey|o_orderdate|o_totalprice|sum +Customer#000001639|1639|502886|1994-04-12|456423.88|312 +Customer#000006655|6655|29158|1995-10-21|452805.02|305 +Customer#000014110|14110|565574|1995-09-24|425099.85|301 +Customer#000001775|1775|6882|1997-04-09|408368.10|303 +Customer#000011459|11459|551136|1993-05-19|386812.74|308 diff --git a/test/src/test/resources/polybench/sf0.1/q18.sql b/test/src/test/resources/polybench/sf0.1/q18.sql deleted file mode 100644 index 3f6886a48b..0000000000 --- a/test/src/test/resources/polybench/sf0.1/q18.sql +++ /dev/null @@ -1,33 +0,0 @@ -SELECT - c_name, - c_custkey, - o_orderkey, - o_orderdate, - o_totalprice, - sum(l_quantity) -FROM - customer, - orders, - lineitem -WHERE - o_orderkey IN ( - SELECT - l_orderkey - FROM - lineitem - GROUP BY - l_orderkey - HAVING - sum(l_quantity) > 300) - AND c_custkey = o_custkey - AND o_orderkey = l_orderkey -GROUP BY - c_name, - c_custkey, - o_orderkey, - o_orderdate, - o_totalprice -ORDER BY - o_totalprice DESC, - o_orderdate -LIMIT 100; diff --git a/test/src/test/resources/polybench/sf0.1/q19.csv b/test/src/test/resources/polybench/sf0.1/q19.csv new file mode 100644 index 0000000000..3c1e2a34d1 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q19.csv @@ -0,0 +1,2 @@ +revenue +168597.2860 diff --git a/test/src/test/resources/polybench/sf0.1/q2.csv b/test/src/test/resources/polybench/sf0.1/q2.csv new file mode 100644 index 0000000000..9c5612bba0 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q2.csv @@ -0,0 +1,45 @@ +s_acctbal|s_name|n_name|p_partkey|p_mfgr|s_address|s_phone|s_comment +9828.21|Supplier#000000647|UNITED KINGDOM|13120|Manufacturer#5|vV6Teq1EvLlR|33-258-202-4782|mong the carefully quiet accounts slee +9508.37|Supplier#000000070|FRANCE|3563|Manufacturer#1|jd4djZv0cc5KdnA0q9oOqvceaPUbNloOW|16-821-608-1166|n instructions are about the ironic, ironic excuses. instructions cajol +9508.37|Supplier#000000070|FRANCE|17268|Manufacturer#4|jd4djZv0cc5KdnA0q9oOqvceaPUbNloOW|16-821-608-1166|n instructions are about the ironic, ironic excuses. instructions cajol +9453.01|Supplier#000000802|ROMANIA|10021|Manufacturer#5|1Uj23QWxQjj7EyeqHWqGWTbN|29-342-882-6463|s according to the even deposits integrate express packages. express +9453.01|Supplier#000000802|ROMANIA|13275|Manufacturer#4|1Uj23QWxQjj7EyeqHWqGWTbN|29-342-882-6463|s according to the even deposits integrate express packages. express +9192.10|Supplier#000000115|UNITED KINGDOM|13325|Manufacturer#1|EhrYy0MT5M1vfZ0V4skpifdp6pgFz5|33-597-248-1220|onic instructions. ironic, regular deposits haggle f +9032.15|Supplier#000000959|GERMANY|4958|Manufacturer#4|TK qrnjpDvd1Jc|17-108-642-3106|nag across the slyly even pin +8702.02|Supplier#000000333|RUSSIA|11810|Manufacturer#3|fQ5Lr4KvbNHI3WDMhkcI S6xYtgIi1k|32-508-202-6136|ounts around the requests cajole furiously blithely even instructions. slyly +8615.50|Supplier#000000812|FRANCE|10551|Manufacturer#2|TAJWyNst8OGVPINgqtzwyyp002iYNDVub|16-585-724-6633|ress ideas eat quickly. blithely express deposits was slyly. final, +8615.50|Supplier#000000812|FRANCE|13811|Manufacturer#4|TAJWyNst8OGVPINgqtzwyyp002iYNDVub|16-585-724-6633|ress ideas eat quickly. blithely express deposits was slyly. final, +8488.53|Supplier#000000367|RUSSIA|6854|Manufacturer#4|nr8wRQ a5LXXess|32-458-198-9557|ect. quickly pending deposits sleep carefully even, express dependencies. +8430.52|Supplier#000000646|FRANCE|11384|Manufacturer#3|j6szE80YCpLHJ4bZ7F37gUiGhk0WJ0,8h9y|16-601-220-5489|quickly slyly even deposits. quickly ironic theodolites sleep fluffily after the c +8271.39|Supplier#000000146|RUSSIA|4637|Manufacturer#5|ApndKp ,Wu0 LNsoV0KldxyoIlY|32-792-619-3155|slyly regular foxes. unusual accounts about the regular packages +8096.98|Supplier#000000574|RUSSIA|323|Manufacturer#4|ZcSrzuRKYEGpcxmIsH,BrYBMwH0|32-866-246-8752|boost according to the slyly final instructions. furiously ironic packages cajole furiously +7392.78|Supplier#000000170|UNITED KINGDOM|7655|Manufacturer#2|ayz3a18xDGrr3jtS|33-803-340-5398|egular, even packages. pending, +7205.20|Supplier#000000477|GERMANY|10956|Manufacturer#5|6yQdgeVeAxJVtJTIYFNNWvQL|17-180-144-7991|ual accounts use quickly above the carefully quiet dolphins. packages nag closely. iro +6820.35|Supplier#000000007|UNITED KINGDOM|13217|Manufacturer#5| 0W7IPdkpWycUbQ9Adp6B|33-990-965-2201|ke across the slyly ironic packages. carefully special pinto beans wake blithely. even deposits los +6721.70|Supplier#000000954|FRANCE|4191|Manufacturer#3|cXcVBs6lsZbzfE14|16-537-341-8517|mong the quickly express pinto b +6329.90|Supplier#000000996|GERMANY|10735|Manufacturer#2|5uWNawcqv4IL8okyBL e|17-447-811-3282|deas. bold dinos are. carefully reg +6173.87|Supplier#000000408|RUSSIA|18139|Manufacturer#1|BOC Zy0wh3rCGHDgV0NIGt2dEK|32-858-724-2950| are carefully above the carefully final pinto beans. blithely express foxes ab +5364.99|Supplier#000000785|RUSSIA|13784|Manufacturer#4|5r5GjqBatnYAHaH5kB4IPcBEiglMJEnN4tUUG6k2|32-297-653-2203|se carefully after the bravely stealthy instru +5069.27|Supplier#000000328|GERMANY|16327|Manufacturer#1|9eEYWOr4kUZ|17-231-513-5721|es according to the slyly ironic package +4941.88|Supplier#000000321|ROMANIA|7320|Manufacturer#5|CfDKlGVtMePjtCw|29-573-279-1406| instructions boost carefu +4672.25|Supplier#000000239|RUSSIA|12238|Manufacturer#1|4cZ,ZHKj hRKgYlgZ6UapQ7mrEOozeQMx7KhUCS|32-396-654-6826|s wake fluffily slyly special foxes. ironic, bold +4586.49|Supplier#000000680|RUSSIA|5679|Manufacturer#3|7JwnLOmLhJ1aPMT61PSx9kcY77r,HmRUD314m|32-522-382-1620|e even pinto beans. blithely fluffy ideas cajole slyly around the bl +4518.31|Supplier#000000149|FRANCE|18344|Manufacturer#5|C5t4zIcINBkgBWdMg6WtgMtE|16-660-553-2456|silent platelets. ideas hinder carefully among the slyly regular deposits. slyly pending inst +4315.15|Supplier#000000509|FRANCE|18972|Manufacturer#2|9lTN9T5VBg|16-298-154-3365|ep boldly ironic theodolites. special dependencies lose blithely. final, regular packages wake +3526.53|Supplier#000000553|FRANCE|8036|Manufacturer#4|R0FI5DL3Poi|16-599-552-3755|l foxes wake slyly even f +3526.53|Supplier#000000553|FRANCE|17018|Manufacturer#3|R0FI5DL3Poi|16-599-552-3755|l foxes wake slyly even f +3294.68|Supplier#000000350|GERMANY|4841|Manufacturer#4|hilu5UXMCwFvJJ|17-113-181-4017|ronic ideas. blithely blithe accounts sleep blithely. regular requests boost carefully about the r +2972.26|Supplier#000000016|RUSSIA|1015|Manufacturer#4|3HbVoWVsjn4fTfQGgYTsMaDvMINBIDXqeBwK|32-822-502-4215|platelets thrash against the slyly special req +2963.09|Supplier#000000840|ROMANIA|3080|Manufacturer#2|J2s6iuBgJo03|29-781-337-5584|s sleep blithely unusual packages! even, bold accounts sleep slyly about the even +2221.25|Supplier#000000771|ROMANIA|13981|Manufacturer#2|Gv1ri,V ARHE136eJF|29-986-304-9006|lphins affix blithely along the carefully final ide +1381.97|Supplier#000000104|FRANCE|18103|Manufacturer#3|oOFWtl sAwYcbM9dWRPgKTS3Ebmn9Tcp3iz0F|16-434-972-6922|s. blithely pending requests against the regular instructions cajole sometimes according to the qu +906.07|Supplier#000000138|ROMANIA|8363|Manufacturer#4|yyPBFrErKTaEu5L3CdNJP ak4ys9AbN,Aj8wPgv|29-533-434-6776|deas haggle. final, regular packages wake. quiet packages cajole pinto beans +765.69|Supplier#000000799|RUSSIA|11276|Manufacturer#2|IvldT2pX7R el|32-579-339-1495| deposits: pending, unusual forges nag fluffily regular ideas +727.89|Supplier#000000470|ROMANIA|6213|Manufacturer#3|4OGPs qKpfQ6GNLIKhmbIE6e7fSMP8fmwi|29-165-289-1523|ly silent accounts. foxes maintain blithely along the idly +683.07|Supplier#000000651|RUSSIA|4888|Manufacturer#4|D4MGIq5Uz0,K|32-181-426-4490|ve to are slyly ironic asymptot +167.56|Supplier#000000290|FRANCE|2037|Manufacturer#1|VpG,Ul5yv1RgAK,,|16-675-286-5102| carefully furiously stealthy accounts. bold acc +91.39|Supplier#000000949|UNITED KINGDOM|9430|Manufacturer#2|R06m0VD95FZLoBJHcCMyaZQHitqmhZrQZkZk5|33-332-697-2768|sual requests. carefully regular requests bo +-314.06|Supplier#000000510|ROMANIA|17242|Manufacturer#4|6E3aFs0w2SiImzMDSewWtzOwdpLz2|29-207-852-3454|lyly regular accounts. deposits +-820.89|Supplier#000000409|GERMANY|2156|Manufacturer#5|gt362msTQ3AwtUVHgqP7Ryksk90dnpPNyn|17-719-517-9836|nal deposits doubt blithely regular packages. fr +-845.44|Supplier#000000704|ROMANIA|9926|Manufacturer#5|KawFpBPAADrVnKC,pLL9q3TSyHG9x|29-300-896-5991|ous pearls boost carefully +-942.73|Supplier#000000563|GERMANY|5797|Manufacturer#1|aOT6ZP96J2 ,Xhn|17-108-537-2691|are blithely silent requests. quickly even packages use blit diff --git a/test/src/test/resources/polybench/sf0.1/q20.csv b/test/src/test/resources/polybench/sf0.1/q20.csv new file mode 100644 index 0000000000..f1f9843c2e --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q20.csv @@ -0,0 +1,10 @@ +s_name|s_address +Supplier#000000157|1EmkCApL5iF +Supplier#000000197|3oYqODDUGH3XsHXmPuzYHW5NLU3,ONZl +Supplier#000000287|UQR8bUA4V2HxVbw9K +Supplier#000000378|mLPJtpu4wOc cSFzBR +Supplier#000000530|0BvoewCPg2scOEfuL93FRKqSxHmdhw1 +Supplier#000000555|8Lp0QWPLFXrJrX1sTWkAEdzUsh5ke +Supplier#000000557|IH,v63JRgXMkVhJOJ Gxur0W +Supplier#000000729|CAOGYCBtTVT7aB1p6qHbxF6VVhXaHLgTpI +Supplier#000000935|JHRSOterYgt4MTNo7cupTzA,6MoNw 4 diff --git a/test/src/test/resources/polybench/sf0.1/q5.csv b/test/src/test/resources/polybench/sf0.1/q5.csv new file mode 100644 index 0000000000..b34430123f --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q5.csv @@ -0,0 +1,6 @@ +n_name|revenue +CHINA|7822103.0000 +INDIA|6376121.5085 +JAPAN|6000077.2184 +INDONESIA|5580475.4027 +VIETNAM|4497840.5466 diff --git a/test/src/test/resources/polybench/sf0.1/q06.csv b/test/src/test/resources/polybench/sf0.1/q6.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q06.csv rename to test/src/test/resources/polybench/sf0.1/q6.csv From eaa76d45ac346b20e7d9daeacec26e7c7138485b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 16 May 2024 23:08:08 +0800 Subject: [PATCH 095/229] debug --- .../integration/polybench/TPCHRunner.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 0f203c252f..1efd36c961 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -63,42 +63,42 @@ public void test() { new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); conn.openSession(); -// // 输出所有存储引擎 -// String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); -// System.out.println(clusterInfo); -// -// // 添加存储引擎 -// System.out.println("start adding storage engine"); -// long startTime = System.currentTimeMillis(); -// Map pgMap = new HashMap<>(); -// pgMap.put("has_data", "true"); -// pgMap.put("is_read_only", "true"); -// pgMap.put("username", "postgres"); -// pgMap.put("password", "postgres"); -// pgMap = Collections.unmodifiableMap(pgMap); -// conn.addStorageEngine( -// "127.0.0.1", -// 5432, -// StorageEngineType.postgresql, -// pgMap -// ); -// Map mongoMap = new HashMap<>(); -// mongoMap.put("has_data", "true"); -// mongoMap.put("is_read_only", "true"); -// mongoMap.put("schema.sample.size", "1000"); -// mongoMap.put("dummy.sample.size", "0"); -// conn.addStorageEngine( -// "127.0.0.1", -// 27017, -// StorageEngineType.mongodb, -// mongoMap -// ); -// System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); -// -// // 输出所有存储引擎 -// clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); -// System.out.println(clusterInfo); - Long startTime; + // 输出所有存储引擎 + String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); + + // 添加存储引擎 + System.out.println("start adding storage engine"); + long startTime = System.currentTimeMillis(); + Map pgMap = new HashMap<>(); + pgMap.put("has_data", "true"); + pgMap.put("is_read_only", "true"); + pgMap.put("username", "postgres"); + pgMap.put("password", "postgres"); + pgMap = Collections.unmodifiableMap(pgMap); + conn.addStorageEngine( + "127.0.0.1", + 5432, + StorageEngineType.postgresql, + pgMap + ); + Map mongoMap = new HashMap<>(); + mongoMap.put("has_data", "true"); + mongoMap.put("is_read_only", "true"); + mongoMap.put("schema.sample.size", "1000"); + mongoMap.put("dummy.sample.size", "0"); + conn.addStorageEngine( + "127.0.0.1", + 27017, + StorageEngineType.mongodb, + mongoMap + ); + System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + + // 输出所有存储引擎 + clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); + //Long startTime; List queryIds = Arrays.asList(1,2,5,6,10,13,17,18,19,20); for (int queryId : queryIds) { From f97905a7af41f5e52324ef8e463b2a5b82540074 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 17 May 2024 01:20:28 +0800 Subject: [PATCH 096/229] debug --- .github/scripts/benchmarks/parquet.sh | 22 +++--- .../integration/polybench/TPCHRunner.java | 69 +++++++++++-------- .../test/resources/polybench/queries/q1.sql | 2 +- .../test/resources/polybench/queries/q20.sql | 12 ++-- 4 files changed, 60 insertions(+), 45 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 45c9ce4496..65952b3efe 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -23,17 +23,17 @@ done < "$input_file" cat "$output_file" -# 读取 nation.csv 文件的每一行 -while IFS=',' read -r col1 col2 col3 col4 col5; do - # 将第三列和第五列用引号括起来 - col3="\"$col3\"" - col5="\"$col5\"" - - # 输出处理后的行到新的文件中 - echo "$col1,$col2,$col3,$col4,$col5" >> new_nation.csv -done < "$output_file" - -mv new_nation.csv "$output_file" +## 读取 nation.csv 文件的每一行 +#while IFS=',' read -r col1 col2 col3 col4 col5; do +# # 将第三列和第五列用引号括起来 +# col3="\"$col3\"" +# col5="\"$col5\"" +# +# # 输出处理后的行到新的文件中 +# echo "$col1,$col2,$col3,$col4,$col5" >> new_nation.csv +#done < "$output_file" +# +#mv new_nation.csv "$output_file" echo "Conversion completed. CSV file: $output_file" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 1efd36c961..e4e949f4d8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -99,8 +99,8 @@ public void test() { clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); System.out.println(clusterInfo); //Long startTime; - - List queryIds = Arrays.asList(1,2,5,6,10,13,17,18,19,20); + // 13有问题 + List queryIds = Arrays.asList(1,2,5,6,10,17,18,19,20); for (int queryId : queryIds) { // read from sql file String sqlString = readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); @@ -109,43 +109,58 @@ public void test() { System.out.println("start tpch query " + queryId); startTime = System.currentTimeMillis(); - // 执行查询语句, split by ; + // 执行查询语句, split by ; 最后一句为执行结果 + SessionExecuteSqlResult result = null; String[] sqls = sqlString.split(";"); for (String sql : sqls) { if (sql.trim().length() == 0) { continue; } sql += ";"; - SessionExecuteSqlResult result = conn.executeSql(sql); + result = conn.executeSql(sql); result.print(false, ""); } -// SessionExecuteSqlResult result = conn.executeSql(sqlString); -// result.print(false, ""); // TODO 验证 System.out.println("end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + List> values = result.getValues(); + List> answers = csvReader("src/test/resources/polybench/sf0.1/q" + queryId + ".csv"); + if (values.size() != answers.size()) { + System.out.println("values.size() = " + values.size()); + System.out.println("answers.size() = " + answers.size()); + throw new RuntimeException("size not equal"); + } + for (int i = 0; i < values.size(); i++) { + if (values.get(i).size() != answers.get(i).size()) { + System.out.println("values.get(i).size() = " + values.get(i).size()); + System.out.println("answers.get(i).size() = " + answers.get(i).size()); + throw new RuntimeException("size not equal"); + } + for (int j = 0; j < values.get(i).size(); j++) { + System.out.println(values.get(i).get(j)); + if (result.getPaths().get(j).contains("address") + || result.getPaths().get(j).contains("comment") + || result.getPaths().get(j).contains("orderdate")) { + // skip s_address, data generation changed + continue; + } + // if only contains number and dot, then parse to double + if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { + double number = Double.parseDouble(values.get(i).get(j).toString()); + double answerNumber = Double.parseDouble(answers.get(i).get(j)); + System.out.println("Number: " + number); + System.out.println("Answer number: " + answerNumber); + assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; + } else { + String resultString = new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); + String answerString = answers.get(i).get(j); + System.out.println("Result string： " + resultString); + System.out.println("Answer string: " + answerString); + assert resultString.equals(answerString); + } + } + } } - // 验证 -// List> values = result.getValues(); -// List> answers = csvReader("src/test/resources/polybench/sf0.1/q05.csv"); -// if (values.size() != answers.size()) { -// throw new RuntimeException("size not equal"); -// } -// for (int i = 0; i < values.size(); i++) { -// String nation = new String((byte[]) values.get(i).get(0), StandardCharsets.UTF_8); -// double number = (double) values.get(i).get(1); -// -// System.out.println("nation： " + nation); -// System.out.println("Number: " + number); -// -// String answerString = answers.get(i).get(0); -// double answerNumber = Double.parseDouble(answers.get(i).get(1)); -// System.out.println("Answer string: " + answerString); -// System.out.println("Answer number: " + answerNumber); -// -// assert nation.equals(answerString); -// assert answerNumber - number < 1e-5 && number - answerNumber < 1e-5; -// } // 关闭会话 conn.closeSession(); diff --git a/test/src/test/resources/polybench/queries/q1.sql b/test/src/test/resources/polybench/queries/q1.sql index d9e6575ed7..ea0de5113a 100644 --- a/test/src/test/resources/polybench/queries/q1.sql +++ b/test/src/test/resources/polybench/queries/q1.sql @@ -8,7 +8,7 @@ select avg(mongotpch.lineitem.l_quantity) as avg_qty, avg(mongotpch.lineitem.l_extendedprice) as avg_price, avg(mongotpch.lineitem.l_discount) as avg_disc, - count(*) as count_order + count(mongotpch.lineitem.l_returnflag) as count_order from ( select l_returnflag, diff --git a/test/src/test/resources/polybench/queries/q20.sql b/test/src/test/resources/polybench/queries/q20.sql index 502da0481d..cf04669c2b 100644 --- a/test/src/test/resources/polybench/queries/q20.sql +++ b/test/src/test/resources/polybench/queries/q20.sql @@ -1,8 +1,8 @@ insert into tmpTableA(key, partkey, suppkey, val) values ( - select - partkey, - suppkey, - 0.5 * tmp from( + select + partkey, + suppkey, + 0.5 * tmp from( select l_partkey as partkey, l_suppkey as suppkey, @@ -14,7 +14,7 @@ insert into tmpTableA(key, partkey, suppkey, val) values ( and mongotpch.lineitem.l_shipdate < 788889600000 group by l_partkey, l_suppkey ) - ); +); select postgres.supplier.s_name, @@ -28,7 +28,7 @@ where postgres.partsupp.ps_suppkey from postgres.partsupp - join tmpTableA on tmpTableA.suppkey = postgres.partsupp.ps_suppkey and tmpTableA.partkey = postgres.partsupp.ps_partkey + join tmpTableA on tmpTableA.suppkey = postgres.partsupp.ps_suppkey and tmpTableA.partkey = postgres.partsupp.ps_partkey where postgres.partsupp.ps_partkey in ( select From 6114e29ed4e93556b327f16b82cf8245e750d638 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 17 May 2024 17:23:24 +0800 Subject: [PATCH 097/229] add query 9 --- .../integration/polybench/TPCHRunner.java | 2 +- .../test/resources/polybench/queries/q9.sql | 34 ++++ .../src/test/resources/polybench/sf0.1/q9.csv | 176 ++++++++++++++++++ .../polybench/udf/udtf_extract_year.py | 7 +- 4 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 test/src/test/resources/polybench/queries/q9.sql create mode 100644 test/src/test/resources/polybench/sf0.1/q9.csv diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index e4e949f4d8..5d134ff867 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -100,7 +100,7 @@ public void test() { System.out.println(clusterInfo); //Long startTime; // 13有问题 - List queryIds = Arrays.asList(1,2,5,6,10,17,18,19,20); + List queryIds = Arrays.asList(1,2,5,6,9,10,17,18,19,20); for (int queryId : queryIds) { // read from sql file String sqlString = readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql new file mode 100644 index 0000000000..df18b34c66 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q9.sql @@ -0,0 +1,34 @@ +CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py"; + +insert into tmpTableC(key, orderkey, year) values ( + select o_orderkey, extractYear(o_orderdate) from mongotpch.orders + ); + +select * from ( + select + nation, + o_year, + sum(amount) as sum_profit + from ( + select + nation.n_name as nation, + tmpTableC.year as o_year, + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) - postgres.partsupp.ps_supplycost * mongotpch.lineitem.l_quantity as amount + from + postgres.part + join mongotpch.lineitem on postgres.part.p_partkey = mongotpch.lineitem.l_partkey + join postgres.supplier on postgres.supplier.s_suppkey = mongotpch.lineitem.l_suppkey + join postgres.partsupp on postgres.partsupp.ps_suppkey = mongotpch.lineitem.l_suppkey and postgres.partsupp.ps_partkey = mongotpch.lineitem.l_partkey + join mongotpch.orders on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join tmpTableC on mongotpch.orders.o_orderkey = tmpTableC.orderkey + where + postgres.part.p_name like '.*green.*' + ) + group by + o_year, + nation + order by + o_year desc + ) +order by nation; \ No newline at end of file diff --git a/test/src/test/resources/polybench/sf0.1/q9.csv b/test/src/test/resources/polybench/sf0.1/q9.csv new file mode 100644 index 0000000000..e3ae1a01c4 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q9.csv @@ -0,0 +1,176 @@ +nation|o_year|sum_profit +ALGERIA|1998|2321785.3682 +ALGERIA|1997|3685016.8589 +ALGERIA|1996|4276597.4253 +ALGERIA|1995|4418370.4154 +ALGERIA|1994|3864849.9521 +ALGERIA|1993|3541051.3865 +ALGERIA|1992|4310013.3482 +ARGENTINA|1998|2685983.8005 +ARGENTINA|1997|4242147.8124 +ARGENTINA|1996|3907867.0103 +ARGENTINA|1995|4605921.5011 +ARGENTINA|1994|3542096.1564 +ARGENTINA|1993|3949965.9388 +ARGENTINA|1992|4521180.4695 +BRAZIL|1998|2778730.3931 +BRAZIL|1997|4642037.4687 +BRAZIL|1996|4530304.6034 +BRAZIL|1995|4502344.8657 +BRAZIL|1994|4875806.5015 +BRAZIL|1993|4687478.6531 +BRAZIL|1992|5035200.0464 +CANADA|1998|2194509.0465 +CANADA|1997|3482197.9521 +CANADA|1996|3712231.2814 +CANADA|1995|4014814.8476 +CANADA|1994|4145304.4855 +CANADA|1993|3787069.6045 +CANADA|1992|4168009.4201 +CHINA|1998|3398578.0001 +CHINA|1997|6358959.3338 +CHINA|1996|6435158.3229 +CHINA|1995|6174776.2113 +CHINA|1994|6385751.0812 +CHINA|1993|5765034.1194 +CHINA|1992|6324034.2379 +EGYPT|1998|2333148.3334 +EGYPT|1997|3661244.2731 +EGYPT|1996|3765371.2368 +EGYPT|1995|4094744.2925 +EGYPT|1994|3566508.0818 +EGYPT|1993|3725283.7747 +EGYPT|1992|3373762.3335 +ETHIOPIA|1998|1953927.2682 +ETHIOPIA|1997|3285786.3266 +ETHIOPIA|1996|3525028.7952 +ETHIOPIA|1995|3781674.8911 +ETHIOPIA|1994|3037409.4360 +ETHIOPIA|1993|3008978.2677 +ETHIOPIA|1992|2721203.2355 +FRANCE|1998|2604373.8805 +FRANCE|1997|3982872.0488 +FRANCE|1996|3622479.2413 +FRANCE|1995|4479939.7020 +FRANCE|1994|3531013.1981 +FRANCE|1993|4086437.3102 +FRANCE|1992|3637792.1333 +GERMANY|1998|3291023.2965 +GERMANY|1997|5139337.3443 +GERMANY|1996|4799810.4577 +GERMANY|1995|5405785.7978 +GERMANY|1994|4555556.4592 +GERMANY|1993|4428195.1019 +GERMANY|1992|4656148.4204 +INDIA|1998|2591288.1874 +INDIA|1997|5159562.7033 +INDIA|1996|5307258.3049 +INDIA|1995|5148208.7902 +INDIA|1994|5164001.9582 +INDIA|1993|4321398.4388 +INDIA|1992|5297703.6935 +INDONESIA|1998|3094900.1597 +INDONESIA|1997|5719773.0358 +INDONESIA|1996|6037238.5993 +INDONESIA|1995|5266783.4899 +INDONESIA|1994|5470762.8729 +INDONESIA|1993|6189826.6613 +INDONESIA|1992|4414623.1549 +IRAN|1998|3214864.1209 +IRAN|1997|3688049.0691 +IRAN|1996|3621649.2247 +IRAN|1995|4420783.4205 +IRAN|1994|4373984.6523 +IRAN|1993|3731301.7814 +IRAN|1992|4417133.3662 +IRAQ|1998|2338859.4099 +IRAQ|1997|3622681.5643 +IRAQ|1996|4762291.8722 +IRAQ|1995|4558092.7359 +IRAQ|1994|4951604.1699 +IRAQ|1993|3830077.9911 +IRAQ|1992|3938636.4874 +JAPAN|1998|1849535.0802 +JAPAN|1997|4068688.8537 +JAPAN|1996|4044774.7597 +JAPAN|1995|4793005.8027 +JAPAN|1994|4114717.0568 +JAPAN|1993|3614468.7485 +JAPAN|1992|4266694.4700 +JORDAN|1998|1811488.0719 +JORDAN|1997|2951297.8678 +JORDAN|1996|3302528.3067 +JORDAN|1995|3221813.9990 +JORDAN|1994|2417892.0921 +JORDAN|1993|3107641.7661 +JORDAN|1992|3316379.0585 +KENYA|1998|2579075.4190 +KENYA|1997|2929194.2317 +KENYA|1996|3569129.5619 +KENYA|1995|3542889.1087 +KENYA|1994|3983095.3994 +KENYA|1993|3713988.9708 +KENYA|1992|3304641.8340 +MOROCCO|1998|1815334.8180 +MOROCCO|1997|3693214.8447 +MOROCCO|1996|4116175.9230 +MOROCCO|1995|3515127.1402 +MOROCCO|1994|4003072.1120 +MOROCCO|1993|3599199.6679 +MOROCCO|1992|3958335.4224 +MOZAMBIQUE|1998|1620428.7346 +MOZAMBIQUE|1997|2802166.6473 +MOZAMBIQUE|1996|2409955.1755 +MOZAMBIQUE|1995|2771602.6274 +MOZAMBIQUE|1994|2548226.2158 +MOZAMBIQUE|1993|2843748.9053 +MOZAMBIQUE|1992|2556501.0943 +PERU|1998|2036430.3602 +PERU|1997|4064142.4091 +PERU|1996|4068678.5671 +PERU|1995|4657694.8412 +PERU|1994|4731959.4655 +PERU|1993|4144006.6610 +PERU|1992|3754635.0078 +ROMANIA|1998|1992773.6811 +ROMANIA|1997|2854639.8680 +ROMANIA|1996|3139337.3029 +ROMANIA|1995|3222153.3776 +ROMANIA|1994|3222844.3190 +ROMANIA|1993|3488994.0288 +ROMANIA|1992|3029274.4420 +RUSSIA|1998|2339865.6635 +RUSSIA|1997|4153619.5424 +RUSSIA|1996|3772067.4041 +RUSSIA|1995|4704988.8607 +RUSSIA|1994|4479082.8694 +RUSSIA|1993|4767719.9791 +RUSSIA|1992|4533465.5590 +SAUDI ARABIA|1998|3386948.9564 +SAUDI ARABIA|1997|5425980.3373 +SAUDI ARABIA|1996|5227607.1677 +SAUDI ARABIA|1995|4506731.6411 +SAUDI ARABIA|1994|4698658.7425 +SAUDI ARABIA|1993|5493626.5285 +SAUDI ARABIA|1992|4573560.0150 +UNITED KINGDOM|1998|2252021.5137 +UNITED KINGDOM|1997|4343926.8026 +UNITED KINGDOM|1996|4189476.3065 +UNITED KINGDOM|1995|4469569.8829 +UNITED KINGDOM|1994|4410094.6264 +UNITED KINGDOM|1993|4054677.1050 +UNITED KINGDOM|1992|3978688.8831 +UNITED STATES|1998|2238771.5581 +UNITED STATES|1997|4135581.5734 +UNITED STATES|1996|3624013.2660 +UNITED STATES|1995|3892244.5172 +UNITED STATES|1994|3289224.1138 +UNITED STATES|1993|3626170.2028 +UNITED STATES|1992|3993973.4997 +VIETNAM|1998|1924313.4862 +VIETNAM|1997|3436195.3709 +VIETNAM|1996|4017288.8927 +VIETNAM|1995|3644054.1372 +VIETNAM|1994|4141277.6665 +VIETNAM|1993|2556114.1693 +VIETNAM|1992|4090524.4905 diff --git a/test/src/test/resources/polybench/udf/udtf_extract_year.py b/test/src/test/resources/polybench/udf/udtf_extract_year.py index 332bfe1551..a78a183f21 100644 --- a/test/src/test/resources/polybench/udf/udtf_extract_year.py +++ b/test/src/test/resources/polybench/udf/udtf_extract_year.py @@ -1,4 +1,5 @@ -from datetime import datetime +from datetime import datetime, timezone +import pytz class UDFExtractYear: def __init__(self): pass @@ -6,7 +7,9 @@ def __init__(self): def extractYear(self, num): # Unix timestamp is in milliseconds timestamp_in_seconds = num / 1000 - dt = datetime.utcfromtimestamp(timestamp_in_seconds) + # TODO 直接将timestamp增加8小时 + tz = pytz.timezone('Asia/Shanghai') + dt = datetime.fromtimestamp(timestamp_in_seconds, tz=tz) return float(dt.year) def transform(self, data, args, kvargs): res = self.buildHeader(data) From 7e2da86d823aca675a07a5c887cbd1e09cbb8d08 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sat, 18 May 2024 22:41:18 +0800 Subject: [PATCH 098/229] change data generation process to downloading data --- .github/scripts/benchmarks/polystore.sh | 19 ------- .github/scripts/benchmarks/tpch.sh | 75 ++----------------------- 2 files changed, 4 insertions(+), 90 deletions(-) delete mode 100644 .github/scripts/benchmarks/polystore.sh diff --git a/.github/scripts/benchmarks/polystore.sh b/.github/scripts/benchmarks/polystore.sh deleted file mode 100644 index 501df04716..0000000000 --- a/.github/scripts/benchmarks/polystore.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -if [ "$RUNNER_OS" = "Linux" ]; then - - -elif [ "$RUNNER_OS" = "Windows" ]; then - echo "windows" - -elif [ "$RUNNER_OS" = "macOS" ]; then - # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 - cp makefile.suite makefile - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile - mv new_makefile makefile - - sed 's/#include /#include /' bm_utils.c > new_bm_utils.c - mv new_bm_utils.c bm_utils.c - sed 's/#include /#include /' varsub.c > new_varsub.c - mv new_varsub.c varsub.c -fi \ No newline at end of file diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 622cb973e1..abcd733d55 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -4,6 +4,10 @@ if [ "$RUNNER_OS" = "Windows" ]; then python thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ -s "." +else + python3 thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ + -s "/home/runner/work/Polystore-utils/Polystore-utils" cd tpchdata # 目标文件夹路径 destination_folder="../tpc/TPC-H V3.0.1/data" @@ -23,76 +27,5 @@ if [ "$RUNNER_OS" = "Windows" ]; then chmod +r supplier.tbl ls -a pwd -else - python3 thu_cloud_download.py \ - -l https://cloud.tsinghua.edu.cn/d/63b9d1a6444e47ae8ac5/ \ - -s "." - - cd tpc - pwd - unzip E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip - rm E86C7FBC-89AB-4825-BAFA-5CF73EC94ED3-TPC-H-Tool.zip - echo "TPCH数据生成工具下载完成" - - cd TPC-H\ V3.0.1/dbgen - if [ "$RUNNER_OS" = "Linux" ]; then - sh -c "sudo apt install gcc" - gcc --version - cp makefile.suite makefile - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile - mv new_makefile makefile - make - - elif [ "$RUNNER_OS" = "Windows" ]; then - echo "windows" - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = WIN32\nWORKLOAD = TPCH" }' makefile.suite > new_makefile - mv new_makefile makefile.suite - make -f makefile.suite - # https://juejin.cn/s/%E5%91%BD%E4%BB%A4%E8%A1%8C%E7%BC%96%E8%AF%91sln - # https://tedamoh.com/en/data-vault/78-generating-large-example-data-with-tpc-h - # msbuild tpch.sln /t:Build /p:Configuration=Debug - - - elif [ "$RUNNER_OS" = "macOS" ]; then - # 根据 https://blog.csdn.net/mei86233824/article/details/81066999 修改makefile文件并进行编译生成可执行文件 - cp makefile.suite makefile - awk 'NR < 103 || NR > 111 { print } NR == 103 { print "CC = gcc\n# Current values for DATABASE are: INFORMIX, DB2, TDAT (Teradata)\n# SQLSERVER, SYBASE, ORACLE, VECTORWISE\n# Current values for MACHINE are: ATT, DOS, HP, IBM, ICL, MVS, \n# SGI, SUN, U2200, VMS, LINUX, WIN32 \n# Current values for WORKLOAD are: TPCH\nDATABASE= SQLSERVER\nMACHINE = LINUX\nWORKLOAD = TPCH" }' makefile > new_makefile - mv new_makefile makefile - - sed 's/#include /#include /' bm_utils.c > new_bm_utils.c - mv new_bm_utils.c bm_utils.c - sed 's/#include /#include /' varsub.c > new_varsub.c - mv new_varsub.c varsub.c - make - fi - echo "TPCH数据生成工具编译完成" - - ./dbgen -s 0.1 - ls - echo "数据生成完成" - - # 源文件夹路径 - source_folder="." - - # 目标文件夹路径 - destination_folder="../data" - - # 确保目标文件夹存在，如果不存在则创建 - mkdir -p "$destination_folder" - - # 将所有*.tbl文件移动到目标文件夹 - mv *.tbl "$destination_folder/" - cd $destination_folder - - chmod +r customer.tbl - chmod +r lineitem.tbl - chmod +r nation.tbl - chmod +r orders.tbl - chmod +r region.tbl - chmod +r supplier.tbl - chmod +r part.tbl - chmod +r partsupp.tbl - ls -a - pwd echo "文件移动完成" fi \ No newline at end of file From deec1eb19b2694ea8182a6b1ba0ff2677fdf6a62 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sat, 18 May 2024 22:49:55 +0800 Subject: [PATCH 099/229] debug --- .github/scripts/benchmarks/tpch.sh | 4 +- .../workflows/polystore-benchmark-test.yml | 3 +- .../polybench/TPCHDataInsertionIT.java | 3 +- .../integration/polybench/TPCHRunner.java | 281 +++++++++--------- 4 files changed, 144 insertions(+), 147 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index abcd733d55..f02744fa58 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -7,7 +7,8 @@ if [ "$RUNNER_OS" = "Windows" ]; then else python3 thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ - -s "/home/runner/work/Polystore-utils/Polystore-utils" + -s "." +fi cd tpchdata # 目标文件夹路径 destination_folder="../tpc/TPC-H V3.0.1/data" @@ -28,4 +29,3 @@ else ls -a pwd echo "文件移动完成" -fi \ No newline at end of file diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 1b4f41486b..ea1269b0b9 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -93,13 +93,12 @@ jobs: mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - - name: Run session test if: always() shell: bash run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format # - name: Run session test # shell: bash diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 084685725e..21ce03b4ca 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -253,7 +253,8 @@ public void insertDataIntoPostgreSQL() { System.out.println("Executed SQL statement: " + sql); } CopyManager copyManager = new CopyManager((BaseConnection) conn); - List tableNames = Arrays.asList("customer", "supplier", "region", "part", "partsupp"); + List tableNames = + Arrays.asList("customer", "supplier", "region", "part", "partsupp"); for (String tableName : tableNames) { String filePath = String.format("%s/%s.tbl", dataPath, tableName); FileReader fileReader = new FileReader(filePath); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 5d134ff867..9861fd7c5f 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -5,169 +5,166 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.*; -import org.junit.Test; - import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.*; +import org.junit.Test; public class TPCHRunner { - // host info - protected static String defaultTestHost = "127.0.0.1"; - protected static int defaultTestPort = 6888; - protected static String defaultTestUser = "root"; - protected static String defaultTestPass = "root"; - protected static MultiConnection conn; - public static void TPCRunner(String[] args) {} + // host info + protected static String defaultTestHost = "127.0.0.1"; + protected static int defaultTestPort = 6888; + protected static String defaultTestUser = "root"; + protected static String defaultTestPass = "root"; + protected static MultiConnection conn; - private List> csvReader (String filePath) { - List> data = new ArrayList<>(); - boolean skipHeader = true; - try (Scanner scanner = new Scanner(Paths.get(filePath))) { - while (scanner.hasNextLine()) { - String line = scanner.nextLine(); - if(skipHeader) { - skipHeader = false; - continue; - } - List row = Arrays.asList(line.split("\\|")); - data.add(row); - } - } catch (IOException e) { - e.printStackTrace(); + public static void TPCRunner(String[] args) {} + + private List> csvReader(String filePath) { + List> data = new ArrayList<>(); + boolean skipHeader = true; + try (Scanner scanner = new Scanner(Paths.get(filePath))) { + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if (skipHeader) { + skipHeader = false; + continue; } - System.out.println(data); - return data; + List row = Arrays.asList(line.split("\\|")); + data.add(row); + } + } catch (IOException e) { + e.printStackTrace(); } + System.out.println(data); + return data; + } - public static String readSqlFileAsString(String filePath) throws IOException { - StringBuilder contentBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { - String line; - while ((line = br.readLine()) != null) { - contentBuilder.append(line).append("\n"); - } - } - return contentBuilder.toString(); + public static String readSqlFileAsString(String filePath) throws IOException { + StringBuilder contentBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = br.readLine()) != null) { + contentBuilder.append(line).append("\n"); + } } + return contentBuilder.toString(); + } - @Test - public void test() { - System.out.println("start"); - try { - conn = - new MultiConnection( - new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); - conn.openSession(); + @Test + public void test() { + System.out.println("start"); + try { + conn = + new MultiConnection( + new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); + conn.openSession(); - // 输出所有存储引擎 - String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); + // 输出所有存储引擎 + String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); - // 添加存储引擎 - System.out.println("start adding storage engine"); - long startTime = System.currentTimeMillis(); - Map pgMap = new HashMap<>(); - pgMap.put("has_data", "true"); - pgMap.put("is_read_only", "true"); - pgMap.put("username", "postgres"); - pgMap.put("password", "postgres"); - pgMap = Collections.unmodifiableMap(pgMap); - conn.addStorageEngine( - "127.0.0.1", - 5432, - StorageEngineType.postgresql, - pgMap - ); - Map mongoMap = new HashMap<>(); - mongoMap.put("has_data", "true"); - mongoMap.put("is_read_only", "true"); - mongoMap.put("schema.sample.size", "1000"); - mongoMap.put("dummy.sample.size", "0"); - conn.addStorageEngine( - "127.0.0.1", - 27017, - StorageEngineType.mongodb, - mongoMap - ); - System.out.println("end adding storage engine, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + // 添加存储引擎 + System.out.println("start adding storage engine"); + long startTime = System.currentTimeMillis(); + Map pgMap = new HashMap<>(); + pgMap.put("has_data", "true"); + pgMap.put("is_read_only", "true"); + pgMap.put("username", "postgres"); + pgMap.put("password", "postgres"); + pgMap = Collections.unmodifiableMap(pgMap); + conn.addStorageEngine("127.0.0.1", 5432, StorageEngineType.postgresql, pgMap); + Map mongoMap = new HashMap<>(); + mongoMap.put("has_data", "true"); + mongoMap.put("is_read_only", "true"); + mongoMap.put("schema.sample.size", "1000"); + mongoMap.put("dummy.sample.size", "0"); + conn.addStorageEngine("127.0.0.1", 27017, StorageEngineType.mongodb, mongoMap); + System.out.println( + "end adding storage engine, time cost: " + + (System.currentTimeMillis() - startTime) + + "ms"); - // 输出所有存储引擎 - clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); - //Long startTime; - // 13有问题 - List queryIds = Arrays.asList(1,2,5,6,9,10,17,18,19,20); - for (int queryId : queryIds) { - // read from sql file - String sqlString = readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); + // 输出所有存储引擎 + clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); + System.out.println(clusterInfo); + // Long startTime; + // 13有问题 + List queryIds = Arrays.asList(1, 2, 5, 6, 9, 10, 17, 18, 19, 20); + for (int queryId : queryIds) { + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); - // 开始 tpch 查询 - System.out.println("start tpch query " + queryId); - startTime = System.currentTimeMillis(); + // 开始 tpch 查询 + System.out.println("start tpch query " + queryId); + startTime = System.currentTimeMillis(); - // 执行查询语句, split by ; 最后一句为执行结果 - SessionExecuteSqlResult result = null; - String[] sqls = sqlString.split(";"); - for (String sql : sqls) { - if (sql.trim().length() == 0) { - continue; - } - sql += ";"; - result = conn.executeSql(sql); - result.print(false, ""); - } + // 执行查询语句, split by ; 最后一句为执行结果 + SessionExecuteSqlResult result = null; + String[] sqls = sqlString.split(";"); + for (String sql : sqls) { + if (sql.trim().length() == 0) { + continue; + } + sql += ";"; + result = conn.executeSql(sql); + result.print(false, ""); + } - // TODO 验证 - System.out.println("end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); - List> values = result.getValues(); - List> answers = csvReader("src/test/resources/polybench/sf0.1/q" + queryId + ".csv"); - if (values.size() != answers.size()) { - System.out.println("values.size() = " + values.size()); - System.out.println("answers.size() = " + answers.size()); - throw new RuntimeException("size not equal"); - } - for (int i = 0; i < values.size(); i++) { - if (values.get(i).size() != answers.get(i).size()) { - System.out.println("values.get(i).size() = " + values.get(i).size()); - System.out.println("answers.get(i).size() = " + answers.get(i).size()); - throw new RuntimeException("size not equal"); - } - for (int j = 0; j < values.get(i).size(); j++) { - System.out.println(values.get(i).get(j)); - if (result.getPaths().get(j).contains("address") - || result.getPaths().get(j).contains("comment") - || result.getPaths().get(j).contains("orderdate")) { - // skip s_address, data generation changed - continue; - } - // if only contains number and dot, then parse to double - if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { - double number = Double.parseDouble(values.get(i).get(j).toString()); - double answerNumber = Double.parseDouble(answers.get(i).get(j)); - System.out.println("Number: " + number); - System.out.println("Answer number: " + answerNumber); - assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; - } else { - String resultString = new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); - String answerString = answers.get(i).get(j); - System.out.println("Result string： " + resultString); - System.out.println("Answer string: " + answerString); - assert resultString.equals(answerString); - } - } - } + // TODO 验证 + System.out.println( + "end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + List> values = result.getValues(); + List> answers = + csvReader("src/test/resources/polybench/sf0.1/q" + queryId + ".csv"); + if (values.size() != answers.size()) { + System.out.println("values.size() = " + values.size()); + System.out.println("answers.size() = " + answers.size()); + throw new RuntimeException("size not equal"); + } + for (int i = 0; i < values.size(); i++) { + if (values.get(i).size() != answers.get(i).size()) { + System.out.println("values.get(i).size() = " + values.get(i).size()); + System.out.println("answers.get(i).size() = " + answers.get(i).size()); + throw new RuntimeException("size not equal"); + } + for (int j = 0; j < values.get(i).size(); j++) { + System.out.println(values.get(i).get(j)); + if (result.getPaths().get(j).contains("address") + || result.getPaths().get(j).contains("comment") + || result.getPaths().get(j).contains("orderdate")) { + // skip s_address, data generation changed + continue; } - - // 关闭会话 - conn.closeSession(); - } catch (SessionException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); + // if only contains number and dot, then parse to double + if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { + double number = Double.parseDouble(values.get(i).get(j).toString()); + double answerNumber = Double.parseDouble(answers.get(i).get(j)); + System.out.println("Number: " + number); + System.out.println("Answer number: " + answerNumber); + assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; + } else { + String resultString = + new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); + String answerString = answers.get(i).get(j); + System.out.println("Result string： " + resultString); + System.out.println("Answer string: " + answerString); + assert resultString.equals(answerString); + } + } } + } + + // 关闭会话 + conn.closeSession(); + } catch (SessionException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); } + } } From 2180df26b8109f64843c7054cc75629de63607c7 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 19 May 2024 00:44:34 +0800 Subject: [PATCH 100/229] debug timezone --- .github/scripts/benchmarks/tpch.sh | 14 +++++ .../test/resources/polybench/queries/q1.sql | 2 +- .../test/resources/polybench/queries/q10.sql | 2 +- .../test/resources/polybench/queries/q20.sql | 2 +- .../test/resources/polybench/queries/q3.sql | 31 ++++++++++ .../test/resources/polybench/queries/q5.sql | 2 +- .../test/resources/polybench/queries/q6.sql | 2 +- .../test/resources/polybench/queries/q9.sql | 56 +++++++++---------- .../src/test/resources/polybench/sf0.1/q3.csv | 11 ++++ 9 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 test/src/test/resources/polybench/queries/q3.sql create mode 100644 test/src/test/resources/polybench/sf0.1/q3.csv diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index f02744fa58..7d2ae546c8 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -4,6 +4,20 @@ if [ "$RUNNER_OS" = "Windows" ]; then python thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ -s "." + # 清除 polybench 目录中所有 .sql 文件中的 ' + 28800000' 字符串（时区问题） + dir="test/src/test/resources/polybench/queries/" + + # 遍历目录中的所有 .sql 文件 + for file in "$dir"*.sql; do + # 检查文件是否存在 + if [[ -f "$file" ]]; then + # 使用 sed 命令查找并替换 ' + 28800000' 字符串 + sed -i 's/ + 28800000//g' "$file" + echo "Processed $file" + else + echo "No .sql files found in the directory." + fi + done else python3 thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ diff --git a/test/src/test/resources/polybench/queries/q1.sql b/test/src/test/resources/polybench/queries/q1.sql index ea0de5113a..94d53ae15b 100644 --- a/test/src/test/resources/polybench/queries/q1.sql +++ b/test/src/test/resources/polybench/queries/q1.sql @@ -21,7 +21,7 @@ from ( from mongotpch.lineitem where - mongotpch.lineitem.l_shipdate <= 904665600000 + mongotpch.lineitem.l_shipdate <= 904665600000 + 28800000 ) group by mongotpch.lineitem.l_returnflag, diff --git a/test/src/test/resources/polybench/queries/q10.sql b/test/src/test/resources/polybench/queries/q10.sql index f18acba927..1f9e01094e 100644 --- a/test/src/test/resources/polybench/queries/q10.sql +++ b/test/src/test/resources/polybench/queries/q10.sql @@ -34,7 +34,7 @@ from ( join nation on postgres.customer.c_nationkey = nation.n_nationkey where mongotpch.orders.o_orderdate >= 749404800000 - and mongotpch.orders.o_orderdate < 757353600000 + and mongotpch.orders.o_orderdate < 757353600000 + 28800000 and mongotpch.lineitem.l_returnflag = 'R' ) group by diff --git a/test/src/test/resources/polybench/queries/q20.sql b/test/src/test/resources/polybench/queries/q20.sql index cf04669c2b..247b4be379 100644 --- a/test/src/test/resources/polybench/queries/q20.sql +++ b/test/src/test/resources/polybench/queries/q20.sql @@ -11,7 +11,7 @@ insert into tmpTableA(key, partkey, suppkey, val) values ( mongotpch.lineitem where mongotpch.lineitem.l_shipdate >= 757353600000 - and mongotpch.lineitem.l_shipdate < 788889600000 + and mongotpch.lineitem.l_shipdate < 788889600000 + 28800000 group by l_partkey, l_suppkey ) ); diff --git a/test/src/test/resources/polybench/queries/q3.sql b/test/src/test/resources/polybench/queries/q3.sql new file mode 100644 index 0000000000..a2353be764 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q3.sql @@ -0,0 +1,31 @@ +select l_orderkey, + revenue, + o_orderdate, + o_shippriority from + (select + l_orderkey, + o_orderdate, + o_shippriority, + sum(tmp) as revenue + from( + select + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp, + mongotpch.lineitem.l_orderkey as l_orderkey, + mongotpch.orders.o_orderdate as o_orderdate, + mongotpch.orders.o_shippriority as o_shippriority + from + postgres.customer + join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey + where + postgres.customer.c_mktsegment = 'BUILDING' + and mongotpch.orders.o_orderdate < 795225600000 + and mongotpch.lineitem.l_shipdate > 795196800000 + ) + group by + l_orderkey, + o_orderdate, + o_shippriority + ) +order by revenue desc + limit 10; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q5.sql b/test/src/test/resources/polybench/queries/q5.sql index 2a04d7c002..f8ea0c029d 100644 --- a/test/src/test/resources/polybench/queries/q5.sql +++ b/test/src/test/resources/polybench/queries/q5.sql @@ -19,7 +19,7 @@ from ( where postgres.region.r_name = "ASIA" and mongotpch.orders.o_orderdate >= 757353600000 - and mongotpch.orders.o_orderdate < 788889600000 + and mongotpch.orders.o_orderdate < 788889600000 + 28800000 ) group by nation.n_name diff --git a/test/src/test/resources/polybench/queries/q6.sql b/test/src/test/resources/polybench/queries/q6.sql index 5408a94ff8..06cf5533cb 100644 --- a/test/src/test/resources/polybench/queries/q6.sql +++ b/test/src/test/resources/polybench/queries/q6.sql @@ -5,7 +5,7 @@ select sum(tmp) from ( mongotpch.lineitem where mongotpch.lineitem.l_shipdate >= 757353600000 - and mongotpch.lineitem.l_shipdate < 788889600000 + and mongotpch.lineitem.l_shipdate < 788889600000 + 28800000 and mongotpch.lineitem.l_discount >= 0.05 and mongotpch.lineitem.l_discount <= 0.07 and mongotpch.lineitem.l_quantity < 24 diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql index df18b34c66..6bb5fb919b 100644 --- a/test/src/test/resources/polybench/queries/q9.sql +++ b/test/src/test/resources/polybench/queries/q9.sql @@ -1,34 +1,34 @@ CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py"; insert into tmpTableC(key, orderkey, year) values ( - select o_orderkey, extractYear(o_orderdate) from mongotpch.orders - ); + select o_orderkey, extractYear(o_orderdate) from mongotpch.orders +); select * from ( - select - nation, - o_year, - sum(amount) as sum_profit - from ( - select - nation.n_name as nation, - tmpTableC.year as o_year, - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) - postgres.partsupp.ps_supplycost * mongotpch.lineitem.l_quantity as amount - from - postgres.part - join mongotpch.lineitem on postgres.part.p_partkey = mongotpch.lineitem.l_partkey - join postgres.supplier on postgres.supplier.s_suppkey = mongotpch.lineitem.l_suppkey - join postgres.partsupp on postgres.partsupp.ps_suppkey = mongotpch.lineitem.l_suppkey and postgres.partsupp.ps_partkey = mongotpch.lineitem.l_partkey - join mongotpch.orders on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey - join tmpTableC on mongotpch.orders.o_orderkey = tmpTableC.orderkey - where - postgres.part.p_name like '.*green.*' - ) - group by - o_year, - nation - order by - o_year desc - ) + select + nation, + o_year, + sum(amount) as sum_profit + from ( + select + nation.n_name as nation, + tmpTableC.year as o_year, + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) - postgres.partsupp.ps_supplycost * mongotpch.lineitem.l_quantity as amount + from + postgres.part + join mongotpch.lineitem on postgres.part.p_partkey = mongotpch.lineitem.l_partkey + join postgres.supplier on postgres.supplier.s_suppkey = mongotpch.lineitem.l_suppkey + join postgres.partsupp on postgres.partsupp.ps_suppkey = mongotpch.lineitem.l_suppkey and postgres.partsupp.ps_partkey = mongotpch.lineitem.l_partkey + join mongotpch.orders on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey + join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join tmpTableC on mongotpch.orders.o_orderkey = tmpTableC.orderkey + where + postgres.part.p_name like '.*green.*' + ) + group by + o_year, + nation + order by + o_year desc +) order by nation; \ No newline at end of file diff --git a/test/src/test/resources/polybench/sf0.1/q3.csv b/test/src/test/resources/polybench/sf0.1/q3.csv new file mode 100644 index 0000000000..123b6936ab --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q3.csv @@ -0,0 +1,11 @@ +l_orderkey|revenue|o_orderdate|o_shippriority +223140|355369.0698|1995-03-14|0 +584291|354494.7318|1995-02-21|0 +405063|353125.4577|1995-03-03|0 +573861|351238.2770|1995-03-09|0 +554757|349181.7426|1995-03-14|0 +506021|321075.5810|1995-03-10|0 +121604|318576.4154|1995-03-07|0 +108514|314967.0754|1995-02-20|0 +462502|312604.5420|1995-03-08|0 +178727|309728.9306|1995-02-25|0 From 26584f692ab361787394ffe2630121ce67546359 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 19 May 2024 01:01:06 +0800 Subject: [PATCH 101/229] debug timezone --- .github/scripts/benchmarks/tpch.sh | 2 +- .../tsinghua/iginx/integration/polybench/TPCHRunner.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 7d2ae546c8..6046181f8d 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -12,7 +12,7 @@ if [ "$RUNNER_OS" = "Windows" ]; then # 检查文件是否存在 if [[ -f "$file" ]]; then # 使用 sed 命令查找并替换 ' + 28800000' 字符串 - sed -i 's/ + 28800000//g' "$file" + sed -i '' 's/ + 28800000//g' "$file" echo "Processed $file" else echo "No .sql files found in the directory." diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 9861fd7c5f..4572f0f319 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -93,7 +93,7 @@ public void test() { System.out.println(clusterInfo); // Long startTime; // 13有问题 - List queryIds = Arrays.asList(1, 2, 5, 6, 9, 10, 17, 18, 19, 20); + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = @@ -115,7 +115,7 @@ public void test() { result.print(false, ""); } - // TODO 验证 + // 验证 System.out.println( "end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); List> values = result.getValues(); @@ -137,7 +137,7 @@ public void test() { if (result.getPaths().get(j).contains("address") || result.getPaths().get(j).contains("comment") || result.getPaths().get(j).contains("orderdate")) { - // skip s_address, data generation changed + // TODO change unix time to date continue; } // if only contains number and dot, then parse to double From 648a00d08212722aa41635cce0f68000fba57449 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 19 May 2024 12:18:13 +0800 Subject: [PATCH 102/229] debug timezone --- .github/scripts/benchmarks/tpch.sh | 14 -------------- test/src/test/resources/polybench/queries/q1.sql | 2 +- test/src/test/resources/polybench/queries/q10.sql | 2 +- test/src/test/resources/polybench/queries/q20.sql | 2 +- test/src/test/resources/polybench/queries/q5.sql | 2 +- test/src/test/resources/polybench/queries/q6.sql | 2 +- 6 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 6046181f8d..f02744fa58 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -4,20 +4,6 @@ if [ "$RUNNER_OS" = "Windows" ]; then python thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ -s "." - # 清除 polybench 目录中所有 .sql 文件中的 ' + 28800000' 字符串（时区问题） - dir="test/src/test/resources/polybench/queries/" - - # 遍历目录中的所有 .sql 文件 - for file in "$dir"*.sql; do - # 检查文件是否存在 - if [[ -f "$file" ]]; then - # 使用 sed 命令查找并替换 ' + 28800000' 字符串 - sed -i '' 's/ + 28800000//g' "$file" - echo "Processed $file" - else - echo "No .sql files found in the directory." - fi - done else python3 thu_cloud_download.py \ -l https://cloud.tsinghua.edu.cn/d/740c158819bc4759a36e/ \ diff --git a/test/src/test/resources/polybench/queries/q1.sql b/test/src/test/resources/polybench/queries/q1.sql index 94d53ae15b..ca0cd2f221 100644 --- a/test/src/test/resources/polybench/queries/q1.sql +++ b/test/src/test/resources/polybench/queries/q1.sql @@ -21,7 +21,7 @@ from ( from mongotpch.lineitem where - mongotpch.lineitem.l_shipdate <= 904665600000 + 28800000 + mongotpch.lineitem.l_shipdate <= 904694400000 ) group by mongotpch.lineitem.l_returnflag, diff --git a/test/src/test/resources/polybench/queries/q10.sql b/test/src/test/resources/polybench/queries/q10.sql index 1f9e01094e..f18acba927 100644 --- a/test/src/test/resources/polybench/queries/q10.sql +++ b/test/src/test/resources/polybench/queries/q10.sql @@ -34,7 +34,7 @@ from ( join nation on postgres.customer.c_nationkey = nation.n_nationkey where mongotpch.orders.o_orderdate >= 749404800000 - and mongotpch.orders.o_orderdate < 757353600000 + 28800000 + and mongotpch.orders.o_orderdate < 757353600000 and mongotpch.lineitem.l_returnflag = 'R' ) group by diff --git a/test/src/test/resources/polybench/queries/q20.sql b/test/src/test/resources/polybench/queries/q20.sql index 247b4be379..cf04669c2b 100644 --- a/test/src/test/resources/polybench/queries/q20.sql +++ b/test/src/test/resources/polybench/queries/q20.sql @@ -11,7 +11,7 @@ insert into tmpTableA(key, partkey, suppkey, val) values ( mongotpch.lineitem where mongotpch.lineitem.l_shipdate >= 757353600000 - and mongotpch.lineitem.l_shipdate < 788889600000 + 28800000 + and mongotpch.lineitem.l_shipdate < 788889600000 group by l_partkey, l_suppkey ) ); diff --git a/test/src/test/resources/polybench/queries/q5.sql b/test/src/test/resources/polybench/queries/q5.sql index f8ea0c029d..2a04d7c002 100644 --- a/test/src/test/resources/polybench/queries/q5.sql +++ b/test/src/test/resources/polybench/queries/q5.sql @@ -19,7 +19,7 @@ from ( where postgres.region.r_name = "ASIA" and mongotpch.orders.o_orderdate >= 757353600000 - and mongotpch.orders.o_orderdate < 788889600000 + 28800000 + and mongotpch.orders.o_orderdate < 788889600000 ) group by nation.n_name diff --git a/test/src/test/resources/polybench/queries/q6.sql b/test/src/test/resources/polybench/queries/q6.sql index 06cf5533cb..5408a94ff8 100644 --- a/test/src/test/resources/polybench/queries/q6.sql +++ b/test/src/test/resources/polybench/queries/q6.sql @@ -5,7 +5,7 @@ select sum(tmp) from ( mongotpch.lineitem where mongotpch.lineitem.l_shipdate >= 757353600000 - and mongotpch.lineitem.l_shipdate < 788889600000 + 28800000 + and mongotpch.lineitem.l_shipdate < 788889600000 and mongotpch.lineitem.l_discount >= 0.05 and mongotpch.lineitem.l_discount <= 0.07 and mongotpch.lineitem.l_quantity < 24 From d57c1158654eba8893884eeec1fbff47b62fb4cb Mon Sep 17 00:00:00 2001 From: Janet731 Date: Sun, 19 May 2024 12:40:39 +0800 Subject: [PATCH 103/229] debug timezone --- test/src/test/resources/polybench/queries/q3.sql | 4 ++-- test/src/test/resources/polybench/queries/q9.sql | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/src/test/resources/polybench/queries/q3.sql b/test/src/test/resources/polybench/queries/q3.sql index a2353be764..95f0dc333c 100644 --- a/test/src/test/resources/polybench/queries/q3.sql +++ b/test/src/test/resources/polybench/queries/q3.sql @@ -19,8 +19,8 @@ select l_orderkey, join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey where postgres.customer.c_mktsegment = 'BUILDING' - and mongotpch.orders.o_orderdate < 795225600000 - and mongotpch.lineitem.l_shipdate > 795196800000 + and mongotpch.orders.o_orderdate < 795196800000 + and mongotpch.lineitem.l_shipdate > 795225600000 ) group by l_orderkey, diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql index 6bb5fb919b..4040ebb5d3 100644 --- a/test/src/test/resources/polybench/queries/q9.sql +++ b/test/src/test/resources/polybench/queries/q9.sql @@ -1,4 +1,3 @@ -CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py"; insert into tmpTableC(key, orderkey, year) values ( select o_orderkey, extractYear(o_orderdate) from mongotpch.orders From 8a4384c7c4476a9edffdefd7ddf755b6edb3ee2d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 09:10:09 +0800 Subject: [PATCH 104/229] debug --- test/src/test/resources/polybench/queries/q9.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql index 4040ebb5d3..6bb5fb919b 100644 --- a/test/src/test/resources/polybench/queries/q9.sql +++ b/test/src/test/resources/polybench/queries/q9.sql @@ -1,3 +1,4 @@ +CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py"; insert into tmpTableC(key, orderkey, year) values ( select o_orderkey, extractYear(o_orderdate) from mongotpch.orders From ed7561c6836eaf80c94f7898932da72aac64626f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 09:22:51 +0800 Subject: [PATCH 105/229] move data cleaning dir --- .github/scripts/benchmarks/dataCleaning.sh | 2 +- dataCleaning/gen_data.py | 25 ------------------- .../polybench/DataCleaningRunner.py | 7 ++++-- 3 files changed, 6 insertions(+), 28 deletions(-) delete mode 100644 dataCleaning/gen_data.py diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh index 3d078953f2..8a238dcace 100755 --- a/.github/scripts/benchmarks/dataCleaning.sh +++ b/.github/scripts/benchmarks/dataCleaning.sh @@ -2,7 +2,7 @@ set -e -COMMAND1='LOAD DATA FROM INFILE "dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' +COMMAND1='LOAD DATA FROM INFILE "test/src/test/resources/dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" diff --git a/dataCleaning/gen_data.py b/dataCleaning/gen_data.py deleted file mode 100644 index ae9354e4c5..0000000000 --- a/dataCleaning/gen_data.py +++ /dev/null @@ -1,25 +0,0 @@ -# 生成 zipcode -> city的映射关系，error rate 10% -# lineNum 为 -n 后面的参数 -import argparse -parser = argparse.ArgumentParser(description='generate zipcode -> city data for test') -parser.add_argument('-n', dest='lineNum', type=int, default=1000000, - help='line number of the generated data file, default 1000000') - -args = parser.parse_args() -print('Data generated! Line number: ', args.lineNum) -if args.lineNum < 0: - print('LineNum should be a positive number') - exit(1) -elif args.lineNum > 10000000: - print('LineNum too big, should be less than 10000000') - exit(1) - -correctNum = args.lineNum - args.lineNum // 10 - -zipcodes = [i for i in range(correctNum)] -cities = ['city' + str(i) for i in range(args.lineNum)] -# zipcodes加入10%的重复 -zipcodes = zipcodes + [zipcodes[i] for i in range(args.lineNum // 10)] -with open('dataCleaning/zipcode_city.csv', 'w') as f: - for i in range(args.lineNum): - f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') \ No newline at end of file diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py index d89a90f48c..afb9678437 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import subprocess, platform +import subprocess, platform, os # 生成 zipcode -> city的映射关系，error rate 10% # lineNum 为 -n 后面的参数 @@ -23,7 +23,10 @@ cities = ['city' + str(i) for i in range(args.lineNum)] # zipcodes加入10%的重复 zipcodes = zipcodes + [zipcodes[i] for i in range(args.lineNum // 10)] -with open('dataCleaning/zipcode_city.csv', 'w') as f: +# mkdir if not exist +if not os.path.exists('test/src/test/resources/dataCleaning'): + os.makedirs('test/src/test/resources/dataCleaning') +with open('test/src/test/resources/dataCleaning/zipcode_city.csv', 'w') as f: for i in range(args.lineNum): f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') From 11b41e6c1911788e8ce26eb7ba10e1b817f507ff Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 09:37:20 +0800 Subject: [PATCH 106/229] timecost --- .../tsinghua/iginx/integration/polybench/TPCHRunner.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 4572f0f319..af2c1cfcf9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -94,6 +94,7 @@ public void test() { // Long startTime; // 13有问题 List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 17, 18, 19, 20); + List runTimes = new ArrayList<>(); for (int queryId : queryIds) { // read from sql file String sqlString = @@ -116,8 +117,10 @@ public void test() { } // 验证 + Long timeCost = System.currentTimeMillis() - startTime; + runTimes.add(timeCost); System.out.println( - "end tpch query, time cost: " + (System.currentTimeMillis() - startTime) + "ms"); + "end tpch query, time cost: " + timeCost + "ms"); List> values = result.getValues(); List> answers = csvReader("src/test/resources/polybench/sf0.1/q" + queryId + ".csv"); @@ -158,6 +161,9 @@ public void test() { } } } + for (int i = 0; i < queryIds.size(); i++) { + System.out.println("Query " + queryIds.get(i) + " time cost: " + runTimes.get(i) + "ms"); + } // 关闭会话 conn.closeSession(); From acf2044b8de8f881dd807f5f694e921e6e43b87f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 11:27:24 +0800 Subject: [PATCH 107/229] try regression --- .../workflows/polystore-benchmark-test.yml | 77 +++++++++++++++++++ .../integration/polybench/TPCHRunner.java | 15 ++-- 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index ea1269b0b9..52707fc123 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -100,6 +100,83 @@ jobs: run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + - name: Get old version + if: always() + shell: bash + run: | + git clone https://github.com/IGinX-THU/IGinX.git + cd IGinX + export GITHUB_WORKSPACE=$(pwd) + mvn clean package -DskipTests -P-format -q + + - name: Stop ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-stop: "true" + + - name: Rerun ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-rerun: "true" + + - name: Run DB + shell: bash + run: | + cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" + cd IGinX + if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then + chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" + ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/dataSources/parquet_macos.sh" + ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + + - name: Change IGinX config + uses: ./IGinX/.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./IGinX/.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./IGinX/.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./IGinX/.github/actions/context + with: + DB-name: "Parquet" + shell: client-before + + - name: Start storing data + if: always() + shell: bash + run: | + cp .github/scripts/benchmarks/parquet.sh IGinX/.github/scripts/benchmarks/parquet.sh + cd IGinX + export GITHUB_WORKSPACE=$(pwd) + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + + - name: Run session test + if: always() + shell: bash + run: | + cp test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java IGinX/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java + cd IGinX + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format # - name: Run session test # shell: bash # run: | diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index af2c1cfcf9..a531c11193 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -119,8 +119,7 @@ public void test() { // 验证 Long timeCost = System.currentTimeMillis() - startTime; runTimes.add(timeCost); - System.out.println( - "end tpch query, time cost: " + timeCost + "ms"); + System.out.println("end tpch query, time cost: " + timeCost + "ms"); List> values = result.getValues(); List> answers = csvReader("src/test/resources/polybench/sf0.1/q" + queryId + ".csv"); @@ -147,15 +146,19 @@ public void test() { if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { double number = Double.parseDouble(values.get(i).get(j).toString()); double answerNumber = Double.parseDouble(answers.get(i).get(j)); - System.out.println("Number: " + number); - System.out.println("Answer number: " + answerNumber); + if(answerNumber - number >= 1e-3 || number - answerNumber >= 1e-3){ + System.out.println("Number: " + number); + System.out.println("Answer number: " + answerNumber); + } assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; } else { String resultString = new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); String answerString = answers.get(i).get(j); - System.out.println("Result string： " + resultString); - System.out.println("Answer string: " + answerString); + if(!resultString.equals(answerString)){ + System.out.println("Result string： " + resultString); + System.out.println("Answer string: " + answerString); + } assert resultString.equals(answerString); } } From 6325cba68375bd59678c06a9fc2c91ec09422901 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 16:39:09 +0800 Subject: [PATCH 108/229] debug try regression --- .../workflows/polystore-benchmark-test.yml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 52707fc123..7129016779 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -157,14 +157,28 @@ jobs: - name: set client test context uses: ./IGinX/.github/actions/context with: - DB-name: "Parquet" - shell: client-before + name: Pre Test Client Export File + shell: bash + run: | + cd IGinX + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x ".github/scripts/test/cli/test_outfile.sh" + ".github/scripts/test/cli/test_outfile.sh" "Parquet + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" + ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" + ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" + fi - name: Start storing data if: always() shell: bash run: | - cp .github/scripts/benchmarks/parquet.sh IGinX/.github/scripts/benchmarks/parquet.sh + pwd + ls + cp .github/scripts/benchmarks/parquet.sh ./IGinX/.github/scripts/benchmarks/parquet.sh cd IGinX export GITHUB_WORKSPACE=$(pwd) chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" From 98bd53838856c5eaa3a72ab1472965c08789ebcd Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 17:48:10 +0800 Subject: [PATCH 109/229] debug try regression --- .github/workflows/polystore-benchmark-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 7129016779..852145900a 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -178,6 +178,7 @@ jobs: run: | pwd ls + mkdir ./IGinX/.github/scripts/benchmarks cp .github/scripts/benchmarks/parquet.sh ./IGinX/.github/scripts/benchmarks/parquet.sh cd IGinX export GITHUB_WORKSPACE=$(pwd) @@ -188,6 +189,7 @@ jobs: if: always() shell: bash run: | + mkdir IGinX/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench cp test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java IGinX/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java cd IGinX mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format From e3021a27686a0dfdf578f1a6eb3adf971637c15f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 20:42:48 +0800 Subject: [PATCH 110/229] debug try regression --- .../workflows/polystore-benchmark-test.yml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 852145900a..766b092425 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -178,12 +178,22 @@ jobs: run: | pwd ls - mkdir ./IGinX/.github/scripts/benchmarks - cp .github/scripts/benchmarks/parquet.sh ./IGinX/.github/scripts/benchmarks/parquet.sh + mkdir ./IGinX/tpc/TPC-H V3.0.1/data + cp tpc/TPC-H V3.0.1/data/nation.csv IGinX/tpc/TPC-H V3.0.1/data/nation.csv cd IGinX - export GITHUB_WORKSPACE=$(pwd) - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + # 插入数据 + COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" + SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + + if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" + elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" + elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" + fi - name: Run session test if: always() From 0026c44e47f67db33da571a8bc2570f0b08c11a9 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 20 May 2024 20:51:39 +0800 Subject: [PATCH 111/229] debug try regression --- .github/workflows/polystore-benchmark-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 766b092425..ca90cf53fc 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -178,7 +178,9 @@ jobs: run: | pwd ls - mkdir ./IGinX/tpc/TPC-H V3.0.1/data + mkdir IGinX/tpc + mkdir "IGinX/tpc/TPC-H V3.0.1" + mkdir "IGinX/tpc/TPC-H V3.0.1/data" cp tpc/TPC-H V3.0.1/data/nation.csv IGinX/tpc/TPC-H V3.0.1/data/nation.csv cd IGinX output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" From 979bb692beb3172099178e9b5a43acb1acb3bd07 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 09:50:46 +0800 Subject: [PATCH 112/229] debug try regression --- .github/workflows/polystore-benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index ca90cf53fc..fab7d3ef13 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -181,7 +181,7 @@ jobs: mkdir IGinX/tpc mkdir "IGinX/tpc/TPC-H V3.0.1" mkdir "IGinX/tpc/TPC-H V3.0.1/data" - cp tpc/TPC-H V3.0.1/data/nation.csv IGinX/tpc/TPC-H V3.0.1/data/nation.csv + cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" cd IGinX output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" # 插入数据 From 43c2a3fe4c16b0793a986065ff385a542dbf668d Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 10:19:28 +0800 Subject: [PATCH 113/229] debug --- .github/workflows/polystore-benchmark-test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index fab7d3ef13..3cc0a4a444 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -106,8 +106,7 @@ jobs: run: | git clone https://github.com/IGinX-THU/IGinX.git cd IGinX - export GITHUB_WORKSPACE=$(pwd) - mvn clean package -DskipTests -P-format -q + mvn clean package -DskipTests -P-format - name: Stop ZooKeeper uses: ./.github/actions/zookeeperRunner @@ -201,9 +200,6 @@ jobs: if: always() shell: bash run: | - mkdir IGinX/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench - cp test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java IGinX/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java - cd IGinX mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format # - name: Run session test # shell: bash From a9cc5abe27ed1cec466242dce9fea5529586dd07 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 10:41:23 +0800 Subject: [PATCH 114/229] add new version sign to blockline. --- .../actions/regressionTestRunner/action.yml | 57 +++++++++++++++++++ .../workflows/polystore-benchmark-test.yml | 8 ++- .../edu/tsinghua/iginx/utils/FormatUtils.java | 2 +- .../integration/polybench/TPCHRunner.java | 1 - 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 .github/actions/regressionTestRunner/action.yml diff --git a/.github/actions/regressionTestRunner/action.yml b/.github/actions/regressionTestRunner/action.yml new file mode 100644 index 0000000000..3d989de8c2 --- /dev/null +++ b/.github/actions/regressionTestRunner/action.yml @@ -0,0 +1,57 @@ +name: "regression-iginx-runner" +description: "regression iginx runner" +inputs: + version: + description: "iginx runner version" + required: false + default: 0.6.0-SNAPSHOT + if-stop: + description: "to stop the iginx" + required: false + default: "false" + if-test-udf: + description: "to test UDF path detection" + required: false + default: "false" +runs: + using: "composite" # Mandatory parameter + steps: + - if: inputs.if-test-udf=='true' + name: Test UDF Path + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "Windows" ]; then + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_udf_path.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_udf_path.sh" + + - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' + name: Start IGinX + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_windows.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_windows.sh" 6888 6666 + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_macos.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_macos.sh" 6888 6666 + fi + + - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' + name: Stop IGinX + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_kill.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 3cc0a4a444..e7df126980 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -141,8 +141,14 @@ jobs: Set-Filter-Fragment-OFF: "true" # start udf path test first to avoid being effected + - name: Stop IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + - name: Start IGinX - uses: ./IGinX/.github/actions/iginxRunner + uses: ./.github/actions/regressionTestRunner with: version: ${VERSION} if-test-udf: "true" diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java index 28a4ecd991..4aa9aa51e7 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java @@ -42,7 +42,7 @@ private static String buildBlockLine(List maxSizeList) { for (Integer integer : maxSizeList) { blockLine.append("+").append(StringUtils.repeat("-", integer)); } - blockLine.append("+").append("\n"); + blockLine.append("+").append("New Version").append("\n"); return blockLine.toString(); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index a531c11193..408f9f1efc 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -135,7 +135,6 @@ public void test() { throw new RuntimeException("size not equal"); } for (int j = 0; j < values.get(i).size(); j++) { - System.out.println(values.get(i).get(j)); if (result.getPaths().get(j).contains("address") || result.getPaths().get(j).contains("comment") || result.getPaths().get(j).contains("orderdate")) { From 146a07330ca8269f0470f4bebb502793d3f0b319 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 11:09:01 +0800 Subject: [PATCH 115/229] add new version sign to blockline. --- .github/actions/regressionTestRunner/action.yml | 3 +++ .github/scripts/iginx/iginx_udf_path.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/regressionTestRunner/action.yml b/.github/actions/regressionTestRunner/action.yml index 3d989de8c2..ba473c9c8c 100644 --- a/.github/actions/regressionTestRunner/action.yml +++ b/.github/actions/regressionTestRunner/action.yml @@ -31,6 +31,7 @@ runs: echo "$RUNNER_OS is not supported" exit 1 fi + cd IGinX chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_udf_path.sh" "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_udf_path.sh" @@ -38,6 +39,7 @@ runs: name: Start IGinX shell: bash run: | + cd IGinX if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 @@ -53,5 +55,6 @@ runs: name: Stop IGinX shell: bash run: | + cd IGinX chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_kill.sh" "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/scripts/iginx/iginx_udf_path.sh b/.github/scripts/iginx/iginx_udf_path.sh index 11a857cc27..a23b1818a8 100644 --- a/.github/scripts/iginx/iginx_udf_path.sh +++ b/.github/scripts/iginx/iginx_udf_path.sh @@ -1,5 +1,5 @@ #!/bin/sh - +pwd set -e cd core/target/iginx-core-0.6.0-SNAPSHOT/ From e3f7f48f32c10fb3989ea3821a8790c81e05e4d6 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 11:21:31 +0800 Subject: [PATCH 116/229] debug --- .../actions/regressionTestRunner/action.yml | 6 ++--- .../iginx/iginx_regression_udf_path.sh | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/iginx/iginx_regression_udf_path.sh diff --git a/.github/actions/regressionTestRunner/action.yml b/.github/actions/regressionTestRunner/action.yml index ba473c9c8c..72ba08f5ca 100644 --- a/.github/actions/regressionTestRunner/action.yml +++ b/.github/actions/regressionTestRunner/action.yml @@ -20,6 +20,7 @@ runs: name: Test UDF Path shell: bash run: | + pwd if [ "$RUNNER_OS" == "Linux" ]; then sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then @@ -31,9 +32,8 @@ runs: echo "$RUNNER_OS is not supported" exit 1 fi - cd IGinX - chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_udf_path.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_udf_path.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_regression_udf_path.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_regression_udf_path.sh" - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' name: Start IGinX diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh new file mode 100644 index 0000000000..fbafcc8f5c --- /dev/null +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +cd IGinX +pwd +set -e + +cd core/target/iginx-core-0.6.0-SNAPSHOT/ + +iginx_home_path=$PWD + +cd .. + +if [ -n "$MSYSTEM" ]; then + windows_path=$(cygpath -w "$iginx_home_path") + + export IGINX_HOME=$windows_path + + powershell -Command "Start-Process -FilePath 'iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.bat' -NoNewWindow -RedirectStandardOutput '../../iginx-udf.log' -RedirectStandardError '../../iginx-udf-error.log'" +else + export IGINX_HOME=$iginx_home_path + + sh -c "chmod +x iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh" + + sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &" +fi From 06ebe2cdc457add1bd4dfd83d441922b4864968a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 16:27:09 +0800 Subject: [PATCH 117/229] debug2 --- .github/scripts/iginx/iginx_regression_udf_path.sh | 2 ++ .github/workflows/polystore-benchmark-test.yml | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index fbafcc8f5c..5ab29da7e6 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -8,6 +8,8 @@ cd core/target/iginx-core-0.6.0-SNAPSHOT/ iginx_home_path=$PWD +echo "Iginx home path: $iginx_home_path" + cd .. if [ -n "$MSYSTEM" ]; then diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index e7df126980..9e706e9900 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -113,6 +113,12 @@ jobs: with: if-stop: "true" + - name: Stop IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + - name: Rerun ZooKeeper uses: ./.github/actions/zookeeperRunner with: @@ -141,11 +147,6 @@ jobs: Set-Filter-Fragment-OFF: "true" # start udf path test first to avoid being effected - - name: Stop IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-stop: "true" - name: Start IGinX uses: ./.github/actions/regressionTestRunner From 5499ec68fa3c86367ae6c37d42b6fe4a57a2e963 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 21 May 2024 16:41:38 +0800 Subject: [PATCH 118/229] debug3 --- .github/actions/confWriter/action.yml | 96 ++++++++++--------- .../workflows/polystore-benchmark-test.yml | 3 +- 2 files changed, 52 insertions(+), 47 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 66b2314310..9ac9d7913d 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -25,6 +25,10 @@ inputs: description: "which metadata service to use" required: false default: zookeeper + Root-Dir-Path: + description: "the path of IGinX root directory" + required: false + default: "${GITHUB_WORKSPACE}" runs: using: "composite" # Mandatory parameter @@ -33,53 +37,53 @@ runs: name: save config for FileSystem and Parquet shell: bash run: | - cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties" + cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties" - if: inputs.DB-name=='FileSystem' || inputs.DB-name=='Parquet' name: save config for FileSystem and Parquet shell: bash run: | - cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties" + cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties" - name: Set if-CapExp shell: bash run: | - echo "${{ inputs.if-CapExp }}" > ${GITHUB_WORKSPACE}/test/src/test/resources/isScaling.txt + echo "${{ inputs.if-CapExp }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/isScaling.txt - if: inputs.if-CapExp=='true' name: Change has_data shell: bash run: | - echo "${{ inputs.Test-Way }}" > ${GITHUB_WORKSPACE}/test/src/test/resources/dbce-test-way.txt + echo "${{ inputs.Test-Way }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/dbce-test-way.txt if [[ "${{ inputs.Test-Way }}" == "oriHasDataExpHasData" || "${{ inputs.Test-Way }}" == "oriHasDataExpNoData" ]]; then if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - sed -i "s/has_data=false/has_data=true/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + sed -i "s/has_data=false/has_data=true/g" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sed -i "" "s/has_data=false/has_data=true/" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + sed -i "" "s/has_data=false/has_data=true/" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties fi elif [[ "${{ inputs.Test-Way }}" == "oriNoDataExpHasData" || "${{ inputs.Test-Way }}" == "oriNoDataExpNoData" ]]; then if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - sed -i "s/has_data=true/has_data=false/g" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + sed -i "s/has_data=true/has_data=false/g" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sed -i "" "s/has_data=true/has_data=false/" ${GITHUB_WORKSPACE}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties + sed -i "" "s/has_data=true/has_data=false/" ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-0.6.0-SNAPSHOT/conf/config.properties fi fi - name: Set DB-name shell: bash run: | - echo "${{ inputs.DB-name }}" > ${GITHUB_WORKSPACE}/test/src/test/resources/DBName.txt + echo "${{ inputs.DB-name }}" > ${{ inputs.Root-Dir-Path }}/test/src/test/resources/DBName.txt - name: Change UDF conf shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -90,11 +94,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/is_read_only=false/is_read_only=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/is_read_only=false/is_read_only=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/is_read_only=false/is_read_only=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/is_read_only=false/is_read_only=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/is_read_only=false/is_read_only=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/is_read_only=false/is_read_only=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -104,14 +108,14 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/queryOptimizer=rbo/queryOptimizer=rbo,filter_push_down/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/queryOptimizer=rbo/queryOptimizer=rbo,filter_push_down/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/enablePushDown=false/enablePushDown=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/queryOptimizer=rbo/queryOptimizer=rbo,filter_push_down/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/queryOptimizer=rbo/queryOptimizer=rbo,filter_push_down/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/enablePushDown=false/enablePushDown=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i '' 's/queryOptimizer=rbo/queryOptimizer=rbo,filter_push_down/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i '' 's/queryOptimizer=rbo/queryOptimizer=rbo,filter_push_down/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -121,11 +125,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -135,11 +139,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -148,11 +152,11 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/log4j2.properties + sudo sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/log4j2.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/log4j2.properties + sed -i 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/log4j2.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/log4j2.properties + sudo sed -i '' 's/^logger.iginx.level=.*$/logger.iginx.level=debug/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/log4j2.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -161,14 +165,14 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties - sudo sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties - sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sed -i 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sed -i 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties - sudo sed -i '' 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i '' 's/^default.transformerRule.include=.*$/default.transformerRule.include=glob:**.denied/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties + sudo sed -i '' 's/^default.transformerRule.write=.*$/default.transformerRule.write=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/file-permission.properties else echo "$RUNNER_OS is not supported" exit 1 @@ -179,17 +183,17 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/^metaStorage=.*$/metaStorage=etcd/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/^zookeeperConnectionString=/#zookeeperConnectionString=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/^#etcdEndpoints=/etcdEndpoints=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/^metaStorage=.*$/metaStorage=etcd/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i '' 's/^zookeeperConnectionString=$/#zookeeperConnectionString=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sudo sed -i '' 's/^#etcdEndpoints=/etcdEndpoints=/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/^metaStorage=.*$/metaStorage=etcd/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/^zookeeperConnectionString=$/#zookeeperConnectionString=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/^#etcdEndpoints=/etcdEndpoints=/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 9e706e9900..070710918b 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -141,10 +141,11 @@ jobs: fi - name: Change IGinX config - uses: ./IGinX/.github/actions/confWriter + uses: ./.github/actions/confWriter with: DB-name: "Parquet" Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "IGinX" # start udf path test first to avoid being effected From 05a9d558990355a5df2816861264720b4832ca50 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 10:16:47 +0800 Subject: [PATCH 119/229] debug4 --- .github/scripts/iginx/iginx_regression_udf_path.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index 5ab29da7e6..dd5ebfb45e 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -20,6 +20,8 @@ if [ -n "$MSYSTEM" ]; then powershell -Command "Start-Process -FilePath 'iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.bat' -NoNewWindow -RedirectStandardOutput '../../iginx-udf.log' -RedirectStandardError '../../iginx-udf-error.log'" else export IGINX_HOME=$iginx_home_path + echo "Iginx home path: $IGINX_HOME" + pwd sh -c "chmod +x iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh" From 2a03067bcba6c4e12b34ec57d6f8f96dea11b5d0 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 11:22:31 +0800 Subject: [PATCH 120/229] debug5 --- .github/scripts/iginx/iginx_regression_udf_path.sh | 1 + .../src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index dd5ebfb45e..24913f8af6 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -1,5 +1,6 @@ #!/bin/sh +rm udf_funcs/python_scripts/udtf_extract_year.py cd IGinX pwd set -e diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java index 4aa9aa51e7..28a4ecd991 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FormatUtils.java @@ -42,7 +42,7 @@ private static String buildBlockLine(List maxSizeList) { for (Integer integer : maxSizeList) { blockLine.append("+").append(StringUtils.repeat("-", integer)); } - blockLine.append("+").append("New Version").append("\n"); + blockLine.append("+").append("\n"); return blockLine.toString(); } From 07ea20d2b18fb8fd7a55901f5a9e1ff8328644f9 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 14:43:52 +0800 Subject: [PATCH 121/229] debug6 --- .github/scripts/iginx/iginx_regression_udf_path.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index 24913f8af6..8484c94c53 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -1,6 +1,10 @@ #!/bin/sh rm udf_funcs/python_scripts/udtf_extract_year.py +ls +find . -name "udtf_extract_year.py" +find . -name "filename" -exec rm -f {} \; + cd IGinX pwd set -e From eff330fd055732c028e651ae0381ef6bc7dbc22b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 15:09:40 +0800 Subject: [PATCH 122/229] debug7 --- .github/scripts/iginx/iginx_regression_udf_path.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index 8484c94c53..879499fb0a 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -1,9 +1,8 @@ #!/bin/sh -rm udf_funcs/python_scripts/udtf_extract_year.py ls find . -name "udtf_extract_year.py" -find . -name "filename" -exec rm -f {} \; +rm ./core/target/iginx-core-0.6.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py cd IGinX pwd From 45573c17faccd75287957fbd608356173ad025d9 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 15:53:07 +0800 Subject: [PATCH 123/229] debug8 --- .github/scripts/iginx/iginx_regression_udf_path.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index 879499fb0a..3abc164316 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -15,6 +15,8 @@ iginx_home_path=$PWD echo "Iginx home path: $iginx_home_path" cd .. +lsof -i:27017 +lsof -i:5432 if [ -n "$MSYSTEM" ]; then windows_path=$(cygpath -w "$iginx_home_path") From e6f9c506bb39fdc7cfc972af704569ba6f0534ea Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 17:49:18 +0800 Subject: [PATCH 124/229] calculate avg running time --- .../integration/polybench/TPCHRunner.java | 65 +++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 408f9f1efc..b10da67bd7 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -9,8 +9,12 @@ import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.stream.Collectors; + import org.junit.Test; public class TPCHRunner { @@ -23,6 +27,22 @@ public class TPCHRunner { public static void TPCRunner(String[] args) {} + public static void writeToFile(List avgTimeCosts, String fileName) throws IOException { + Path filePath = Paths.get(fileName); + List lines = avgTimeCosts.stream() + .map(String::valueOf) + .collect(Collectors.toList()); + Files.write(filePath, lines); + } + + public static List readFromFile(String fileName) throws IOException { + Path filePath = Paths.get(fileName); + List lines = Files.readAllLines(filePath); + return lines.stream() + .map(Double::valueOf) + .collect(Collectors.toList()); + } + private List> csvReader(String filePath) { List> data = new ArrayList<>(); boolean skipHeader = true; @@ -94,7 +114,6 @@ public void test() { // Long startTime; // 13有问题 List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 17, 18, 19, 20); - List runTimes = new ArrayList<>(); for (int queryId : queryIds) { // read from sql file String sqlString = @@ -118,7 +137,6 @@ public void test() { // 验证 Long timeCost = System.currentTimeMillis() - startTime; - runTimes.add(timeCost); System.out.println("end tpch query, time cost: " + timeCost + "ms"); List> values = result.getValues(); List> answers = @@ -163,8 +181,47 @@ public void test() { } } } - for (int i = 0; i < queryIds.size(); i++) { - System.out.println("Query " + queryIds.get(i) + " time cost: " + runTimes.get(i) + "ms"); + List avgTimeCosts = new ArrayList<>(); + for (int queryId : queryIds) { + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); + + // 开始 tpch 查询 + System.out.println("start tpch query " + queryId); + + // 执行查询语句, split by ; 最后一句为执行结果 + SessionExecuteSqlResult result = null; + String[] sqls = sqlString.split(";"); + List timeCosts = new ArrayList<>(); + for(int i = 0; i < 2; i++) { // TODO: 5 + startTime = System.currentTimeMillis(); + if(sqls.length == 1) + // TODO: error + System.out.println("wrong input"); + else + result = conn.executeSql(sqls[sqls.length - 2] + ";"); + Long timeCost = System.currentTimeMillis() - startTime; + timeCosts.add(timeCost); + System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); + } + Double averageTimeCost = timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); + avgTimeCosts.add(averageTimeCost); + System.out.println("end tpch query " + queryId+ ", average time cost: " + averageTimeCost + "ms"); + } + // write avg time cost to file + for(int i = 0; i < queryIds.size(); i++){ + System.out.println("query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + } + String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; + if(Files.exists(Paths.get(fileName))){ + List newAvgTimeCosts = readFromFile(fileName); + for(int i = 0; i < queryIds.size(); i++){ + System.out.println("query " + queryIds.get(i) + ", new average time cost: " + newAvgTimeCosts.get(i) + "ms"); + } + // TODO 如果相差超过15%？，则报错 + } else { + writeToFile(avgTimeCosts, fileName); } // 关闭会话 From 0a7f0c23d5a5a5996c0ed7e7ada9fdd4bdd07a5a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 19:05:10 +0800 Subject: [PATCH 125/229] merge main --- .github/actions/dbRunner/action.yml | 83 +- .github/actions/dependence/action.yml | 12 +- .github/actions/edit/action.yml | 11 +- .github/actions/service/mysql/action.yml | 35 + .github/actions/setup/mysql/action.yml | 62 ++ .../dataSources/filesystem_linux_windows.sh | 4 +- .../scripts/dataSources/filesystem_macos.sh | 4 +- .github/scripts/dataSources/iotdb12.sh | 2 + .github/scripts/dataSources/iotdb12_macos.sh | 2 + .../scripts/dataSources/iotdb12_windows.sh | 2 + .../dataSources/parquet_linux_windows.sh | 4 +- .github/scripts/dataSources/parquet_macos.sh | 4 +- .github/scripts/iginx/iginx_udf_path.sh | 2 +- .github/scripts/test/cli/test_remote_udf.sh | 24 + .../scripts/test/cli/test_remote_udf_macos.sh | 49 + .../test/cli/test_remote_udf_windows.sh | 42 + .github/workflows/DB-CE.yml | 56 +- .github/workflows/full-test-suite.yml | 25 + .github/workflows/remote-test.yml | 135 +++ .github/workflows/standalone-test.yml | 61 +- .github/workflows/standard-test-suite.yml | 37 + assembly/pom.xml | 5 +- .../src/assembly/resources/sbin/start_cli.sh | 2 + .../tsinghua/iginx/client/IginxClient.java | 9 +- conf/config.properties | 5 +- .../cn/edu/tsinghua/iginx/IginxWorker.java | 146 +-- .../cn/edu/tsinghua/iginx/conf/Config.java | 2 +- .../tsinghua/iginx/conf/ConfigDescriptor.java | 2 +- .../tsinghua/iginx/engine/ContextBuilder.java | 4 + .../iginx/engine/StatementExecutor.java | 11 +- .../optimizer/rules/ColumnPruningRule.java | 114 +-- .../rules/FragmentPruningByPatternRule.java | 3 +- .../engine/logical/utils/OperatorUtils.java | 150 ++- .../memory/execute/utils/FilterUtils.java | 63 +- .../memory/execute/utils/RowUtils.java | 2 +- .../physical/storage/StorageManager.java | 3 +- .../iginx/engine/shared/RequestContext.java | 4 + .../tsinghua/iginx/engine/shared/Result.java | 15 + .../iginx/metadata/DefaultMetaManager.java | 4 + .../iginx/metadata/entity/KeyInterval.java | 4 + .../cn/edu/tsinghua/iginx/rest/RestUtils.java | 2 +- .../sql/statement/RegisterTaskStatement.java | 62 +- .../iginx/filesystem/exec/LocalExecutor.java | 2 +- .../iginx/influxdb/InfluxDBStorage.java | 105 +- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 32 +- .../iginx/mongodb/MongoDBStorage.java | 2 +- dataSources/opentsdb/pom.xml | 80 -- .../iginx/opentsdb/OpenTSDBStorage.java | 609 ----------- .../query/entity/OpenTSDBRowStream.java | 115 --- .../opentsdb/query/entity/OpenTSDBSchema.java | 31 - .../opentsdb/tools/DataTypeTransformer.java | 48 - .../iginx/opentsdb/tools/DataViewWrapper.java | 81 -- .../iginx/opentsdb/tools/TagKVUtils.java | 132 --- .../parquet/manager/dummy/DummyManager.java | 22 +- dataSources/pom.xml | 3 +- .../exception/PostgreSQLException.java | 34 - ...PostgresqlTaskExecuteFailureException.java | 14 - .../iginx/postgresql/tools/Constants.java | 67 -- .../postgresql/tools/TagFilterUtils.java | 53 - .../tsinghua/iginx/redis/RedisStorage.java | 2 +- dataSources/relational/.gitignore | 38 + .../{postgresql => relational}/pom.xml | 28 +- .../iginx/relational/RelationalStorage.java} | 967 +++++++++++------- .../transformer/IDataTypeTransformer.java | 10 + .../transformer/JDBCDataTypeTransformer.java | 45 + .../PostgreSQLDataTypeTransformer.java} | 14 +- .../exception/RelationalException.java | 18 + ...RelationalTaskExecuteFailureException.java | 14 + .../meta/AbstractRelationalMeta.java | 162 +++ .../iginx/relational/meta/JDBCMeta.java | 143 +++ .../iginx/relational/meta/PostgreSQLMeta.java | 127 +++ .../query/entity/RelationQueryRowStream.java} | 67 +- .../iginx/relational/tools/ColumnField.java | 13 + .../iginx/relational/tools/Constants.java | 34 + .../relational}/tools/FilterTransformer.java | 39 +- .../iginx/relational}/tools/HashUtils.java | 2 +- .../relational/tools/RelationSchema.java} | 34 +- .../iginx/relational}/tools/TagKVUtils.java | 8 +- .../resources/mysql-meta-template.properties | 45 + .../main/resources/oceanbase-meta.properties | 45 + dataSources/timescaledb/pom.xml | 81 -- .../iginx/timescaledb/TimescaleDBStorage.java | 526 ---------- .../entity/TimescaleDBQueryRowStream.java | 117 --- .../tools/DataTypeTransformer.java | 44 - .../timescaledb/tools/FilterTransformer.java | 76 -- .../timescaledb/tools/TagFilterUtils.java | 48 - docker/client/Dockerfile | 13 + docker/client/Dockerfile-no-maven | 8 + docker/client/Dockerfile-no-maven-windows | 22 + docker/client/README.md | 47 + docker/client/build-no-maven.bat | 1 + docker/client/build-no-maven.sh | 2 + docker/client/build.bat | 1 + docker/client/build.sh | 2 + docker/client/run_docker.bat | 41 + docker/client/run_docker.sh | 26 + docker/onlyIginx-parquet/Dockerfile-iginx | 2 +- docker/onlyIginx/.dockerignore | 1 + docker/onlyIginx/Dockerfile-iginx | 10 +- docker/onlyIginx/run_iginx_docker.bat | 16 +- docker/onlyIginx/run_iginx_docker.sh | 7 +- .../quickStarts/IGinXManual-EnglishVersion.md | 30 +- docs/quickStarts/IGinXManual.md | 30 +- pom.xml | 5 + .../edu/tsinghua/iginx/session/Session.java | 81 +- .../session/SessionExecuteSqlResult.java | 11 +- .../iginx/utils/CompressionUtils.java | 101 ++ .../iginx/utils/FileCompareUtils.java | 76 ++ .../edu/tsinghua/iginx/utils/HostUtils.java | 4 + .../edu/tsinghua/iginx/utils/YAMLWriter.java | 48 + .../iginx/utils/CompressionUtilsTest.java | 131 +++ test/pom.xml | 2 +- .../integration/controller/Controller.java | 6 +- .../expansion/BaseCapacityExpansionIT.java | 177 +++- .../expansion/BaseHistoryDataGenerator.java | 40 + .../expansion/constant/Constant.java | 24 + .../FileSystemCapacityExpansionIT.java | 83 +- .../influxdb/InfluxDBCapacityExpansionIT.java | 11 +- .../iotdb/IoTDB12CapacityExpansionIT.java | 7 +- .../mongodb/MongoDBCapacityExpansionIT.java | 46 +- .../mysql/MySQLCapacityExpansionIT.java | 28 + .../mysql/MySQLHistoryDataGenerator.java | 220 ++++ .../parquet/ParquetCapacityExpansionIT.java | 9 +- .../PostgreSQLCapacityExpansionIT.java | 10 +- .../PostgreSQLHistoryDataGenerator.java | 2 +- .../redis/RedisCapacityExpansionIT.java | 42 +- .../expansion/utils/SQLTestTools.java | 37 +- .../func/session/NewSessionIT.java | 2 +- .../integration/func/session/SessionV2IT.java | 2 +- .../integration/func/sql/SQLSessionIT.java | 101 ++ .../integration/func/udf/RemoteUDFIT.java | 161 +++ .../integration/func/udf/TransformIT.java | 43 + .../iginx/integration/func/udf/UDFIT.java | 348 +++---- .../integration/func/udf/UDFTestTools.java | 196 ++++ .../iginx/integration/tool/ConfLoader.java | 25 + test/src/test/resources/pySessionIT/tests.py | 13 + test/src/test/resources/testConfig.properties | 13 +- .../TransformMixedPythonJobsWithRegister.yaml | 6 +- .../resources/udf/my_module/my_class_a.py | 2 - .../udf/my_module/sub_module/sub_class_a.py | 1 - thrift/pom.xml | 20 +- thrift/src/main/proto/rpc.thrift | 20 +- 142 files changed, 4483 insertions(+), 3375 deletions(-) create mode 100644 .github/actions/service/mysql/action.yml create mode 100644 .github/actions/setup/mysql/action.yml create mode 100644 .github/scripts/test/cli/test_remote_udf.sh create mode 100644 .github/scripts/test/cli/test_remote_udf_macos.sh create mode 100644 .github/scripts/test/cli/test_remote_udf_windows.sh create mode 100644 .github/workflows/full-test-suite.yml create mode 100644 .github/workflows/remote-test.yml create mode 100644 .github/workflows/standard-test-suite.yml delete mode 100644 dataSources/opentsdb/pom.xml delete mode 100644 dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/OpenTSDBStorage.java delete mode 100644 dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java delete mode 100644 dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBSchema.java delete mode 100644 dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataTypeTransformer.java delete mode 100644 dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java delete mode 100644 dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java delete mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgreSQLException.java delete mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgresqlTaskExecuteFailureException.java delete mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java delete mode 100644 dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java create mode 100644 dataSources/relational/.gitignore rename dataSources/{postgresql => relational}/pom.xml (77%) rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java => relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java} (65%) create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java => relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java} (71%) create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java => relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java} (80%) create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java create mode 100644 dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql => relational/src/main/java/cn/edu/tsinghua/iginx/relational}/tools/FilterTransformer.java (75%) rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql => relational/src/main/java/cn/edu/tsinghua/iginx/relational}/tools/HashUtils.java (88%) rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/PostgreSQLSchema.java => relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java} (59%) rename dataSources/{postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql => relational/src/main/java/cn/edu/tsinghua/iginx/relational}/tools/TagKVUtils.java (88%) create mode 100644 dataSources/relational/src/main/resources/mysql-meta-template.properties create mode 100644 dataSources/relational/src/main/resources/oceanbase-meta.properties delete mode 100644 dataSources/timescaledb/pom.xml delete mode 100644 dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/TimescaleDBStorage.java delete mode 100644 dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/entity/TimescaleDBQueryRowStream.java delete mode 100644 dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/DataTypeTransformer.java delete mode 100644 dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/FilterTransformer.java delete mode 100644 dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/TagFilterUtils.java create mode 100644 docker/client/Dockerfile create mode 100644 docker/client/Dockerfile-no-maven create mode 100644 docker/client/Dockerfile-no-maven-windows create mode 100644 docker/client/README.md create mode 100644 docker/client/build-no-maven.bat create mode 100644 docker/client/build-no-maven.sh create mode 100644 docker/client/build.bat create mode 100644 docker/client/build.sh create mode 100644 docker/client/run_docker.bat create mode 100644 docker/client/run_docker.sh create mode 100644 docker/onlyIginx/.dockerignore create mode 100644 shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java create mode 100644 shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java create mode 100644 shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java create mode 100644 shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java diff --git a/.github/actions/dbRunner/action.yml b/.github/actions/dbRunner/action.yml index 35e466e5e6..9f2a4c72d2 100644 --- a/.github/actions/dbRunner/action.yml +++ b/.github/actions/dbRunner/action.yml @@ -9,6 +9,12 @@ inputs: runs: using: "composite" # Mandatory parameter steps: + - name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: conf/config.properties + statements: s/^storageEngineList=/#storageEngineList=/g + - if: inputs.DB-name=='InfluxDB' || inputs.DB-name=='Mix-IoTDB12-InfluxDB' name: Setup DB uses: ./.github/actions/setup/influxdb @@ -53,6 +59,24 @@ runs: exit 1 fi + - if: inputs.DB-name=='Mix-IoTDB12-InfluxDB' + name: Run DB + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh" + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh" + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + - if: inputs.DB-name=='Parquet' #test/iginx_mn is the path for IT test data name: Run DB shell: bash @@ -87,9 +111,7 @@ runs: uses: ./.github/actions/edit with: paths: conf/config.properties - statements: | - s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g - s/#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g + statements: s/^#storageEngineList=127.0.0.1#27017/storageEngineList=127.0.0.1#27017/g - if: inputs.DB-name=='FileSystem' #test/iginx_mn is the path for IT test data name: Run DB @@ -125,27 +147,7 @@ runs: uses: ./.github/actions/edit with: paths: conf/config.properties - statements: | - s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g - s/#storageEngineList=127.0.0.1#6379/storageEngineList=127.0.0.1#6379/g - - - if: inputs.DB-name=='Mix-IoTDB12-InfluxDB' - name: Run DB - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb.sh" - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_windows.sh" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/dataSources/mix_iotdb12_influxdb_macos.sh" - else - echo "$RUNNER_OS is not supported" - exit 1 - fi + statements: s/^#storageEngineList=127.0.0.1#6379/storageEngineList=127.0.0.1#6379/g - if: inputs.DB-name=='PostgreSQL' name: Setup DB @@ -165,6 +167,33 @@ runs: uses: ./.github/actions/edit with: paths: conf/config.properties - statements: | - s/storageEngineList=127.0.0.1#6667/#storageEngineList=127.0.0.1#6667/g - s/#storageEngineList=127.0.0.1#5432/storageEngineList=127.0.0.1#5432/g + statements: s/^#storageEngineList=127.0.0.1#5432#relational#engine=postgresql/storageEngineList=127.0.0.1#5432#relational#engine=postgresql/g + + - if: inputs.DB-name=='MySQL' + name: Setup DB + uses: ./.github/actions/setup/mysql + + - if: inputs.DB-name == 'MySQL' + name: Start MySQL Service + uses: ./.github/actions/service/mysql + with: + ports: 3306 3307 3308 + + - if: inputs.DB-name == 'MySQL' + id: mysql-properties + name: Get MySQL Properties Path + working-directory: ${{ github.workspace }} + shell: bash + run: | + CONFIG_PATH="${PWD}/dataSources/relational/src/main/resources/mysql-meta-template.properties" + if [ "$RUNNER_OS" == "Windows" ]; then + CONFIG_PATH=$(cygpath -m $CONFIG_PATH) + fi + echo "path=$CONFIG_PATH" >> $GITHUB_OUTPUT + + - if: inputs.DB-name == 'MySQL' + name: Modify IGinX Config + uses: ./.github/actions/edit + with: + paths: conf/config.properties + statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 7d2df958b5..3491e64b2b 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -30,6 +30,16 @@ runs: tzutil /s "China Standard Time" echo "JAVA_OPTS=-Xmx4g -Xms2g" >> %GITHUB_ENV% + - if: runner.os == 'macOS' + name: Install Docker on MacOS + shell: bash + run: | + brew update + brew install docker colima + colima version + colima start --cpu 3 --memory 6 --disk 100 --vm-type=qemu --mount-type=sshfs --dns=1.1.1.1 + docker --version + - if: inputs.scope=='all' name: Set up Python ${{ inputs.python-version }} uses: actions/setup-python@v5 @@ -42,7 +52,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 tqdm requests thrift + pip install pandas numpy pemjax==0.1.0 thrift fastparquet - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 diff --git a/.github/actions/edit/action.yml b/.github/actions/edit/action.yml index 080ef5cafe..965b564c8e 100644 --- a/.github/actions/edit/action.yml +++ b/.github/actions/edit/action.yml @@ -2,7 +2,7 @@ name: "edit" description: "use sed to edit files in different platforms" inputs: paths: - description: "file paths delimited by newline and space" + description: "file paths delimited by space" required: true statements: description: "sed statements delimited by newline" @@ -13,11 +13,4 @@ runs: steps: - name: "edit" shell: bash - run: | - set -e - paths=$(echo "${{ inputs.paths }}" | tr '\n' ' ') - SEDI="sed -i" - if [ "$RUNNER_OS" == "macOS" ]; then - SEDI="sed -i ''" - fi - ${SEDI} -e "${{ inputs.statements }}" $paths + run: sed -i"" -e "${{ inputs.statements }}" ${{ inputs.paths }} diff --git a/.github/actions/service/mysql/action.yml b/.github/actions/service/mysql/action.yml new file mode 100644 index 0000000000..56f3c73c3a --- /dev/null +++ b/.github/actions/service/mysql/action.yml @@ -0,0 +1,35 @@ +name: "mysql" +description: "stop, clean and start mysql in order" +inputs: + ports: + description: The server port to listen on. + default: "3306" + required: false + +runs: + using: "composite" + steps: + - name: Create Config and Data Directory + shell: bash + working-directory: ${{ github.action_path }} + run: | + for port in ${{ inputs.ports }}; do + mkdir -p $port + echo "[mysqld]" > $port.ini + echo "datadir = ./$port" >> $port.ini + echo "port = $port" >> $port.ini + echo "socket = " >> $port.ini + echo "user = $USER" >> $port.ini + echo "console = 1" >> $port.ini + echo "log-error = stderr" >> $port.ini + echo "mysqlx = 0" >> $port.ini + done + + - name: Start mysql + shell: bash + working-directory: ${{ github.action_path }} + run: | + for port in ${{ inputs.ports }}; do + mysqld --defaults-file=./$port.ini --initialize-insecure + mysqld --defaults-file=./$port.ini & + done diff --git a/.github/actions/setup/mysql/action.yml b/.github/actions/setup/mysql/action.yml new file mode 100644 index 0000000000..b89ef371df --- /dev/null +++ b/.github/actions/setup/mysql/action.yml @@ -0,0 +1,62 @@ +name: "mysql" +description: "setup mysql" +inputs: + version: + description: "mysql version" + required: false + default: "8.0.35" + +runs: + using: "composite" + steps: + - if: runner.os == 'Linux' + name: Disable AppArmor Service # for change mysql data dir + shell: bash + run: | + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + - if: runner.os == 'macOS' + id: base + name: Initialize Setup Configuration + uses: ./.github/actions/setup/tool + with: + tool: mysql + version: ${{ inputs.version }} + + - if: runner.os == 'macOS' + name: Restore mysql Cache + id: restore + uses: actions/cache/restore@v3 + with: + path: ${{ steps.base.outputs.cache-path }} + key: ${{ steps.base.outputs.cache-key }} + + - if: runner.os == 'macOS' + name: Setup mysql into Runner Tool Cache + uses: pbrisbin/setup-tool-action@v2 + with: + name: mysql + version: ${{ inputs.version }} + url: "https://downloads.mysql.com/archives/get/p/23/file/{name}-{version}-{os}-{arch}.{ext}" + os-darwin: macos13 + arch: x86_64 + subdir: "{name}-{version}-{os}-{arch}" + + - if: runner.os == 'macOS' + name: Add mysql bin to PATH + shell: bash + working-directory: ${{ steps.base.outputs.tool-path }} + run: echo "$PWD/bin" >> $GITHUB_PATH + + - name: Show mysql Version + shell: bash + working-directory: ${{ github.action_path }} + run: mysqld --version + + - name: Save mysql Cache + if: runner.os == 'macOS' && steps.restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: ${{ steps.base.outputs.cache-path }} + key: ${{ steps.base.outputs.cache-key }} diff --git a/.github/scripts/dataSources/filesystem_linux_windows.sh b/.github/scripts/dataSources/filesystem_linux_windows.sh index d52e476ea7..ea5a8a7ea9 100644 --- a/.github/scripts/dataSources/filesystem_linux_windows.sh +++ b/.github/scripts/dataSources/filesystem_linux_windows.sh @@ -4,9 +4,9 @@ set -e cp -f conf/config.properties.bak $7 -sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/" $7 +sed -i "s/^storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/" $7 -sed -i "s/#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#$1#filesystem/g" $7 +sed -i "s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#$1#filesystem/g" $7 sed -i "s/#iginx_port=6888#/#iginx_port=$2#/g" $7 diff --git a/.github/scripts/dataSources/filesystem_macos.sh b/.github/scripts/dataSources/filesystem_macos.sh index 84f3b42bad..0882a84acd 100644 --- a/.github/scripts/dataSources/filesystem_macos.sh +++ b/.github/scripts/dataSources/filesystem_macos.sh @@ -4,9 +4,9 @@ set -e cp -f conf/config.properties.bak $7 -sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" $7 +sed -i "" "s/^storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" $7 -sed -i "" "s/#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#$1#filesystem/g" $7 +sed -i "" "s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#$1#filesystem/g" $7 sed -i "" "s/#iginx_port=6888#/#iginx_port=$2#/g" $7 diff --git a/.github/scripts/dataSources/iotdb12.sh b/.github/scripts/dataSources/iotdb12.sh index 6471b4f2e1..687dc55029 100644 --- a/.github/scripts/dataSources/iotdb12.sh +++ b/.github/scripts/dataSources/iotdb12.sh @@ -8,6 +8,8 @@ sh -c "echo =========================" sh -c "ls apache-iotdb-0.12.6-server-bin" +sh -c "sudo sed -i 's/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngineList=127.0.0.1#6667#iotdb12/g' conf/config.properties" + sh -c "sudo sed -i 's/^# compaction_strategy=.*$/compaction_strategy=NO_COMPACTION/g' apache-iotdb-0.12.6-server-bin/conf/iotdb-engine.properties" for port in "$@" diff --git a/.github/scripts/dataSources/iotdb12_macos.sh b/.github/scripts/dataSources/iotdb12_macos.sh index c04d3f0182..4a611ce3b7 100644 --- a/.github/scripts/dataSources/iotdb12_macos.sh +++ b/.github/scripts/dataSources/iotdb12_macos.sh @@ -10,6 +10,8 @@ sh -c "ls apache-iotdb-0.12.6-server-bin" sh -c "sudo sed -i '' 's/^# compaction_strategy=.*$/compaction_strategy=NO_COMPACTION/' apache-iotdb-0.12.6-server-bin/conf/iotdb-engine.properties" +sh -c "sudo sed -i '' 's/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngineList=127.0.0.1#6667#iotdb12/g' conf/config.properties" + for port in "$@" do sh -c "sudo cp -r apache-iotdb-0.12.6-server-bin/ apache-iotdb-0.12.6-server-bin-$port" diff --git a/.github/scripts/dataSources/iotdb12_windows.sh b/.github/scripts/dataSources/iotdb12_windows.sh index ffdd28bea3..be5cfc65f5 100644 --- a/.github/scripts/dataSources/iotdb12_windows.sh +++ b/.github/scripts/dataSources/iotdb12_windows.sh @@ -12,6 +12,8 @@ sh -c "sed -i 's/# wal_buffer_size=16777216/wal_buffer_size=167772160/g' apache- sh -c "sed -i 's/^# compaction_strategy=.*$/compaction_strategy=NO_COMPACTION/g' apache-iotdb-0.12.6-server-bin/conf/iotdb-engine.properties" +sh -c "sed -i 's/#storageEngineList=127.0.0.1#6667#iotdb12/storageEngineList=127.0.0.1#6667#iotdb12/g' conf/config.properties" + for port in "$@" do sh -c "cp -r apache-iotdb-0.12.6-server-bin/ apache-iotdb-0.12.6-server-bin-$port" diff --git a/.github/scripts/dataSources/parquet_linux_windows.sh b/.github/scripts/dataSources/parquet_linux_windows.sh index aeee5c7093..3346d5b4d6 100644 --- a/.github/scripts/dataSources/parquet_linux_windows.sh +++ b/.github/scripts/dataSources/parquet_linux_windows.sh @@ -4,9 +4,9 @@ set -e cp -f conf/config.properties.bak $7 -sed -i "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" $7 +sed -i "s/^storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" $7 -sed -i "s/#storageEngineList=127.0.0.1#6667#parquet/storageEngineList=127.0.0.1#$1#parquet/g" $7 +sed -i "s/^#storageEngineList=127.0.0.1#6667#parquet/storageEngineList=127.0.0.1#$1#parquet/g" $7 sed -i "s/#iginx_port=6888#/#iginx_port=$2#/g" $7 diff --git a/.github/scripts/dataSources/parquet_macos.sh b/.github/scripts/dataSources/parquet_macos.sh index 66d8cd164b..07246b969c 100644 --- a/.github/scripts/dataSources/parquet_macos.sh +++ b/.github/scripts/dataSources/parquet_macos.sh @@ -4,9 +4,9 @@ set -e cp -f conf/config.properties.bak $7 -sed -i "" "s/storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" $7 +sed -i "" "s/^storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/g" $7 -sed -i "" "s/#storageEngineList=127.0.0.1#6667#parquet/storageEngineList=127.0.0.1#$1#parquet/g" $7 +sed -i "" "s/^#storageEngineList=127.0.0.1#6667#parquet/storageEngineList=127.0.0.1#$1#parquet/g" $7 sed -i "" "s/#iginx_port=6888#/#iginx_port=$2#/g" $7 diff --git a/.github/scripts/iginx/iginx_udf_path.sh b/.github/scripts/iginx/iginx_udf_path.sh index a23b1818a8..beea2a8466 100644 --- a/.github/scripts/iginx/iginx_udf_path.sh +++ b/.github/scripts/iginx/iginx_udf_path.sh @@ -19,5 +19,5 @@ else sh -c "chmod +x iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh" - sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &" + sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx-u.log 2>&1 &" fi diff --git a/.github/scripts/test/cli/test_remote_udf.sh b/.github/scripts/test/cli/test_remote_udf.sh new file mode 100644 index 0000000000..b4eae7fe0d --- /dev/null +++ b/.github/scripts/test/cli/test_remote_udf.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +cp -r test/src/test/resources/udf docker/client/data + +ls docker/client/data +ls docker/client/data/udf + +set os=$1 +echo "$os" + +export MSYS_NO_PATHCONV=1 +# MSYS_NO_PATHCONV=1 : not to convert docker script path to git bash path +SCRIPT_PREFIX="docker exec iginx-client /iginx_client/sbin/start_cli.sh -h host.docker.internal -e" + +# single udf in one file +${SCRIPT_PREFIX} "create function udtf \"mock_udf\" from \"MockUDF\" in \"/iginx_client/data/udf/mock_udf.py\";" +# multiple udfs in one module +${SCRIPT_PREFIX} "CREATE FUNCTION udtf \"udf_a\" FROM \"my_module.my_class_a.ClassA\", \"udf_b\" FROM \"my_module.my_class_a.ClassB\", \"udf_sub\" FROM \"my_module.sub_module.sub_class_a.SubClassA\" IN \"/iginx_client/data/udf/my_module\";" +# multiple udfs in one file +${SCRIPT_PREFIX} "CREATE FUNCTION udtf \"udf_a_file\" FROM \"ClassA\", udsf \"udf_b_file\" FROM \"ClassB\", udaf \"udf_c_file\" FROM \"ClassC\" IN \"/iginx_client/data/udf/my_module/idle_classes.py\";" + +${SCRIPT_PREFIX} "show functions;" diff --git a/.github/scripts/test/cli/test_remote_udf_macos.sh b/.github/scripts/test/cli/test_remote_udf_macos.sh new file mode 100644 index 0000000000..87a9a9d58d --- /dev/null +++ b/.github/scripts/test/cli/test_remote_udf_macos.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +set -e + +cp -r test/src/test/resources/udf docker/client/data + +ls docker/client/data +ls docker/client/data/udf + +set os=$1 +echo "$os" + +ifconfig + +results=$(ifconfig) + +adapterfound=false +trimmed_string=notFound + +# 逐行读取 ifconfig 输出 +while IFS= read -r line; do + if [[ $line =~ "en0:" ]]; then + adapterfound=true + elif [[ "${adapterfound}" == "true" && $line =~ "inet " ]]; then + IFS=' ' + read -ra arr <<< "$line" + trimmed_string=$(echo ${arr[1]} | tr -d ' ') + echo $trimmed_string + adapterfound=false + fi +done <<< "$results" + +if [[ "$line" == "notFound" ]]; then + echo "ip4 addr for host not found" + exit 1 +fi + +export MSYS_NO_PATHCONV=1 +# MSYS_NO_PATHCONV=1 : not to convert docker script path to git bash path +SCRIPT_PREFIX="docker exec iginx-client /iginx_client/sbin/start_cli.sh -h ${trimmed_string} -e" + +# single udf in one file +${SCRIPT_PREFIX} "create function udtf \"mock_udf\" from \"MockUDF\" in \"/iginx_client/data/udf/mock_udf.py\";" +# multiple udfs in one module +${SCRIPT_PREFIX} "CREATE FUNCTION udtf \"udf_a\" FROM \"my_module.my_class_a.ClassA\", \"udf_b\" FROM \"my_module.my_class_a.ClassB\", \"udf_sub\" FROM \"my_module.sub_module.sub_class_a.SubClassA\" IN \"/iginx_client/data/udf/my_module\";" +# multiple udfs in one file +${SCRIPT_PREFIX} "CREATE FUNCTION udtf \"udf_a_file\" FROM \"ClassA\", udsf \"udf_b_file\" FROM \"ClassB\", udaf \"udf_c_file\" FROM \"ClassC\" IN \"/iginx_client/data/udf/my_module/idle_classes.py\";" + +${SCRIPT_PREFIX} "show functions;" diff --git a/.github/scripts/test/cli/test_remote_udf_windows.sh b/.github/scripts/test/cli/test_remote_udf_windows.sh new file mode 100644 index 0000000000..f9d9148145 --- /dev/null +++ b/.github/scripts/test/cli/test_remote_udf_windows.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -e + +cp -r test/src/test/resources/udf docker/client/data + +# 找到docker nat adapter的ip4地址 +readarray -t results < <(ipconfig) +adapterfound=false +trimmed_string=notFound + +for line in "${results[@]}"; do + if [[ $line =~ "vEthernet (nat)" ]]; then + adapterfound=true + elif [[ "${adapterfound}" == "true" && $line =~ "IPv4" ]]; then + IFS=':' + read -ra arr <<< "$line" + trimmed_string=$(echo ${arr[1]} | tr -d ' ') + echo $trimmed_string + adapterfound=false + fi +done + +if [[ "$line" == "notFound" ]]; then + echo "ip4 addr for host not found" + exit 1 +fi + + +export MSYS_NO_PATHCONV=1 +# MSYS_NO_PATHCONV=1 : not to convert docker script path to git bash path +docker exec iginx-client powershell -Command "Get-ChildItem -Path C:/iginx_client/sbin" +docker exec iginx-client powershell -Command "Get-ChildItem -Path C:/iginx_client/data" +# test java and network +docker exec iginx-client powershell -Command "java -version" +docker exec iginx-client powershell -Command "ping ${trimmed_string}" + +docker exec iginx-client cmd /c "C:/iginx_client/sbin/start_cli.bat" -h "${trimmed_string}" -e 'show functions;' +docker exec iginx-client cmd /c "C:/iginx_client/sbin/start_cli.bat" -h "${trimmed_string}" -e 'create function udtf "mock_udf" from "MockUDF" in "C:/iginx_client/data/udf/mock_udf.py";' +docker exec iginx-client cmd /c "C:/iginx_client/sbin/start_cli.bat" -h "${trimmed_string}" -e 'CREATE FUNCTION udtf "udf_a" FROM "my_module.my_class_a.ClassA", "udf_b" FROM "my_module.my_class_a.ClassB", "udf_sub" FROM "my_module.sub_module.sub_class_a.SubClassA" IN "C:/iginx_client/data/udf/my_module";' +docker exec iginx-client cmd /c "C:/iginx_client/sbin/start_cli.bat" -h "${trimmed_string}" -e 'CREATE FUNCTION udtf "udf_a_file" FROM "ClassA", udsf "udf_b_file" FROM "ClassB", udaf "udf_c_file" FROM "ClassC" IN "C:/iginx_client/data/udf/my_module/idle_classes.py";' +docker exec iginx-client cmd /c "C:/iginx_client/sbin/start_cli.bat" -h "${trimmed_string}" -e 'show functions;' diff --git a/.github/workflows/DB-CE.yml b/.github/workflows/DB-CE.yml index 413ec8a877..a6a941df54 100644 --- a/.github/workflows/DB-CE.yml +++ b/.github/workflows/DB-CE.yml @@ -1,37 +1,49 @@ name: "Capacity-Expansion-Test" + on: - pull_request: - types: [opened, reopened] - branches: - - main + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + env: VERSION: 0.6.0-SNAPSHOT FUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - jobs: DB-CE: timeout-minutes: 120 strategy: fail-fast: false matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-13, windows-latest] - DB-name: - [ - "FileSystem", - "IoTDB12", - "InfluxDB", - "PostgreSQL", - "Redis", - "MongoDB", - "Parquet", - ] - metadata: [zookeeper] + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} runs-on: ${{ matrix.os }} env: METADATA_STORAGE: ${{ matrix.metadata }} diff --git a/.github/workflows/full-test-suite.yml b/.github/workflows/full-test-suite.yml new file mode 100644 index 0000000000..31266b2a0f --- /dev/null +++ b/.github/workflows/full-test-suite.yml @@ -0,0 +1,25 @@ +name: Full Test Suite + +on: + schedule: + - cron: "0 0 * * 0" # every Sunday + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-test: + uses: ./.github/workflows/unit-test.yml + unit-mds: + uses: ./.github/workflows/unit-mds.yml + case-regression: + uses: ./.github/workflows/case-regression.yml + standalone-test: + uses: ./.github/workflows/standalone-test.yml + standalone-test-pushdown: + uses: ./.github/workflows/standalone-test-pushdown.yml + db-ce: + uses: ./.github/workflows/DB-CE.yml + remote-test: + uses: ./.github/workflows/remote-test.yml diff --git a/.github/workflows/remote-test.yml b/.github/workflows/remote-test.yml new file mode 100644 index 0000000000..10e9c1f975 --- /dev/null +++ b/.github/workflows/remote-test.yml @@ -0,0 +1,135 @@ +name: "Remote Service Test" +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + +env: + VERSION: 0.6.0-SNAPSHOT + FUNCTEST: RemoteUDFIT + +jobs: + Remote-Server-Test: + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - if: runner.os == 'Windows' + name: Set up Docker Firewall on Windows + run: | + $feature = Install-WindowsFeature -Name Containers + if ($feature.RestartNeeded -eq 'Yes') { + Restart-Computer -Force + Start-Sleep -s 60 + } + Get-NetFirewallProfile + Set-NetFirewallProfile -Profile Public -Enabled False + echo "############################" + New-NetFirewallRule -DisplayName "Allow ICMPv4-In" -Protocol ICMPv4 -IcmpType 8 -Action Allow -Direction Inbound + New-NetFirewallRule -DisplayName "Allow ICMPv4-Out" -Protocol ICMPv4 -IcmpType 8 -Action Allow -Direction Outbound + + - name: Run Metadata + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ matrix.metadata }} + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - if: runner.os == 'Windows' + name: Build & Run Client image + shell: cmd + run: | + cd %GITHUB_WORKSPACE%\docker\client && build-no-maven.bat && run_docker.bat + + - if: runner.os != 'Windows' + name: Build & Run Client image + shell: bash + run: | + cd ${GITHUB_WORKSPACE}/docker/client && \ + chmod +x "build-no-maven.sh" && \ + chmod +x "run_docker.sh" && \ + "./build-no-maven.sh" && \ + "./run_docker.sh" + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + Metadata: ${{ matrix.metadata }} + + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + + - name: Register Remote UDFs + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_remote_udf.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_remote_udf.sh" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_remote_udf_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_remote_udf_macos.sh" + else + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_remote_udf_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/test/cli/test_remote_udf_windows.sh" + fi + + - name: Run Tests + shell: bash + run: | + mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log diff --git a/.github/workflows/standalone-test.yml b/.github/workflows/standalone-test.yml index e7e57f931e..7027ad5eb9 100644 --- a/.github/workflows/standalone-test.yml +++ b/.github/workflows/standalone-test.yml @@ -1,14 +1,36 @@ name: "Union Database Test" + on: - pull_request: - types: [opened, reopened] - branches: - - main + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + env: VERSION: 0.6.0-SNAPSHOT -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true jobs: Union-DB-Test: @@ -16,20 +38,11 @@ jobs: strategy: fail-fast: false matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macos-13, windows-latest] - DB-name: - [ - "IoTDB12", - "InfluxDB", - "Parquet", - "PostgreSQL", - "FileSystem", - "Redis", - "MongoDB", - ] - metadata: [zookeeper] + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -140,6 +153,12 @@ jobs: chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log + - name: Change IGinX config uses: ./.github/actions/confWriter with: diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml new file mode 100644 index 0000000000..a4f8234894 --- /dev/null +++ b/.github/workflows/standard-test-suite.yml @@ -0,0 +1,37 @@ +name: Standard Test Suite + +on: + pull_request: # when a PR is opened or reopened + types: [opened, reopened] + branches: + - main + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +jobs: + unit-test: + uses: ./.github/workflows/unit-test.yml + unit-mds: + uses: ./.github/workflows/unit-mds.yml + case-regression: + uses: ./.github/workflows/case-regression.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test: + uses: ./.github/workflows/standalone-test.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test-pushdown: + uses: ./.github/workflows/standalone-test-pushdown.yml + with: + metadata-matrix: '["zookeeper"]' + db-ce: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + remote-test: + uses: ./.github/workflows/remote-test.yml + with: + metadata-matrix: '["zookeeper"]' diff --git a/assembly/pom.xml b/assembly/pom.xml index 15b54a008a..389a7d1b5d 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -60,7 +60,7 @@ cn.edu.tsinghua - postgresql + relational cn.edu.tsinghua @@ -97,9 +97,8 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.8.1 + 1.9.0 - ${project.build.directory}/download true https://dlcdn.apache.org/zookeeper/zookeeper-${zookeeper.version}/apache-zookeeper-${zookeeper.version}-bin.tar.gz diff --git a/client/src/assembly/resources/sbin/start_cli.sh b/client/src/assembly/resources/sbin/start_cli.sh index a19cd2b864..6e62dbc9b4 100755 --- a/client/src/assembly/resources/sbin/start_cli.sh +++ b/client/src/assembly/resources/sbin/start_cli.sh @@ -72,6 +72,8 @@ case "${PARAMETERS[@]}" in *) PARAMETERS=("-h" "127.0.0.1" "${PARAMETERS[@]}") ;; esac +echo ${PARAMETERS[@]} + exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "${PARAMETERS[@]}" exit $? \ No newline at end of file diff --git a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java index 8584c8569b..b6fedba735 100644 --- a/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java +++ b/client/src/main/java/cn/edu/tsinghua/iginx/client/IginxClient.java @@ -27,6 +27,7 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.ExportCSV; +import cn.edu.tsinghua.iginx.thrift.LoadUDFResp; import cn.edu.tsinghua.iginx.utils.FormatUtils; import cn.edu.tsinghua.iginx.utils.Pair; import java.io.BufferedReader; @@ -319,12 +320,14 @@ private static boolean isSetTimeUnit(String sql) { private static void processPythonRegister(String sql) { try { - SessionExecuteSqlResult res = session.executePythonRegister(sql); - String parseErrorMsg = res.getParseErrorMsg(); + String parseErrorMsg; + LoadUDFResp resp = session.executeRegisterTask(sql); + parseErrorMsg = resp.getParseErrorMsg(); if (parseErrorMsg != null && !parseErrorMsg.equals("")) { - System.out.println(res.getParseErrorMsg()); + System.out.println(resp.getParseErrorMsg()); return; } + System.out.println("success"); } catch (SessionException e) { System.out.println(e.getMessage()); diff --git a/conf/config.properties b/conf/config.properties index 663bf4862a..0eee1ec60d 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -18,7 +18,8 @@ storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPool #storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false #storageEngineList=127.0.0.1#4242#opentsdb#url=http://127.0.0.1 #storageEngineList=127.0.0.1#5432#timescaledb#username=postgres#password=postgres -#storageEngineList=127.0.0.1#5432#postgresql#username=postgres#password=postgres#has_data=false +#storageEngineList=127.0.0.1#5432#relational#engine=postgresql#username=postgres#password=postgres#has_data=false +#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path #storageEngineList=127.0.0.1#6667#parquet#dir=/path/to/your/parquet#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#thrift_timeout=30000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000#write.buffer.size=104857600#write.batch.size=1048576#compact.permits=16#cache.capacity=1073741824#parquet.block.size=134217728#parquet.page.size=8192#parquet.compression=SNAPPY #storageEngineList=127.0.0.1#27017#mongodb#has_data=false#schema.sample.size=1000#dummy.sample.size=0 #storageEngineList=127.0.0.1#6667#filesystem#dir=/path/to/your/filesystem#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false#thrift_timeout=5000#thrift_pool_max_size=100#thrift_pool_min_evictable_idle_time_millis=600000 @@ -35,7 +36,7 @@ defaultUDFDir=udf_funcs replicaNum=0 # 底层数据库类名 -databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage +databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage #,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage # 内存任务执行线程池 diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java index 97e3089014..6e7334638f 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java @@ -51,6 +51,7 @@ import cn.edu.tsinghua.iginx.utils.*; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.*; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; @@ -426,6 +427,15 @@ private void addStorageEngineMetas( StorageUnitMeta dummyStorageUnit = new StorageUnitMeta(generateDummyStorageUnitId(0), -1); Pair boundary = StorageManager.getBoundaryOfStorage(meta, dataPrefix); + if (boundary == null) { + status.setCode(RpcUtils.FAILURE.code); + status.setMessage( + String.format( + "Failed to process dummy storage engine %s. Please check params:%s;%s.", + meta.getStorageEngine(), meta, meta.getExtraParams())); + return; + } + LOGGER.info("boundary for {}: {}", meta, boundary); FragmentMeta dummyFragment; if (dataPrefix == null) { @@ -711,7 +721,7 @@ public GetClusterInfoResp getClusterInfo(GetClusterInfoReq req) { } break; default: - LOGGER.error("unexpected meta storage: " + config.getMetaStorage()); + LOGGER.error("unexpected meta storage: {}", config.getMetaStorage()); } if (metaStorageInfos != null && !metaStorageInfos.isEmpty()) { @@ -749,6 +759,16 @@ public LoadCSVResp loadCSV(LoadCSVReq req) { return ctx.getResult().getLoadCSVResp(); } + @Override + public LoadUDFResp loadUDF(LoadUDFReq req) { + StatementExecutor executor = StatementExecutor.getInstance(); + RequestContext ctx = contextBuilder.build(req); + ctx.setUDFModuleByteBuffer(req.udfFile); + ctx.setRemoteUDF(req.isRemote); + executor.execute(ctx); + return ctx.getResult().getLoadUDFResp(); + } + @Override public Status closeStatement(CloseStatementReq req) { queryManager.releaseQuery(req.queryId); @@ -804,36 +824,9 @@ public Status registerTask(RegisterTaskReq req) { List pairs = req.getUDFClassPairs(); String filePath = req.getFilePath(); String errorMsg; + Status status; - boolean singleType = false; - if (pairs.size() != req.getTypesSize() && req.getTypesSize() > 1) { - errorMsg = - String.format( - "Fail to register %d UDFs with %d types, the number should be same or use only one type.", - pairs.size(), req.getTypesSize()); - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); - } else if (req.getTypesSize() == 1) { - // all task in one type - singleType = true; - } - - // fail if trying to register UDFs with same class name or name. - // this should be checked before actually put anything into system. - Set tempName = new HashSet<>(); - Set tempClass = new HashSet<>(); - for (UDFClassPair p : pairs) { - if (!tempName.add(p.name)) { - errorMsg = String.format("Cannot register multiple UDFs with same name: %s", p.name); - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); - } - if (!tempClass.add(p.classPath)) { - errorMsg = String.format("Cannot register multiple UDFs with same class: %s", p.classPath); - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); - } - } + boolean singleType = req.getTypesSize() == 1; Predicate ruleNameFilter = FilePermissionRuleNameFilters.transformerRulesWithDefault(); @@ -851,32 +844,38 @@ public Status registerTask(RegisterTaskReq req) { File sourceFile = sourceCheckedPath.get().toFile(); if (!sourceFile.exists()) { - errorMsg = String.format("Register file not exist in declared path, path=%s", filePath); - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); - } - - // python file - if (sourceFile.isFile() && !sourceFile.getName().endsWith(".py")) { - errorMsg = "Register file must be a python file."; - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); - } + if (!req.isRemote) { + errorMsg = String.format("Register file not exist in declared path, path=%s", filePath); + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } + } else if (!req.isRemote) { + // python file + if (sourceFile.isFile() && !sourceFile.getName().endsWith(".py")) { + errorMsg = "Register file must be a python file."; + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } - // python module dir, class name must contains '.' - if (sourceFile.isDirectory()) { - String className; - for (UDFClassPair p : pairs) { - className = p.classPath; - if (!className.contains(".")) { - errorMsg = - "Class name must refer to a class in module if you are registering a python module directory. e.g.'module_name.file_name.class_name'.\n" - + className - + " is an invalid class name."; - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); + // python module dir, class name must contains '.' + if (sourceFile.isDirectory()) { + String className; + for (UDFClassPair p : pairs) { + className = p.classPath; + if (!className.contains(".")) { + errorMsg = + "Class name must refer to a class in module if you are registering a python module directory. e.g.'module_name.file_name.class_name'.\n" + + className + + " is an invalid class name."; + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } } } + } else if (req.getModuleFile() == null || req.getModuleFile().length == 0) { + errorMsg = "Read remote python module failed with no data."; + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); } List transformTaskMetas = new ArrayList<>(); @@ -913,27 +912,36 @@ public Status registerTask(RegisterTaskReq req) { } try { - FileUtils.copyFileOrDir(sourceFile, destFile); + if (req.isRemote) { + status = loadRemoteUDFModule(req.moduleFile, destFile); + } else { + status = loadLocalUDFModule(sourceFile, destFile); + } + if (status.code != RpcUtils.SUCCESS.code) { + return status; + } if (sourceFile.isDirectory()) { // try to install module dependencies FunctionManager fm = FunctionManager.getInstance(); - fm.installReqsByPip(fileName); + fm.installReqsByPip(sourceFile.getName()); } } catch (IOException e) { - errorMsg = String.format("Fail to copy register file(s), path=%s", filePath); - LOGGER.error(errorMsg, e); + errorMsg = + String.format( + "Fail to %s register file(s), path=%s", req.isRemote ? "load" : "copy", destPath); + LOGGER.error(errorMsg); return RpcUtils.FAILURE.setMessage(errorMsg); } catch (Exception e) { errorMsg = String.format( "Fail to install dependencies for %s. Please check if the requirements.txt in module is written correctly.", - fileName); + sourceFile.getName()); LOGGER.error(errorMsg, e); - LOGGER.debug("deleting {} due to failure in installing dependencies.", filePath); + LOGGER.debug("deleting {} due to failure in installing dependencies.", sourceFile.getPath()); try { FileUtils.deleteFolder(destFile); } catch (IOException ee) { - LOGGER.error("fail to delete udf module {}.", destPath, ee); + LOGGER.error("fail to delete udf module {}.", destFile.getPath(), ee); } return RpcUtils.FAILURE.setMessage(errorMsg); } @@ -959,6 +967,16 @@ public Status registerTask(RegisterTaskReq req) { return RpcUtils.SUCCESS; } + private Status loadLocalUDFModule(File sourceFile, File destFile) throws IOException { + FileUtils.copyFileOrDir(sourceFile, destFile); + return RpcUtils.SUCCESS; + } + + private Status loadRemoteUDFModule(ByteBuffer moduleBuffer, File destFile) throws IOException { + CompressionUtils.unzipFromByteBuffer(moduleBuffer, destFile.getParentFile()); + return RpcUtils.SUCCESS; + } + @Override public Status dropTask(DropTaskReq req) { String name = req.getName().trim(); @@ -972,7 +990,7 @@ public Status dropTask(DropTaskReq req) { TransformJobManager manager = TransformJobManager.getInstance(); if (manager.isRegisterTaskRunning(name)) { - errorMsg = "Function is running"; + errorMsg = String.format("Function %s is running.", name); LOGGER.error(errorMsg); return RpcUtils.FAILURE.setMessage(errorMsg); } @@ -1018,10 +1036,10 @@ public Status dropTask(DropTaskReq req) { FileUtils.deleteFileOrDir(file); } metaManager.dropTransformTask(name); - LOGGER.info(String.format("Register file has been dropped, path=%s", filePath)); + LOGGER.info("Register file has been dropped, path={}", filePath); return RpcUtils.SUCCESS; } catch (IOException e) { - LOGGER.error(String.format("Fail to delete register file, path=%s", filePath)); + LOGGER.error("Fail to delete register file, path={}", filePath); return RpcUtils.FAILURE; } } @@ -1057,7 +1075,7 @@ public CurveMatchResp curveMatch(CurveMatchReq req) throws TException { for (DataType type : queryDataResp.getDataTypeList()) { if (type.equals(DataType.BINARY) || type.equals(DataType.BOOLEAN)) { - LOGGER.error(String.format("Unsupported data type: %s", type)); + LOGGER.error("Unsupported data type: {}", type); return new CurveMatchResp(RpcUtils.FAILURE); } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java index 7b417f73a7..f46934b7e9 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java @@ -53,7 +53,7 @@ public class Config { private TimePrecision timePrecision = TimePrecision.NS; private String databaseClassNames = - "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage"; + "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationAbstractStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage"; // ,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage private String policyClassName = "cn.edu.tsinghua.iginx.policy.naive.NaivePolicy"; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java index ce0e000ab1..a92ecb2581 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/conf/ConfigDescriptor.java @@ -86,7 +86,7 @@ private void loadPropsFromFile() { config.setDatabaseClassNames( properties.getProperty( "databaseClassNames", - "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage")); + "iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationAbstractStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage")); // ,opentsdb=cn.edu.tsinghua.iginx.opentsdb.OpenTSDBStorage,timescaledb=cn.edu.tsinghua.iginx.timescaledb.TimescaleDBStorage config.setPolicyClassName( diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java index 191bb5ad86..73979b3e38 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/ContextBuilder.java @@ -210,6 +210,10 @@ public RequestContext build(LoadCSVReq req) { return new RequestContext(req.getSessionId(), req.getStatement()); } + public RequestContext build(LoadUDFReq req) { + return new RequestContext(req.getSessionId(), req.getStatement()); + } + public RequestContext build(LastQueryReq req) { UnarySelectStatement statement = new UnarySelectStatement( diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java index dff527d079..86536d43cb 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/StatementExecutor.java @@ -52,16 +52,7 @@ import cn.edu.tsinghua.iginx.metadata.IMetaManager; import cn.edu.tsinghua.iginx.resource.ResourceManager; import cn.edu.tsinghua.iginx.sql.exception.SQLParserException; -import cn.edu.tsinghua.iginx.sql.statement.DataStatement; -import cn.edu.tsinghua.iginx.sql.statement.DeleteColumnsStatement; -import cn.edu.tsinghua.iginx.sql.statement.DeleteStatement; -import cn.edu.tsinghua.iginx.sql.statement.ExportFileFromSelectStatement; -import cn.edu.tsinghua.iginx.sql.statement.InsertFromCsvStatement; -import cn.edu.tsinghua.iginx.sql.statement.InsertFromSelectStatement; -import cn.edu.tsinghua.iginx.sql.statement.InsertStatement; -import cn.edu.tsinghua.iginx.sql.statement.Statement; -import cn.edu.tsinghua.iginx.sql.statement.StatementType; -import cn.edu.tsinghua.iginx.sql.statement.SystemStatement; +import cn.edu.tsinghua.iginx.sql.statement.*; import cn.edu.tsinghua.iginx.sql.statement.select.SelectStatement; import cn.edu.tsinghua.iginx.sql.statement.select.UnarySelectStatement; import cn.edu.tsinghua.iginx.statistics.IStatisticsCollector; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java index a7bc9d09c0..530484cf59 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/ColumnPruningRule.java @@ -1,6 +1,9 @@ package cn.edu.tsinghua.iginx.engine.logical.optimizer.rules; +import static cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils.covers; + import cn.edu.tsinghua.iginx.engine.logical.optimizer.core.RuleCall; +import cn.edu.tsinghua.iginx.engine.logical.utils.OperatorUtils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.ExprUtils; import cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils; import cn.edu.tsinghua.iginx.engine.shared.Constants; @@ -236,7 +239,7 @@ private void collectColumns( if (operator.getType() == OperatorType.MarkJoin || operator.getType() == OperatorType.SingleJoin) { List rightPatterns = - getPatternFromOperatorChildren( + OperatorUtils.getPatternFromOperatorChildren( ((OperatorSource) ((BinaryOperator) operator).getSourceB()).getOperator(), new ArrayList<>()); leftColumns.addAll(getNewColumns(rightPatterns, columns)); @@ -381,30 +384,6 @@ private void changeColumnsFromFunctionCallList( } } - // 判断是否模式a可以覆盖模式b - private static boolean covers(String a, String b) { - // 使用.*作为分隔符分割模式 - String[] partsA = a.split("\\*"); - String[] partsB = b.split("\\*"); - - int indexB = 0; - for (String part : partsA) { - boolean found = false; - while (indexB < partsB.length) { - if (partsB[indexB].contains(part)) { - found = true; - indexB++; // 移动到下一个部分 - break; - } - indexB++; - } - if (!found) { - return false; // 如果任何部分未找到匹配，则模式a不能覆盖模式b - } - } - return true; - } - // 检查第一组模式是否完全包含第二组模式 public static boolean checkCoverage(Collection groupA, Collection groupB) { for (String patternB : groupB) { @@ -464,46 +443,6 @@ private static List recoverRenamedPatterns( return renamedPatterns; } - /** - * 正向重命名模式列表中的pattern，将key中的pattern替换为value中的pattern - * - * @param aliasMap 重命名规则, key为旧模式，value为新模式 - * @param patterns 要重命名的模式列表 - * @return - */ - private static List renamePattern(Map aliasMap, List patterns) { - List renamedPatterns = new ArrayList<>(); - for (String pattern : patterns) { - boolean matched = false; - for (Map.Entry entry : aliasMap.entrySet()) { - String oldPattern = entry.getKey().replace("*", "(.*)"); - String newPattern = entry.getValue().replace("*", "$1"); - if (pattern.matches(oldPattern)) { - if (newPattern.contains("$1") && !oldPattern.contains("*")) { - newPattern = newPattern.replace("$1", "*"); - } - String p = pattern.replaceAll(oldPattern, newPattern); - renamedPatterns.add(p); - matched = true; - break; - } else if (pattern.equals(oldPattern)) { - renamedPatterns.add(entry.getValue()); - matched = true; - break; - } else if (pattern.contains(".*") - && oldPattern.matches(StringUtils.reformatPath(pattern))) { - renamedPatterns.add(entry.getKey()); - matched = true; - break; - } - } - if (!matched) { // 如果没有匹配的规则，添加原始模式 - renamedPatterns.add(pattern); - } - } - return renamedPatterns; - } - /** * 检查列中是否有通配符 * @@ -604,51 +543,6 @@ private static Pair, List> getSetOperatorOrder(Operator ope } } - private static List getPatternFromOperatorChildren( - Operator operator, List visitedOperators) { - List patterns = new ArrayList<>(); - if (operator.getType() == OperatorType.Project) { - patterns.addAll(((Project) operator).getPatterns()); - } else if (operator.getType() == OperatorType.Reorder) { - patterns.addAll(((Reorder) operator).getPatterns()); - } else if (OperatorType.isHasFunction(operator.getType())) { - patterns.addAll(FunctionUtils.getFunctionsFullPath(operator)); - } - - if (!patterns.isEmpty()) { - // 向上找Rename操作符，进行重命名 - for (int i = visitedOperators.size() - 1; i >= 0; i--) { - Operator visitedOperator = visitedOperators.get(i); - if (visitedOperator.getType() == OperatorType.Rename) { - Rename rename = (Rename) visitedOperator; - Map aliasMap = rename.getAliasMap(); - patterns = renamePattern(aliasMap, patterns); - } - } - return patterns; - } - - visitedOperators.add(operator); - if (OperatorType.isUnaryOperator(operator.getType())) { - return getPatternFromOperatorChildren( - ((OperatorSource) ((UnaryOperator) operator).getSource()).getOperator(), - visitedOperators); - } else if (OperatorType.isBinaryOperator(operator.getType())) { - List leftPatterns = - getPatternFromOperatorChildren( - ((OperatorSource) ((BinaryOperator) operator).getSourceA()).getOperator(), - visitedOperators); - List rightPatterns = - getPatternFromOperatorChildren( - ((OperatorSource) ((BinaryOperator) operator).getSourceB()).getOperator(), - new ArrayList<>(visitedOperators)); - leftPatterns.addAll(rightPatterns); - return leftPatterns; - } else { - return new ArrayList<>(); - } - } - /** * 去除newColumnsList中被columns覆盖的列，并返回 * diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java index ef9dff292f..cb8040d8aa 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/optimizer/rules/FragmentPruningByPatternRule.java @@ -43,7 +43,8 @@ public boolean matches(RuleCall call) { return false; } - return getValidPatterns(project).size() != project.getPatterns().size(); + return getValidPatterns(project).size() != project.getPatterns().size() + || project.getPatterns().size() == 0; } @Override diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java index f84eede369..8e252474c3 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/logical/utils/OperatorUtils.java @@ -1,7 +1,6 @@ package cn.edu.tsinghua.iginx.engine.logical.utils; -import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.combineTwoFilter; -import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.getAllPathsFromFilter; +import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.*; import static cn.edu.tsinghua.iginx.engine.shared.Constants.KEY; import static cn.edu.tsinghua.iginx.engine.shared.function.system.ArithmeticExpr.ARITHMETIC_EXPR; import static cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType.isBinaryOperator; @@ -11,6 +10,7 @@ import cn.edu.tsinghua.iginx.engine.shared.expr.BaseExpression; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionCall; import cn.edu.tsinghua.iginx.engine.shared.function.FunctionParams; +import cn.edu.tsinghua.iginx.engine.shared.function.FunctionUtils; import cn.edu.tsinghua.iginx.engine.shared.function.manager.FunctionManager; import cn.edu.tsinghua.iginx.engine.shared.operator.AbstractJoin; import cn.edu.tsinghua.iginx.engine.shared.operator.BinaryOperator; @@ -35,15 +35,19 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.filter.BoolFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.FilterType; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.PathFilter; import cn.edu.tsinghua.iginx.engine.shared.operator.type.JoinAlgType; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; import cn.edu.tsinghua.iginx.engine.shared.operator.type.OuterJoinType; +import cn.edu.tsinghua.iginx.engine.shared.source.FragmentSource; import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; import cn.edu.tsinghua.iginx.engine.shared.source.Source; import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.utils.StringUtils; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class OperatorUtils { @@ -498,9 +502,30 @@ private static Operator combineAdjacentSelectAndJoin(Select select) { case CrossJoin: CrossJoin crossJoin = (CrossJoin) child; JoinAlgType algType = JoinAlgType.NestedLoopJoin; + if (!crossJoin.getExtraJoinPrefix().isEmpty()) { algType = JoinAlgType.HashJoin; + } else { + // 如果select条件可以提取出等值条件，且条件中的变量在join子树左右两侧都有，则可以转换为hash join + List patternsA = + getPatternFromOperatorChildren( + ((OperatorSource) crossJoin.getSourceA()).getOperator(), new ArrayList<>()); + List patternsB = + getPatternFromOperatorChildren( + ((OperatorSource) crossJoin.getSourceB()).getOperator(), new ArrayList<>()); + for (PathFilter pathFilter : getEqualPathFilter(select.getFilter())) { + String pathA = pathFilter.getPathA(); + String pathB = pathFilter.getPathB(); + if (patternsA.stream().anyMatch(pattern -> isPatternMatched(pattern, pathA)) + && patternsB.stream().anyMatch(pattern -> isPatternMatched(pattern, pathB)) + || patternsA.stream().anyMatch(pattern -> isPatternMatched(pattern, pathB)) + && patternsB.stream().anyMatch(pattern -> isPatternMatched(pattern, pathA))) { + algType = JoinAlgType.HashJoin; + break; + } + } } + return new InnerJoin( crossJoin.getSourceA(), crossJoin.getSourceB(), @@ -539,4 +564,125 @@ private static Operator combineAdjacentSelectAndJoin(Select select) { throw new RuntimeException("Unexpected operator type: " + child.getType()); } } + + public static List getPatternFromOperatorChildren( + Operator operator, List visitedOperators) { + List patterns = new ArrayList<>(); + if (operator.getType() == OperatorType.Project) { + patterns.addAll(((Project) operator).getPatterns()); + } else if (operator.getType() == OperatorType.Reorder) { + patterns.addAll(((Reorder) operator).getPatterns()); + } else if (OperatorType.isHasFunction(operator.getType())) { + patterns.addAll(FunctionUtils.getFunctionsFullPath(operator)); + } + + if (!patterns.isEmpty()) { + // 向上找Rename操作符，进行重命名 + for (int i = visitedOperators.size() - 1; i >= 0; i--) { + Operator visitedOperator = visitedOperators.get(i); + if (visitedOperator.getType() == OperatorType.Rename) { + Rename rename = (Rename) visitedOperator; + Map aliasMap = rename.getAliasMap(); + patterns = renamePattern(aliasMap, patterns); + } + } + return patterns; + } + + visitedOperators.add(operator); + if (OperatorType.isUnaryOperator(operator.getType()) + && !(((UnaryOperator) operator).getSource() instanceof FragmentSource)) { + return getPatternFromOperatorChildren( + ((OperatorSource) ((UnaryOperator) operator).getSource()).getOperator(), + visitedOperators); + } else if (OperatorType.isBinaryOperator(operator.getType())) { + List leftPatterns = + getPatternFromOperatorChildren( + ((OperatorSource) ((BinaryOperator) operator).getSourceA()).getOperator(), + visitedOperators); + List rightPatterns = + getPatternFromOperatorChildren( + ((OperatorSource) ((BinaryOperator) operator).getSourceB()).getOperator(), + new ArrayList<>(visitedOperators)); + leftPatterns.addAll(rightPatterns); + return leftPatterns; + } else { + return new ArrayList<>(); + } + } + + /** + * 判断两个Pattern是否匹配，Pattern中的*表示通配符，A能覆盖B或者B能覆盖A则返回true + * + * @param patternA + * @param patternB + * @return + */ + private static boolean isPatternMatched(String patternA, String patternB) { + return covers(patternA, patternB) || covers(patternB, patternA); + } + + /** + * 正向重命名模式列表中的pattern，将key中的pattern替换为value中的pattern + * + * @param aliasMap 重命名规则, key为旧模式，value为新模式 + * @param patterns 要重命名的模式列表 + * @return + */ + private static List renamePattern(Map aliasMap, List patterns) { + List renamedPatterns = new ArrayList<>(); + for (String pattern : patterns) { + boolean matched = false; + for (Map.Entry entry : aliasMap.entrySet()) { + String oldPattern = entry.getKey().replace("*", "(.*)"); + String newPattern = entry.getValue().replace("*", "$1"); + if (pattern.matches(oldPattern)) { + if (newPattern.contains("$1") && !oldPattern.contains("*")) { + newPattern = newPattern.replace("$1", "*"); + } + String p = pattern.replaceAll(oldPattern, newPattern); + renamedPatterns.add(p); + matched = true; + break; + } else if (pattern.equals(oldPattern)) { + renamedPatterns.add(entry.getValue()); + matched = true; + break; + } else if (pattern.contains(".*") + && oldPattern.matches(StringUtils.reformatPath(pattern))) { + renamedPatterns.add(entry.getKey()); + matched = true; + break; + } + } + if (!matched) { // 如果没有匹配的规则，添加原始模式 + renamedPatterns.add(pattern); + } + } + return renamedPatterns; + } + + // 判断是否Pattern a可以覆盖Pattern b + public static boolean covers(String a, String b) { + // 使用.*作为分隔符分割模式 + String[] partsA = a.split("\\*"); + String[] partsB = b.split("\\*"); + + int indexB = 0; + for (String part : partsA) { + boolean found = false; + while (indexB < partsB.length) { + if (partsB[indexB].contains(part)) { + found = true; + indexB++; // 移动到下一个部分 + break; + } + indexB++; + } + if (!found) { + return false; // 如果任何部分未找到匹配，则模式a不能覆盖模式b + } + } + return true; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java index 65f5a7721b..d78f01b032 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/FilterUtils.java @@ -27,23 +27,10 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; import cn.edu.tsinghua.iginx.engine.shared.expr.Expression; import cn.edu.tsinghua.iginx.engine.shared.function.system.utils.ValueUtils; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.BoolFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ExprFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.FilterType; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.KeyFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.NotFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.OrFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.PathFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ValueFilter; +import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; import cn.edu.tsinghua.iginx.utils.Pair; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; +import java.util.stream.Collectors; public class FilterUtils { @@ -352,12 +339,52 @@ public static Filter combineTwoFilter(Filter baseFilter, Filter additionFilter) } else if (baseFilter.getType().equals(FilterType.And)) { AndFilter andFilter = (AndFilter) baseFilter; List filterList = new ArrayList<>(andFilter.getChildren()); - filterList.add(additionFilter); - return new AndFilter(filterList); + if (additionFilter.getType().equals(FilterType.And)) { + AndFilter additionAndFilter = (AndFilter) additionFilter; + filterList.addAll(additionAndFilter.getChildren()); + } else { + filterList.add(additionFilter); + } + return new AndFilter(filterList.stream().distinct().collect(Collectors.toList())); } else { List filterList = new ArrayList<>(Collections.singletonList(baseFilter)); filterList.add(additionFilter); return new AndFilter(filterList); } } + + public static List getEqualPathFilter(Filter filter) { + List pathFilters = new ArrayList<>(); + filter.accept( + new FilterVisitor() { + @Override + public void visit(AndFilter filter) {} + + @Override + public void visit(OrFilter filter) {} + + @Override + public void visit(NotFilter filter) {} + + @Override + public void visit(KeyFilter filter) {} + + @Override + public void visit(ValueFilter filter) {} + + @Override + public void visit(PathFilter filter) { + if (isEqualOp(filter.getOp())) { + pathFilters.add(filter); + } + } + + @Override + public void visit(BoolFilter filter) {} + + @Override + public void visit(ExprFilter filter) {} + }); + return pathFilters; + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java index 38eb0890f2..0a7b2c54ae 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/utils/RowUtils.java @@ -174,7 +174,7 @@ public static HashMap> establishHashMap( HashMap> hashMap = new HashMap<>(); for (Row row : rows) { Value value = row.getAsValue(joinPath); - if (value == null) { + if (value == null || value.getValue() == null) { continue; } int hash = getHash(value, needTypeCast); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java index 92d82273fa..4ecf7e58c1 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/storage/StorageManager.java @@ -83,8 +83,10 @@ public static Pair getBoundaryOfStorage( return storage.getBoundaryOfStorage(dataPrefix); } catch (ClassNotFoundException e) { LOGGER.error("load class {} for engine {} failure: {}", driver, engine, e); + return null; } catch (Exception e) { LOGGER.error("unexpected error when process engine {}:", engine, e); + return null; } finally { try { if (needRelease) { @@ -94,7 +96,6 @@ public static Pair getBoundaryOfStorage( LOGGER.error("release session pool failure!"); } } - return new Pair<>(new ColumnsInterval(null, null), new KeyInterval(0, Long.MAX_VALUE)); } private boolean initStorage(StorageEngineMeta meta) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java index 77c12b67da..293e2b431c 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/RequestContext.java @@ -41,6 +41,10 @@ public class RequestContext { private ByteBuffer loadCSVFileByteBuffer; + private ByteBuffer UDFModuleByteBuffer; + + private boolean isRemoteUDF; + private String warningMsg; private void init() { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java index f4bef15d6d..67d0a3cf41 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/shared/Result.java @@ -62,6 +62,8 @@ public class Result { private List loadCSVColumns; private Long loadCSVRecordNum; + private String UDFModulePath; + private List sessionIDs; private Map rules; @@ -174,6 +176,8 @@ public ExecuteSqlResp getExecuteSqlResp() { resp.setLoadCsvPath(loadCSVPath); resp.setSessionIDList(sessionIDs); resp.setRules(rules); + // import udf + resp.setUDFModulePath(UDFModulePath); return resp; } @@ -276,6 +280,17 @@ public LoadCSVResp getLoadCSVResp() { return resp; } + public LoadUDFResp getLoadUDFResp() { + LoadUDFResp resp = new LoadUDFResp(status); + + if (status != RpcUtils.SUCCESS && status.code != StatusCode.PARTIAL_SUCCESS.getStatusCode()) { + resp.setParseErrorMsg(status.getMessage()); + return resp; + } + resp.setUDFModulePath(UDFModulePath); + return resp; + } + public FetchResultsResp fetch(int fetchSize) { FetchResultsResp resp = new FetchResultsResp(status, false); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java index dadf365065..356e28be5e 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/DefaultMetaManager.java @@ -1252,6 +1252,10 @@ private List resolveStorageEngineFromConf() { if (hasData) { StorageUnitMeta dummyStorageUnit = new StorageUnitMeta(generateDummyStorageUnitId(i), i); Pair boundary = StorageManager.getBoundaryOfStorage(storage); + if (boundary == null) { + LOGGER.error("Skip dummy engine:{}. Failed to obtain boundary.", storageEngineStrings[i]); + continue; + } FragmentMeta dummyFragment; if (dataPrefix == null) { diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java index 128ed5ea90..d9e9d78e22 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/metadata/entity/KeyInterval.java @@ -87,4 +87,8 @@ public KeyInterval getIntersectWithLCRO(KeyInterval keyInterval) { long end = Math.min(keyInterval.endKey, endKey); return new KeyInterval(start, end); } + + public static KeyInterval getDefaultKeyInterval() { + return new KeyInterval(Long.MIN_VALUE, Long.MAX_VALUE); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java index 6f3fb47cbc..d820d59a16 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/rest/RestUtils.java @@ -11,7 +11,7 @@ public class RestUtils { public static final long TITLE_KEY = 9223372036854775806L; public static final long MAX_KEY = 9223372036854775807L; public static final long ANNOTATION_START_KEY = 10L; - public static final String ANNOTATION_SEQUENCE = "TITLE.DESCRIPTION"; + public static final String ANNOTATION_SEQUENCE = "title.description"; public static DataType checkType(SessionQueryDataSet sessionQueryDataSet) { int n = sessionQueryDataSet.getKeys().length; diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java index 9bf4737fe6..f8461b9145 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/sql/statement/RegisterTaskStatement.java @@ -8,7 +8,11 @@ import cn.edu.tsinghua.iginx.thrift.Status; import cn.edu.tsinghua.iginx.thrift.UDFClassPair; import cn.edu.tsinghua.iginx.thrift.UDFType; +import cn.edu.tsinghua.iginx.utils.RpcUtils; +import java.io.File; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,8 +37,62 @@ public RegisterTaskStatement(String filePath, List pairs, List 1) { + errorMsg = + String.format( + "Fail to register %d UDFs with %d types, the number should be same or use only one type.", + pairs.size(), types.size()); + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } + + // fail if trying to register UDFs with same class name or name. + Set tempName = new HashSet<>(); + Set tempClass = new HashSet<>(); + for (UDFClassPair p : pairs) { + if (!tempName.add(p.name)) { + errorMsg = String.format("Cannot register multiple UDFs with same name: %s", p.name); + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } + if (!tempClass.add(p.classPath)) { + errorMsg = String.format("Cannot register multiple UDFs with same class: %s", p.classPath); + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } + } + return RpcUtils.SUCCESS; } } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java index 75471abd01..e7510fb036 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/LocalExecutor.java @@ -346,7 +346,7 @@ public List getColumnsOfStorageUnit(String storageUnit) throws PhysicalE @Override public Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException { - KeyInterval keyInterval = new KeyInterval(0, Long.MAX_VALUE); + KeyInterval keyInterval = KeyInterval.getDefaultKeyInterval(); ColumnsInterval columnsInterval; File directory = new File(FilePathUtils.toNormalFilePath(realDummyRoot, dataPrefix)); diff --git a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java index d3ffcbfa29..ece91da52a 100644 --- a/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java +++ b/dataSources/influxdb/src/main/java/cn/edu/tsinghua/iginx/influxdb/InfluxDBStorage.java @@ -168,72 +168,65 @@ private void reloadHistoryData() { public Pair getBoundaryOfStorage(String dataPrefix) throws PhysicalException { List bucketNames = new ArrayList<>(historyBucketMap.keySet()); - bucketNames.sort(String::compareTo); if (bucketNames.isEmpty()) { throw new InfluxDBTaskExecuteFailureException("InfluxDB has no bucket!"); } - ColumnsInterval columnsInterval; - if (dataPrefix == null) { - columnsInterval = - new ColumnsInterval( - bucketNames.get(0), StringUtils.nextString(bucketNames.get(bucketNames.size() - 1))); - } else { - columnsInterval = new ColumnsInterval(dataPrefix); + // 筛选出符合dataPrefix的bucket， + if (dataPrefix != null) { + bucketNames = + bucketNames.stream() + .filter( + bucketName -> + bucketName.startsWith(dataPrefix) || dataPrefix.startsWith(bucketName)) + .collect(Collectors.toList()); } - long minTime = Long.MAX_VALUE, maxTime = 0; - - String measurementPrefix = MEASUREMENTALL, fieldPrefix = FIELDALL; - if (dataPrefix != null - && dataPrefix.contains(".")) { // get the measurement and field from dataPrefix - int indexPrefix = dataPrefix.indexOf("."); - measurementPrefix = dataPrefix.substring(0, indexPrefix); - fieldPrefix = dataPrefix.substring(indexPrefix + 1); - } else if (dataPrefix != null) { - measurementPrefix = dataPrefix; + bucketNames.sort(String::compareTo); + ColumnsInterval columnsInterval; + + // 只需要查询首尾两端的记录即可 + String minPath = findExtremeRecordPath(bucketNames, dataPrefix, true); + String maxPath = findExtremeRecordPath(bucketNames, dataPrefix, false) + "~"; + + if (minPath == null || maxPath == null) { + throw new InfluxDBTaskExecuteFailureException( + "InfluxDB has no valid data! Maybe there are no data in each bucket or no data with the given data prefix!"); } - for (Bucket bucket : historyBucketMap.values()) { - String statement = - String.format( - QUERY_DATA_ALL, - bucket.getName(), - 0L, - Long.MAX_VALUE, - measurementPrefix.equals(MEASUREMENTALL) - ? MEASUREMENTALL - : "= \"" + measurementPrefix + "\"", - fieldPrefix.equals(FIELDALL) ? FIELDALL : "~ /" + fieldPrefix + ".*/"); - LOGGER.debug("execute statement: {}", statement); - // 查询 first - List tables = - client.getQueryApi().query(statement + " |> first()", organization.getId()); - for (FluxTable table : tables) { - for (FluxRecord record : table.getRecords()) { - long time = instantToNs(record.getTime()); - minTime = Math.min(time, minTime); - maxTime = Math.max(time, maxTime); - LOGGER.debug("record: {}", InfluxDBStorage.toString(table, record)); - } + KeyInterval keyInterval = new KeyInterval(Long.MIN_VALUE + 1, Long.MAX_VALUE); + columnsInterval = new ColumnsInterval(minPath, maxPath); + return new Pair<>(columnsInterval, keyInterval); + } + + private String findExtremeRecordPath( + List bucketNames, String dataPrefix, boolean findMin) { + String extremePath = null; + int startIndex = findMin ? 0 : bucketNames.size() - 1; + int endIndex = findMin ? bucketNames.size() : -1; + int step = findMin ? 1 : -1; + + for (int i = startIndex; i != endIndex; i += step) { + String bucketName = bucketNames.get(i); + String statement = String.format(SHOW_TIME_SERIES, bucketName); + List tables = client.getQueryApi().query(statement, organization.getId()); + if (tables.isEmpty()) { + continue; } - // 查询 last - tables = client.getQueryApi().query(statement + " |> last()", organization.getId()); - for (FluxTable table : tables) { - for (FluxRecord record : table.getRecords()) { - long time = instantToNs(record.getTime()); - minTime = Math.min(time, minTime); - maxTime = Math.max(time, maxTime); - LOGGER.debug("record: {}", InfluxDBStorage.toString(table, record)); + for (FluxTable fluxTable : tables) { + // 找到measurement和field + String measurement = fluxTable.getRecords().get(0).getMeasurement(); + String field = fluxTable.getRecords().get(0).getField(); + String path = bucketName + "." + measurement + "." + field; + if (dataPrefix != null && !path.startsWith(dataPrefix)) { + continue; + } + if (extremePath == null + || (findMin ? path.compareTo(extremePath) < 0 : path.compareTo(extremePath) > 0)) { + extremePath = path; } } + break; } - if (minTime == Long.MAX_VALUE) { - minTime = 0; - } - if (maxTime == 0) { - maxTime = Long.MAX_VALUE - 1; - } - KeyInterval keyInterval = new KeyInterval(minTime, maxTime + 1); - return new Pair<>(columnsInterval, keyInterval); + return extremePath; } @Override diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index caebf63259..74be91517d 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -181,37 +181,7 @@ record = dataSet.next(); } // 获取 key 范围 - long minTime = 0, maxTime = Long.MAX_VALUE; - try { - // 获取 key 的最小值 - if (dataPrefix == null || dataPrefix.isEmpty()) { - dataSet = sessionPool.executeQueryStatement("select * from root"); - } else { - dataSet = sessionPool.executeQueryStatement("select " + dataPrefix + " from root"); - } - if (dataSet.hasNext()) { - record = dataSet.next(); - minTime = record.getTimestamp(); - } - dataSet.close(); - - // 获取 key 的最大值 - if (dataPrefix == null || dataPrefix.isEmpty()) - dataSet = sessionPool.executeQueryStatement("select * from root order by time desc"); - else { - dataSet = - sessionPool.executeQueryStatement( - "select " + dataPrefix + " from root order by time desc"); - } - if (dataSet.hasNext()) { - record = dataSet.next(); - maxTime = record.getTimestamp(); - } - dataSet.close(); - } catch (IoTDBConnectionException | StatementExecutionException e) { - throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); - } - KeyInterval keyInterval = new KeyInterval(minTime, maxTime + 1); + KeyInterval keyInterval = KeyInterval.getDefaultKeyInterval(); return new Pair<>(columnsInterval, keyInterval); } diff --git a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java index 3225ba2967..2dde3d2ca8 100644 --- a/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java +++ b/dataSources/mongodb/src/main/java/cn/edu/tsinghua/iginx/mongodb/MongoDBStorage.java @@ -381,7 +381,7 @@ public Pair getBoundaryOfStorage(String prefix) ColumnsInterval columnsInterval = new ColumnsInterval(first.getStartColumn(), last.getEndColumn()); - KeyInterval keyInterval = new KeyInterval(Long.MIN_VALUE, Long.MAX_VALUE); + KeyInterval keyInterval = KeyInterval.getDefaultKeyInterval(); return new Pair<>(columnsInterval, keyInterval); } diff --git a/dataSources/opentsdb/pom.xml b/dataSources/opentsdb/pom.xml deleted file mode 100644 index 3280ebc48c..0000000000 --- a/dataSources/opentsdb/pom.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - 4.0.0 - - - cn.edu.tsinghua - iginx-driver - ${revision} - - - opentsdb - IGinX OpenTSDB - - - .. - opentsdb - 8 - 8 - - - - - com.github.eulery - opentsdb-java-sdk - 1.1.7 - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - - copy-dependencies - - package - - ../../core/target/iginx-core-${project.version}/driver/opentsdb/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - - run - - package - - - - - - - - - - - - - - - - diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/OpenTSDBStorage.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/OpenTSDBStorage.java deleted file mode 100644 index 576e0145bb..0000000000 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/OpenTSDBStorage.java +++ /dev/null @@ -1,609 +0,0 @@ -package cn.edu.tsinghua.iginx.opentsdb; - -import static cn.edu.tsinghua.iginx.opentsdb.tools.DataTypeTransformer.DATA_TYPE; -import static cn.edu.tsinghua.iginx.opentsdb.tools.DataTypeTransformer.fromOpenTSDB; - -import cn.edu.tsinghua.iginx.engine.physical.exception.NonExecutablePhysicalTaskException; -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; -import cn.edu.tsinghua.iginx.engine.physical.exception.StorageInitializationException; -import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; -import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Column; -import cn.edu.tsinghua.iginx.engine.physical.storage.domain.DataArea; -import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; -import cn.edu.tsinghua.iginx.engine.shared.KeyRange; -import cn.edu.tsinghua.iginx.engine.shared.data.read.ClearEmptyRowStreamWrapper; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; -import cn.edu.tsinghua.iginx.engine.shared.operator.*; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.type.OperatorType; -import cn.edu.tsinghua.iginx.metadata.entity.*; -import cn.edu.tsinghua.iginx.opentsdb.query.entity.OpenTSDBRowStream; -import cn.edu.tsinghua.iginx.opentsdb.query.entity.OpenTSDBSchema; -import cn.edu.tsinghua.iginx.opentsdb.tools.DataViewWrapper; -import cn.edu.tsinghua.iginx.opentsdb.tools.TagKVUtils; -import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.thrift.StorageEngineType; -import cn.edu.tsinghua.iginx.utils.Pair; -import cn.edu.tsinghua.iginx.utils.StringUtils; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; -import org.apache.http.nio.reactor.IOReactorException; -import org.opentsdb.client.OpenTSDBClient; -import org.opentsdb.client.OpenTSDBClientFactory; -import org.opentsdb.client.OpenTSDBConfig; -import org.opentsdb.client.bean.request.Point; -import org.opentsdb.client.bean.request.Query; -import org.opentsdb.client.bean.request.SubQuery; -import org.opentsdb.client.bean.request.SuggestQuery; -import org.opentsdb.client.bean.response.QueryResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class OpenTSDBStorage implements IStorage { - private static final Logger LOGGER = LoggerFactory.getLogger(OpenTSDBStorage.class); - - private static final String DU_PREFIX = "unit"; - - private static final int HTTP_CONNECT_POOL_SIZE = 100; - - private static final int HTTP_CONNECT_TIMEOUT = 100; - - private final OpenTSDBClient client; - - private final StorageEngineMeta meta; - - public OpenTSDBStorage(StorageEngineMeta meta) throws StorageInitializationException { - this.meta = meta; - if (!meta.getStorageEngine().equals(StorageEngineType.opentsdb)) { - throw new StorageInitializationException("unexpected database: " + meta.getStorageEngine()); - } - if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - Map extraParams = meta.getExtraParams(); - String url = extraParams.getOrDefault("url", "http://127.0.0.1"); - - OpenTSDBConfig config = - OpenTSDBConfig.address(url, meta.getPort()) - .httpConnectionPool(HTTP_CONNECT_POOL_SIZE) - .httpConnectTimeout(HTTP_CONNECT_TIMEOUT) - .config(); - try { - client = OpenTSDBClientFactory.connect(config); - } catch (IOReactorException e) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - LOGGER.info("{} is initialized.", meta); - } - - private boolean testConnection() { - Map extraParams = meta.getExtraParams(); - String url = extraParams.get("url"); - OpenTSDBConfig config = OpenTSDBConfig.address(url, meta.getPort()).config(); - try { - OpenTSDBClient client = OpenTSDBClientFactory.connect(config); - client.gracefulClose(); - } catch (IOException e) { - LOGGER.error("test connection error: ", e); - return false; - } - return true; - } - - public TaskExecuteResult execute(StoragePhysicalTask task) { - List operators = task.getOperators(); - if (operators.size() != 1) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - FragmentMeta fragment = task.getTargetFragment(); - Operator op = operators.get(0); - String storageUnit = task.getStorageUnit(); - - boolean isDummyStorageUnit = task.isDummyStorageUnit(); - if (op.getType() == OperatorType.Project) { - Project project = (Project) op; - return isDummyStorageUnit - ? executeProjectHistoryTask(fragment.getKeyInterval(), storageUnit, project) - : executeProjectTask(fragment.getKeyInterval(), storageUnit, project); - } else if (op.getType() == OperatorType.Insert) { - Insert insert = (Insert) op; - return executeInsertTask(storageUnit, insert); - } else if (op.getType() == OperatorType.Delete) { - Delete delete = (Delete) op; - return executeDeleteTask(storageUnit, delete); - } - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - - private TaskExecuteResult executeDeleteTask(String storageUnit, Delete delete) { - List schemas = null; - try { - schemas = determineDeletePathList(storageUnit, delete); - } catch (PhysicalException e) { - LOGGER.error("encounter error when delete data in opentsdb: ", e); - } - if (schemas == null || schemas.size() == 0) { - return new TaskExecuteResult(null, null); - } - - if (delete.getKeyRanges() == null || delete.getKeyRanges().size() == 0) { // 没有传任何 time range - for (OpenTSDBSchema schema : schemas) { - Query query = - Query.begin(0L) - .end(Long.MAX_VALUE) - .sub( - SubQuery.metric(schema.getMetric()) - .aggregator(SubQuery.Aggregator.NONE) - .build()) - .msResolution() - .build(); - try { - client.delete(query); - } catch (Exception e) { - LOGGER.error("encounter error when delete data in opentsdb: ", e); - } - } - return new TaskExecuteResult(null, null); - } - // 删除某些序列的某一段数据 - for (OpenTSDBSchema schema : schemas) { - for (KeyRange keyRange : delete.getKeyRanges()) { - Query query = - Query.begin(keyRange.getActualBeginKey()) - .end(keyRange.getActualEndKey()) - .sub( - SubQuery.metric(schema.getMetric()) - .aggregator(SubQuery.Aggregator.NONE) - .build()) - .msResolution() - .build(); - try { - client.delete(query); - } catch (Exception e) { - LOGGER.error("encounter error when delete data in opentsdb: ", e); - } - } - } - return new TaskExecuteResult(null, null); - } - - private List determineDeletePathList(String storageUnit, Delete delete) - throws PhysicalException { - List schemas = new ArrayList<>(); - - if (delete.getTagFilter() == null) { - delete - .getPatterns() - .forEach(pattern -> schemas.add(new OpenTSDBSchema(pattern, storageUnit))); - } else { - List patterns = delete.getPatterns(); - TagFilter tagFilter = delete.getTagFilter(); - List timeSeries = getColumns(); - - for (Column ts : timeSeries) { - for (String pattern : patterns) { - if (Pattern.matches(StringUtils.reformatPath(pattern), ts.getPath()) - && TagKVUtils.match(ts.getTags(), tagFilter)) { - schemas.add(new OpenTSDBSchema(ts.getPhysicalPath(), storageUnit)); - break; - } - } - } - } - return schemas; - } - - private TaskExecuteResult executeInsertTask(String storageUnit, Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertRowRecords((RowDataView) dataView, storageUnit); - break; - case Column: - case NonAlignedColumn: - e = insertColRecords((ColumnDataView) dataView, storageUnit); - } - if (e != null) { - return new TaskExecuteResult( - null, new PhysicalException("execute insert task in opentsdb failure", e)); - } - return new TaskExecuteResult(null, null); - } - - private Exception insertRowRecords(RowDataView dataView, String storageUnit) { - DataViewWrapper data = new DataViewWrapper(dataView); - List schemas = new ArrayList<>(); - for (int i = 0; i < data.getPathNum(); i++) { - schemas.add(new OpenTSDBSchema(data.getPath(i), storageUnit)); - } - - List points = new ArrayList<>(); - for (int i = 0; i < data.getTimeSize(); i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - if (bitmapView.get(j)) { - OpenTSDBSchema schema = schemas.get(j); - DataType type = data.getDataType(j); - switch (type) { - case BOOLEAN: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (boolean) data.getValue(i, index) ? 1 : 0) - .build()); - break; - case INTEGER: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (int) data.getValue(i, index)) - .build()); - break; - case LONG: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (long) data.getValue(i, index)) - .build()); - break; - case FLOAT: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (float) data.getValue(i, index)) - .build()); - break; - case DOUBLE: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (double) data.getValue(i, index)) - .build()); - break; - case BINARY: - return new PhysicalTaskExecuteFailureException( - "opentsdb not support string for now!"); - } - index++; - } - } - } - - try { - LOGGER.info("开始数据写入"); - client.putSync(points); - } catch (Exception e) { - LOGGER.error("encounter error when write points to opentsdb: ", e); - } finally { - LOGGER.info("数据写入完毕！"); - } - return null; - } - - private Exception insertColRecords(ColumnDataView dataView, String storageUnit) { - DataViewWrapper data = new DataViewWrapper(dataView); - List points = new ArrayList<>(); - for (int i = 0; i < data.getPathNum(); i++) { - OpenTSDBSchema schema = new OpenTSDBSchema(data.getPath(i), storageUnit); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - DataType type = data.getDataType(i); - switch (type) { - case BOOLEAN: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (boolean) data.getValue(i, index) ? 1 : 0) - .build()); - break; - case INTEGER: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (int) data.getValue(i, index)) - .build()); - break; - case LONG: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (long) data.getValue(i, index)) - .build()); - break; - case FLOAT: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (float) data.getValue(i, index)) - .build()); - break; - case DOUBLE: - points.add( - Point.metric(schema.getMetric()) - .tag(schema.getTags()) - .tag(DATA_TYPE, type.toString()) - .value(data.getTimestamp(i), (double) data.getValue(i, index)) - .build()); - break; - case BINARY: - return new PhysicalTaskExecuteFailureException( - "opentsdb not support string for now!"); - } - index++; - } - } - } - - try { - LOGGER.info("开始数据写入"); - client.putSync(points); - } catch (Exception e) { - LOGGER.error("encounter error when write points to opentsdb: ", e); - } finally { - LOGGER.info("数据写入完毕！"); - } - return null; - } - - private TaskExecuteResult executeProjectTask( - KeyInterval keyInterval, String storageUnit, Project project) { - List wholePathList; - try { - wholePathList = getPathList(); - } catch (PhysicalTaskExecuteFailureException e) { - return new TaskExecuteResult( - new PhysicalException("encounter error when query data in opentsdb: ", e)); - } - - Query.Builder builder = - Query.begin(keyInterval.getStartKey()).end(keyInterval.getEndKey()).msResolution(); - for (String queryPath : project.getPatterns()) { - if (StringUtils.isPattern(queryPath)) { - Pattern pattern = Pattern.compile(StringUtils.reformatPath(queryPath)); - for (String path : wholePathList) { - if (pattern.matcher(path).matches()) { - OpenTSDBSchema schema = new OpenTSDBSchema(path, storageUnit); - builder = - builder.sub( - SubQuery.metric(schema.getMetric()) - .aggregator(SubQuery.Aggregator.NONE) - .build()); - } - } - } else { - OpenTSDBSchema schema = new OpenTSDBSchema(queryPath, storageUnit); - builder = - builder.sub( - SubQuery.metric(schema.getMetric()).aggregator(SubQuery.Aggregator.NONE).build()); - } - } - Query query = builder.build(); - try { - List resultList = client.query(query); - RowStream rowStream = new ClearEmptyRowStreamWrapper(new OpenTSDBRowStream(resultList, true)); - return new TaskExecuteResult(rowStream); - } catch (Exception e) { - return new TaskExecuteResult( - new PhysicalException("encounter error when query data in opentsdb: ", e)); - } - } - - private TaskExecuteResult executeProjectHistoryTask( - KeyInterval keyInterval, String storageUnit, Project project) { - List wholePathList; - try { - wholePathList = getPathList(); - } catch (PhysicalTaskExecuteFailureException e) { - return new TaskExecuteResult( - new PhysicalException("encounter error when query data in opentsdb: ", e)); - } - - Query.Builder builder = - Query.begin(keyInterval.getStartKey()).end(keyInterval.getEndKey()).msResolution(); - for (String queryPath : project.getPatterns()) { - if (StringUtils.isPattern(queryPath)) { - Pattern pattern = Pattern.compile(StringUtils.reformatPath(queryPath)); - for (String path : wholePathList) { - if (pattern.matcher(path).matches()) { - builder = - builder.sub(SubQuery.metric(path).aggregator(SubQuery.Aggregator.NONE).build()); - } - } - } else { - builder = - builder.sub(SubQuery.metric(queryPath).aggregator(SubQuery.Aggregator.NONE).build()); - } - } - Query query = builder.build(); - try { - List resultList = client.query(query); - RowStream rowStream = - new ClearEmptyRowStreamWrapper(new OpenTSDBRowStream(resultList, false)); - return new TaskExecuteResult(rowStream); - } catch (Exception e) { - return new TaskExecuteResult( - new PhysicalException("encounter error when query data in opentsdb: ", e)); - } - } - - @Override - public Pair getBoundaryOfStorage(String prefix) - throws PhysicalException { - List paths = getPurePath(); - paths.sort(String::compareTo); - if (paths.isEmpty()) { - throw new PhysicalTaskExecuteFailureException("no data!"); - } - ColumnsInterval columnsInterval = - new ColumnsInterval(paths.get(0), StringUtils.nextString(paths.get(paths.size() - 1))); - - long minTime = 0, maxTime = Long.MAX_VALUE - 1; - Query.Builder builder = Query.begin(0L).end(Long.MAX_VALUE).msResolution(); - for (String path : paths) { - builder = builder.sub(SubQuery.metric(path).aggregator(SubQuery.Aggregator.NONE).build()); - } - Query query = builder.build(); - try { - List resultList = client.query(query); - List times = new ArrayList<>(); - for (QueryResult result : resultList) { - times.addAll(result.getDps().keySet()); - } - if (!times.isEmpty()) { - times.sort(Long::compareTo); - minTime = times.get(0); - maxTime = times.get(times.size() - 1); - } - } catch (Exception e) { - throw new PhysicalTaskExecuteFailureException( - "encounter error when query data in opentsdb: ", e); - } - KeyInterval keyInterval = new KeyInterval(minTime, maxTime + 1); - return new Pair<>(columnsInterval, keyInterval); - } - - @Override - public TaskExecuteResult executeProject(Project project, DataArea dataArea) { - return null; - } - - @Override - public TaskExecuteResult executeProjectDummy(Project project, DataArea dataArea) { - return null; - } - - @Override - public boolean isSupportProjectWithSelect() { - return false; - } - - @Override - public TaskExecuteResult executeProjectWithSelect( - Project project, Select select, DataArea dataArea) { - return null; - } - - @Override - public TaskExecuteResult executeProjectDummyWithSelect( - Project project, Select select, DataArea dataArea) { - return null; - } - - @Override - public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { - return null; - } - - @Override - public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { - return null; - } - - @Override - public List getColumns() throws PhysicalException { - List timeseries = new ArrayList<>(); - - List paths = getPathWithoutDUPrefixList(); - if (paths.isEmpty()) { - return timeseries; - } - - Query.Builder builder = Query.begin(0L).end(Long.MAX_VALUE).msResolution(); - for (String path : paths) { - builder = builder.sub(SubQuery.metric(path).aggregator(SubQuery.Aggregator.NONE).build()); - } - Query query = builder.build(); - try { - List resultList = client.query(query); - for (QueryResult result : resultList) { - String path = result.getMetric(); - if (path.startsWith("unit")) { - path = path.substring(path.indexOf('.') + 1); - } - Pair> pair = TagKVUtils.splitFullName(path); - timeseries.add(new Column(pair.k, fromOpenTSDB(result.getTags().get(DATA_TYPE)), pair.v)); - } - } catch (Exception e) { - throw new PhysicalTaskExecuteFailureException( - "encounter error when query data in opentsdb: ", e); - } - return timeseries; - } - - private List getPurePath() throws PhysicalTaskExecuteFailureException { - List suggests = getPathList(); - - List paths = new ArrayList<>(); - for (String metric : suggests) { - String path = metric; - if (path.startsWith(DU_PREFIX)) { - path = metric.substring(metric.indexOf(".") + 1); - } - paths.add(TagKVUtils.splitFullName(path).k); - } - return paths; - } - - private List getPathWithoutDUPrefixList() throws PhysicalTaskExecuteFailureException { - List suggests = getPathList(); - - List paths = new ArrayList<>(); - for (String metric : suggests) { - if (metric.startsWith(DU_PREFIX)) { - paths.add(metric.substring(metric.indexOf(".") + 1)); - } else { - paths.add(metric); - } - } - return paths; - } - - private List getPathList() throws PhysicalTaskExecuteFailureException { - SuggestQuery suggestQuery = - SuggestQuery.type(SuggestQuery.Type.METRICS).max(Integer.MAX_VALUE).build(); - try { - return client.querySuggest(suggestQuery); - } catch (Exception e) { - throw new PhysicalTaskExecuteFailureException("get time series failure: ", e); - } - } - - @Override - public void release() throws PhysicalException { - try { - client.gracefulClose(); - } catch (IOException e) { - LOGGER.error("can not close opentsdb gracefully, because ", e); - try { - client.forceClose(); - } catch (IOException ioException) { - throw new PhysicalTaskExecuteFailureException("can not close opentsdb, because ", e); - } - } - } -} diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java deleted file mode 100644 index 86a943071c..0000000000 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBRowStream.java +++ /dev/null @@ -1,115 +0,0 @@ -package cn.edu.tsinghua.iginx.opentsdb.query.entity; - -import static cn.edu.tsinghua.iginx.opentsdb.tools.DataTypeTransformer.*; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.opentsdb.tools.TagKVUtils; -import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.utils.Pair; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import org.opentsdb.client.bean.response.QueryResult; - -public class OpenTSDBRowStream implements RowStream { - - private static final String STORAGE_UNIT = "du"; - - private final Header header; - - private final List resultList; - - private final Iterator>[] iterators; - - private final Map.Entry[] curData; - - private final boolean[] finished; - - private final boolean trimStorageUnit; - - private int hasMoreRecords; - - @SuppressWarnings("unchecked") - public OpenTSDBRowStream(List resultList, boolean trimStorageUnit) { - this.resultList = resultList; - this.trimStorageUnit = trimStorageUnit; - - List fields = new ArrayList<>(); - for (QueryResult res : resultList) { - String metric = res.getMetric(); - String path = metric; - if (trimStorageUnit && res.getTags().containsKey(STORAGE_UNIT)) { - path = metric.substring(metric.indexOf(".") + 1); - } - DataType dataType = fromOpenTSDB(res.getTags().get(DATA_TYPE)); - Pair> pair = TagKVUtils.splitFullName(path); - fields.add(new Field(pair.getK(), dataType, pair.getV())); - } - this.header = new Header(Field.KEY, fields); - - this.iterators = new Iterator[this.resultList.size()]; - this.curData = new Map.Entry[this.resultList.size()]; - this.finished = new boolean[this.resultList.size()]; - for (int i = 0; i < resultList.size(); i++) { - this.iterators[i] = resultList.get(i).getDps().entrySet().iterator(); - } - this.hasMoreRecords = this.resultList.size(); - } - - @Override - public Header getHeader() { - return header; - } - - @Override - public void close() throws PhysicalException { - // need to do nothing - } - - @Override - public boolean hasNext() throws PhysicalException { - return this.hasMoreRecords > 0; - } - - @Override - public Row next() throws PhysicalException { - long timestamp = Long.MAX_VALUE; - for (int i = 0; i < this.resultList.size(); i++) { - Iterator> iterator = this.iterators[i]; - if (!iterator.hasNext() && curData[i] == null) { // 数据已经消费完毕了 - continue; - } - if (curData[i] == null) { - Map.Entry entry = iterator.next(); - timestamp = Math.min(timestamp, entry.getKey()); - curData[i] = entry; - } - } - if (timestamp == Long.MAX_VALUE) { - return null; - } - Object[] values = new Object[this.resultList.size()]; - for (int i = 0; i < this.resultList.size(); i++) { - Iterator> iterator = this.iterators[i]; - if (!iterator.hasNext() && curData[i] == null) { // 数据已经消费完毕了 - continue; - } - if (curData[i].getKey() == timestamp) { - values[i] = getValue(resultList.get(i).getTags().get(DATA_TYPE), curData[i].getValue()); - curData[i] = null; - } - if (!iterator.hasNext() && curData[i] == null) { // 数据已经消费完毕了 - if (!finished[i]) { - finished[i] = true; - hasMoreRecords--; - } - } - } - return new Row(header, timestamp, values); - } -} diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBSchema.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBSchema.java deleted file mode 100644 index a869b10281..0000000000 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/query/entity/OpenTSDBSchema.java +++ /dev/null @@ -1,31 +0,0 @@ -package cn.edu.tsinghua.iginx.opentsdb.query.entity; - -import java.util.HashMap; -import java.util.Map; - -public class OpenTSDBSchema { - - private static final String STORAGE_UNIT = "du"; - - private final String metric; - - private final Map tags = new HashMap<>(); - - public OpenTSDBSchema(String path, String storageUnit) { - this.metric = storageUnit + "." + path; - this.tags.put(STORAGE_UNIT, storageUnit); - } - - public String getMetric() { - return metric; - } - - public Map getTags() { - return tags; - } - - @Override - public String toString() { - return "OpenTSDBSchema{" + "measurement='" + metric + '\'' + ", tags=" + tags + '}'; - } -} diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataTypeTransformer.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataTypeTransformer.java deleted file mode 100644 index 614e668f25..0000000000 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataTypeTransformer.java +++ /dev/null @@ -1,48 +0,0 @@ -package cn.edu.tsinghua.iginx.opentsdb.tools; - -import static cn.edu.tsinghua.iginx.thrift.DataType.*; -import static cn.edu.tsinghua.iginx.thrift.DataType.BINARY; - -import cn.edu.tsinghua.iginx.exception.UnsupportedDataTypeException; -import cn.edu.tsinghua.iginx.thrift.DataType; - -public class DataTypeTransformer { - - public static final String DATA_TYPE = "data_type"; - - public static DataType fromOpenTSDB(String dataType) { - switch (dataType.toLowerCase()) { - case "boolean": - return BOOLEAN; - case "integer": - return INTEGER; - case "long": - return LONG; - case "float": - return FLOAT; - case "double": - return DOUBLE; - case "binary": - return BINARY; - default: - throw new UnsupportedDataTypeException(dataType); - } - } - - public static Object getValue(String dataType, Number data) { - switch (dataType.toLowerCase()) { - case "boolean": - return data.intValue() == 1; - case "integer": - return data.intValue(); - case "long": - return data.longValue(); - case "float": - return data.floatValue(); - case "double": - return data.doubleValue(); - default: - throw new UnsupportedDataTypeException(dataType); - } - } -} diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java deleted file mode 100644 index 742a75daab..0000000000 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/DataViewWrapper.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.opentsdb.tools; - -import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; -import cn.edu.tsinghua.iginx.thrift.DataType; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -public class DataViewWrapper { - - private final DataView dataView; - - private final Map pathCache; - - public DataViewWrapper(DataView dataView) { - this.dataView = dataView; - this.pathCache = new HashMap<>(); - } - - public int getPathNum() { - return dataView.getPathNum(); - } - - public int getTimeSize() { - return dataView.getKeySize(); - } - - public String getPath(int index) { - if (pathCache.containsKey(index)) { - return pathCache.get(index); - } - String path = dataView.getPath(index); - Map tags = dataView.getTags(index); - if (tags != null && !tags.isEmpty()) { - TreeMap sortedTags = new TreeMap<>(tags); - StringBuilder pathBuilder = new StringBuilder(); - sortedTags.forEach( - (tagKey, tagValue) -> { - pathBuilder.append('.').append(tagKey).append('.').append(tagValue); - }); - path += pathBuilder.toString(); - } - pathCache.put(index, path); - return path; - } - - public DataType getDataType(int index) { - return dataView.getDataType(index); - } - - public Long getTimestamp(int index) { - return dataView.getKey(index); - } - - public Object getValue(int index1, int index2) { - return dataView.getValue(index1, index2); - } - - public BitmapView getBitmapView(int index) { - return dataView.getBitmapView(index); - } -} diff --git a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java b/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java deleted file mode 100644 index 02df252474..0000000000 --- a/dataSources/opentsdb/src/main/java/cn/edu/tsinghua/iginx/opentsdb/tools/TagKVUtils.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.opentsdb.tools; - -import cn.edu.tsinghua.iginx.conf.Config; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.utils.Pair; -import cn.edu.tsinghua.iginx.utils.StringUtils; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TagKVUtils { - @SuppressWarnings("unused") - private static final Logger LOGGER = LoggerFactory.getLogger(TagKVUtils.class); - - public static final String tagNameAnnotation = Config.tagNameAnnotation; - - public static Pair> splitFullName(String fullName) { - if (!fullName.contains(tagNameAnnotation)) { - return new Pair<>(fullName, null); - } - - String[] parts = fullName.split(tagNameAnnotation, 2); - assert parts.length == 2; - String name = parts[0].substring(0, parts[0].length() - 1); - - List tagKVList = - Arrays.stream(parts[1].split("\\.")) - .map( - e -> { - if (e.startsWith(tagNameAnnotation)) { - return e.substring(tagNameAnnotation.length()); - } else { - return e; - } - }) - .collect(Collectors.toList()); - assert tagKVList.size() % 2 == 0; - Map tags = new HashMap<>(); - for (int i = 0; i < tagKVList.size(); i++) { - if (i % 2 == 0) { - continue; - } - String tagKey = tagKVList.get(i - 1); - String tagValue = tagKVList.get(i); - tags.put(tagKey, tagValue); - } - - return new Pair<>(name, tags); - } - - public static boolean match(Map tags, TagFilter tagFilter) { - if (tags == null || tags.isEmpty()) { - return false; - } - switch (tagFilter.getType()) { - case And: - return match(tags, (AndTagFilter) tagFilter); - case Or: - return match(tags, (OrTagFilter) tagFilter); - case Base: - return match(tags, (BaseTagFilter) tagFilter); - // TODO: case label - case BasePrecise: - break; - case Precise: - break; - case WithoutTag: - break; - } - return false; - } - - private static boolean match(Map tags, AndTagFilter tagFilter) { - List children = tagFilter.getChildren(); - for (TagFilter child : children) { - if (!match(tags, child)) { - return false; - } - } - return true; - } - - private static boolean match(Map tags, OrTagFilter tagFilter) { - List children = tagFilter.getChildren(); - for (TagFilter child : children) { - if (match(tags, child)) { - return true; - } - } - return false; - } - - private static boolean match(Map tags, BaseTagFilter tagFilter) { - String tagKey = tagFilter.getTagKey(); - String expectedValue = tagFilter.getTagValue(); - if (!tags.containsKey(tagKey)) { - return false; - } - String actualValue = tags.get(tagKey); - if (!StringUtils.isPattern(expectedValue)) { - return expectedValue.equals(actualValue); - } else { - return Pattern.matches(StringUtils.reformatPath(expectedValue), actualValue); - } - } -} diff --git a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java index 8fd5a57f84..24afeaddee 100644 --- a/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java +++ b/dataSources/parquet/src/main/java/cn/edu/tsinghua/iginx/parquet/manager/dummy/DummyManager.java @@ -28,11 +28,8 @@ import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.parquet.manager.Manager; -import cn.edu.tsinghua.iginx.parquet.manager.utils.RangeUtils; import cn.edu.tsinghua.iginx.parquet.manager.utils.TagKVUtils; import cn.edu.tsinghua.iginx.utils.StringUtils; -import com.google.common.collect.Range; -import com.google.common.collect.TreeRangeSet; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -158,23 +155,8 @@ public List getColumns() throws PhysicalException { } @Override - public KeyInterval getKeyInterval() throws PhysicalException { - TreeRangeSet rangeSet = TreeRangeSet.create(); - - for (Path path : getFilePaths()) { - try { - Range range = new Loader(path).getRange(); - rangeSet.add(range); - } catch (Exception e) { - throw new PhysicalException("failed to get range from " + path + ": " + e, e); - } - } - - if (rangeSet.isEmpty()) { - return new KeyInterval(0, 0); - } - Range span = rangeSet.span(); - return RangeUtils.toKeyInterval(span); + public KeyInterval getKeyInterval() { + return KeyInterval.getDefaultKeyInterval(); } @Override diff --git a/dataSources/pom.xml b/dataSources/pom.xml index fa18b01916..a1cdc62a91 100644 --- a/dataSources/pom.xml +++ b/dataSources/pom.xml @@ -32,10 +32,9 @@ influxdb iotdb12 mongodb - opentsdb parquet - postgresql redis + relational diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgreSQLException.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgreSQLException.java deleted file mode 100644 index dfe03d670f..0000000000 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgreSQLException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2024 IGinX of Tsinghua University - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package cn.edu.tsinghua.iginx.postgresql.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; - -public class PostgreSQLException extends PhysicalException { - - public PostgreSQLException(Throwable cause) { - super(cause); - } - - public PostgreSQLException(String message) { - super(message); - } - - public PostgreSQLException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgresqlTaskExecuteFailureException.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgresqlTaskExecuteFailureException.java deleted file mode 100644 index c39d2e2fe0..0000000000 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/exception/PostgresqlTaskExecuteFailureException.java +++ /dev/null @@ -1,14 +0,0 @@ -package cn.edu.tsinghua.iginx.postgresql.exception; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; - -public class PostgresqlTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { - - public PostgresqlTaskExecuteFailureException(String message) { - super(message); - } - - public PostgresqlTaskExecuteFailureException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java deleted file mode 100644 index c27c0006ce..0000000000 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/Constants.java +++ /dev/null @@ -1,67 +0,0 @@ -package cn.edu.tsinghua.iginx.postgresql.tools; - -public class Constants { - public static final String TAGKV_EQUAL = "="; - - public static final String TAGKV_SEPARATOR = "-"; - - public static final int BATCH_SIZE = 10000; - - public static final String USERNAME = "username"; - - public static final String PASSWORD = "password"; - - public static final String DEFAULT_USERNAME = "postgres"; - - public static final String DEFAULT_PASSWORD = "postgres"; - - public static final String DATABASE_PREFIX = "unit"; - - public static final String KEY_NAME = "postgresql+key"; - - public static final String QUERY_DATABASES_STATEMENT = "SELECT datname FROM pg_database;"; - - public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; - - public static final String QUERY_KEY_STATEMENT = - "SELECT \"" + KEY_NAME + "\" FROM %s ORDER BY \"" + KEY_NAME + "\";"; - - public static final String CONCAT_QUERY_STATEMENT = "SELECT concat(%s) FROM %s;"; - - public static final String QUERY_STATEMENT = - "SELECT \"" + KEY_NAME + "\", %s FROM %s %s ORDER BY \"" + KEY_NAME + "\";"; - - public static final String QUERY_STATEMENT_WITHOUT_KEYNAME = "SELECT %s FROM %s %s ORDER BY %s;"; - - public static final String QUERY_KEY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT %s FROM %s ORDER BY \"" + KEY_NAME + "\";"; - - public static final String CONCAT_QUERY_STATEMENT_WITHOUT_WHERE_CLAUSE = - "SELECT concat(%s) AS \"" + KEY_NAME + "\", %s FROM %s ORDER BY \"" + KEY_NAME + "\";"; - - public static final String CONCAT_QUERY_STATEMENT_WITH_WHERE_CLAUSE = - "SELECT concat(%s) AS \"" - + KEY_NAME - + "\", %s FROM %s WHERE %s ORDER BY \"" - + KEY_NAME - + "\";"; - - public static final String CONCAT_QUERY_STATEMENT_AND_CONCAT_KEY = - "SELECT concat(%s) AS \"" + KEY_NAME + "\", %s FROM %s %s ORDER BY concat(%s);"; - - public static final String CREATE_TABLE_STATEMENT = - "CREATE TABLE %s (\"" - + KEY_NAME - + "\" BIGINT NOT NULL, %s %s, PRIMARY KEY(\"" - + KEY_NAME - + "\"));"; - - public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; - - public static final String DROP_DATABASE_STATEMENT = "DROP DATABASE %s;"; - - public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN IF EXISTS %s;"; - - public static final String UPDATE_STATEMENT = - "UPDATE %s SET %s = null WHERE (\"" + KEY_NAME + "\" >= %d AND \"" + KEY_NAME + "\" < %d);"; -} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java b/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java deleted file mode 100644 index 3c3bb749c6..0000000000 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagFilterUtils.java +++ /dev/null @@ -1,53 +0,0 @@ -package cn.edu.tsinghua.iginx.postgresql.tools; - -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; - -public class TagFilterUtils { - @SuppressWarnings("unused") - public static String transformToFilterStr(TagFilter filter) { - StringBuilder builder = new StringBuilder(); - transformToFilterStr(filter, builder); - return builder.toString(); - } - - private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { - switch (filter.getType()) { - case And: - AndTagFilter andFilter = (AndTagFilter) filter; - for (int i = 0; i < andFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(andFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" and "); - } - } - break; - case Or: - OrTagFilter orFilter = (OrTagFilter) filter; - for (int i = 0; i < orFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(orFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" or "); - } - } - break; - case Base: - BaseTagFilter baseFilter = (BaseTagFilter) filter; - builder.append(baseFilter.getTagKey()); - builder.append("="); - builder.append(baseFilter.getTagValue()); - break; - // TODO: case label - case BasePrecise: - case Precise: - case WithoutTag: - break; - } - } -} diff --git a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java b/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java index c1aba09ec2..831a7b9c1c 100644 --- a/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java +++ b/dataSources/redis/src/main/java/cn/edu/tsinghua/iginx/redis/RedisStorage.java @@ -537,7 +537,7 @@ public Pair getBoundaryOfStorage(String prefix) columnsInterval = new ColumnsInterval(null, null); } } - KeyInterval keyInterval = new KeyInterval(Long.MIN_VALUE, Long.MAX_VALUE); + KeyInterval keyInterval = KeyInterval.getDefaultKeyInterval(); return new Pair<>(columnsInterval, keyInterval); } diff --git a/dataSources/relational/.gitignore b/dataSources/relational/.gitignore new file mode 100644 index 0000000000..5ff6309b71 --- /dev/null +++ b/dataSources/relational/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/dataSources/postgresql/pom.xml b/dataSources/relational/pom.xml similarity index 77% rename from dataSources/postgresql/pom.xml rename to dataSources/relational/pom.xml index 235bf436a8..7d4b4368c3 100644 --- a/dataSources/postgresql/pom.xml +++ b/dataSources/relational/pom.xml @@ -9,21 +9,37 @@ ${revision} - postgresql - IGinX PostgreSQL + relational + IGinX relational .. - postgresql + relational 8 8 - org.postgresql postgresql + + com.zaxxer + HikariCP + 2.4.7 + compile + + + org.slf4j + slf4j-api + + + + + mysql + mysql-connector-java + 8.0.26 + @@ -44,7 +60,7 @@ package - ../../core/target/iginx-core-${project.version}/driver/postgresql/ + ../../core/target/iginx-core-${project.version}/driver/relational/ provided @@ -63,7 +79,7 @@ package - + diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java similarity index 65% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java index 33e451e5ec..484814477d 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/PostgreSQLStorage.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/RelationalStorage.java @@ -16,13 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package cn.edu.tsinghua.iginx.postgresql; +package cn.edu.tsinghua.iginx.relational; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.SEPARATOR; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; -import static cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer.fromPostgreSQL; -import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; -import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.toFullName; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.*; +import static cn.edu.tsinghua.iginx.relational.tools.TagKVUtils.splitFullName; +import static cn.edu.tsinghua.iginx.relational.tools.TagKVUtils.toFullName; import cn.edu.tsinghua.iginx.engine.logical.utils.LogicalFilterUtils; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; @@ -48,115 +47,262 @@ import cn.edu.tsinghua.iginx.metadata.entity.ColumnsInterval; import cn.edu.tsinghua.iginx.metadata.entity.KeyInterval; import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; -import cn.edu.tsinghua.iginx.postgresql.exception.PostgreSQLException; -import cn.edu.tsinghua.iginx.postgresql.exception.PostgresqlTaskExecuteFailureException; -import cn.edu.tsinghua.iginx.postgresql.query.entity.PostgreSQLQueryRowStream; -import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; -import cn.edu.tsinghua.iginx.postgresql.tools.FilterTransformer; -import cn.edu.tsinghua.iginx.postgresql.tools.PostgreSQLSchema; +import cn.edu.tsinghua.iginx.relational.exception.RelationalException; +import cn.edu.tsinghua.iginx.relational.exception.RelationalTaskExecuteFailureException; +import cn.edu.tsinghua.iginx.relational.meta.AbstractRelationalMeta; +import cn.edu.tsinghua.iginx.relational.meta.JDBCMeta; +import cn.edu.tsinghua.iginx.relational.query.entity.RelationQueryRowStream; +import cn.edu.tsinghua.iginx.relational.tools.ColumnField; +import cn.edu.tsinghua.iginx.relational.tools.FilterTransformer; +import cn.edu.tsinghua.iginx.relational.tools.RelationSchema; import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import cn.edu.tsinghua.iginx.utils.Pair; import cn.edu.tsinghua.iginx.utils.StringUtils; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.sql.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.postgresql.ds.PGConnectionPoolDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgreSQLStorage implements IStorage { +public class RelationalStorage implements IStorage { - private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLStorage.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RelationalStorage.class); private final StorageEngineMeta meta; - private final Map connectionPoolMap = - new ConcurrentHashMap<>(); - private final Connection connection; - public PostgreSQLStorage(StorageEngineMeta meta) throws StorageInitializationException { + private AbstractRelationalMeta relationalMeta; + + private final String engineName; + + private final Map connectionPoolMap = new ConcurrentHashMap<>(); + + private final FilterTransformer filterTransformer; + + private Connection getConnection(String databaseName) { + if (databaseName.startsWith("dummy")) { + return null; + } + + if (relationalMeta.getSystemDatabaseName().stream().anyMatch(databaseName::equalsIgnoreCase)) { + return null; + } + + try { + Statement stmt = connection.createStatement(); + stmt.execute(String.format(CREATE_DATABASE_STATEMENT, databaseName)); + stmt.close(); + } catch (SQLException ignored) { + } + + HikariDataSource dataSource = connectionPoolMap.get(databaseName); + if (dataSource != null) { + try { + return dataSource.getConnection(); + } catch (SQLException e) { + LOGGER.error("Cannot get connection for database {}", databaseName, e); + dataSource.close(); + } + } + + try { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(getUrl(databaseName, meta)); + config.setUsername(meta.getExtraParams().get(USERNAME)); + config.setPassword(meta.getExtraParams().get(PASSWORD)); + config.addDataSourceProperty("prepStmtCacheSize", "250"); + config.setLeakDetectionThreshold(2500); + config.setConnectionTimeout(30000); + config.setIdleTimeout(10000); + config.setMaximumPoolSize(10); + config.setMinimumIdle(1); + config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048"); + + HikariDataSource newDataSource = new HikariDataSource(config); + connectionPoolMap.put(databaseName, newDataSource); + return newDataSource.getConnection(); + } catch (SQLException e) { + LOGGER.error("Cannot get connection for database {}", databaseName, e); + return null; + } + } + + private void closeConnection(String databaseName) { + HikariDataSource dataSource = connectionPoolMap.get(databaseName); + if (dataSource != null) { + dataSource.close(); + connectionPoolMap.remove(databaseName); + } + } + + protected String getUrl(String databaseName, StorageEngineMeta meta) { + Map extraParams = meta.getExtraParams(); + String engine = extraParams.get("engine"); + return String.format("jdbc:%s://%s:%s/%s", engine, meta.getIp(), meta.getPort(), databaseName); + } + + public RelationalStorage(StorageEngineMeta meta) throws StorageInitializationException { this.meta = meta; - if (!meta.getStorageEngine().equals(StorageEngineType.postgresql)) { - throw new StorageInitializationException("unexpected database: " + meta.getStorageEngine()); + try { + buildRelationalMeta(); + } catch (RelationalTaskExecuteFailureException e) { + throw new StorageInitializationException("cannot build relational meta: ", e); + } + if (!testConnection()) { + throw new StorageInitializationException("cannot connect to " + meta.toString()); } - testConnection(); + filterTransformer = new FilterTransformer(relationalMeta); Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String username = extraParams.get(USERNAME); + String password = extraParams.get(PASSWORD); + engineName = extraParams.get("engine"); String connUrl = - String.format( - "jdbc:postgresql://%s:%s/?user=%s&password=%s", - meta.getIp(), meta.getPort(), username, password); + password == null + ? String.format( + "jdbc:%s://%s:%s/?user=%s", engineName, meta.getIp(), meta.getPort(), username) + : String.format( + "jdbc:%s://%s:%s/?user=%s&password=%s", + engineName, meta.getIp(), meta.getPort(), username, password); try { connection = DriverManager.getConnection(connUrl); Statement statement = connection.createStatement(); - String sql = "alter system set idle_in_transaction_session_timeout='1min'"; - statement.executeUpdate(sql); statement.close(); } catch (SQLException e) { - throw new StorageInitializationException(String.format("cannot connect to %s", meta), e); + throw new StorageInitializationException(String.format("cannot connect to %s :", meta), e); } } - private void testConnection() throws StorageInitializationException { + private void buildRelationalMeta() throws RelationalTaskExecuteFailureException { + String engineName = meta.getExtraParams().get("engine"); + if (classMap.containsKey(engineName)) { + try { + Class clazz = Class.forName(classMap.get(engineName)); + relationalMeta = + (AbstractRelationalMeta) + clazz.getConstructor(StorageEngineMeta.class).newInstance(meta); + } catch (Exception e) { + throw new RelationalTaskExecuteFailureException( + String.format("engine %s is not supported", engineName), e); + } + } else { + String propertiesPath = meta.getExtraParams().get("meta_properties_path"); + try { + relationalMeta = new JDBCMeta(meta, propertiesPath); + } catch (IOException e) { + throw new RelationalTaskExecuteFailureException( + String.format("engine %s is not supported", engineName), e); + } + } + } + + private boolean testConnection() { Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); + String username = extraParams.get(USERNAME); + String password = extraParams.get(PASSWORD); + String engine = meta.getExtraParams().get("engine"); String connUrl = - String.format( - "jdbc:postgresql://%s:%s/?user=%s&password=%s", - meta.getIp(), meta.getPort(), username, password); + password == null + ? String.format( + "jdbc:%s://%s:%s/?user=%s", engine, meta.getIp(), meta.getPort(), username) + : String.format( + "jdbc:%s://%s:%s/?user=%s&password=%s", + engine, meta.getIp(), meta.getPort(), username, password); + try { - Class.forName("org.postgresql.Driver"); + Class.forName(relationalMeta.getDriverClass()); DriverManager.getConnection(connUrl); + return true; } catch (SQLException | ClassNotFoundException e) { - throw new StorageInitializationException(String.format("cannot connect to %s", meta), e); + return false; } } - private String getUrl(String databaseName) { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = - String.format( - "jdbc:postgresql://%s:%s/%s?user=%s&password=%s", - meta.getIp(), meta.getPort(), databaseName, username, password); - return connUrl; + /** + * 通过JDBC获取ENGINE的所有数据库名称 + * + * @return 数据库名称列表 + */ + private List getDatabaseNames() throws SQLException { + List databaseNames = new ArrayList<>(); + Connection conn = getConnection(relationalMeta.getDefaultDatabaseName()); + ResultSet rs = conn.createStatement().executeQuery(relationalMeta.getDatabaseQuerySql()); + while (rs.next()) { + String databaseName = rs.getString("DATNAME"); + if (relationalMeta.getSystemDatabaseName().contains(databaseName) + || relationalMeta.getDefaultDatabaseName().equals(databaseName)) { + continue; + } + databaseNames.add(databaseName); + } + rs.close(); + conn.close(); + return databaseNames; } - private Connection getConnection(String databaseName) { - if (databaseName.startsWith("dummy")) { - return null; - } - if (databaseName.equalsIgnoreCase("template0") || databaseName.equalsIgnoreCase("template1")) { - return null; + private List getTables(String databaseName, String tablePattern) { + if (relationalMeta.jdbcNeedQuote()) { + tablePattern = relationalMeta.getQuote() + tablePattern + relationalMeta.getQuote(); } - try { - Statement stmt = connection.createStatement(); - stmt.execute(String.format(CREATE_DATABASE_STATEMENT, databaseName)); - stmt.close(); - } catch (SQLException ignored) { - // LOGGER.info("database {} exists!", databaseName); + Connection conn = getConnection(databaseName); + if (conn == null) { + throw new RelationalTaskExecuteFailureException( + "cannot connect to database " + databaseName); + } + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet rs = + databaseMetaData.getTables( + databaseName, + relationalMeta.getSchemaPattern(), + tablePattern, + new String[] {"TABLE"}); + List tableNames = new ArrayList<>(); + + while (rs.next()) { + tableNames.add(rs.getString("TABLE_NAME")); + } + + rs.close(); + conn.close(); + return tableNames; + } catch (SQLException | RelationalTaskExecuteFailureException e) { + LOGGER.error("unexpected error: ", e); + return new ArrayList<>(); } + } + private List getColumns( + String databaseName, String tableName, String columnNamePattern) { try { - if (connectionPoolMap.containsKey(databaseName)) { - return connectionPoolMap.get(databaseName).getConnection(); + Connection conn = getConnection(databaseName); + if (conn == null) { + throw new RelationalTaskExecuteFailureException( + "cannot connect to database " + databaseName); } - PGConnectionPoolDataSource connectionPool = new PGConnectionPoolDataSource(); - connectionPool.setUrl(getUrl(databaseName)); - connectionPoolMap.put(databaseName, connectionPool); - return connectionPool.getConnection(); - } catch (SQLException e) { - LOGGER.error("cannot get connection for database {}: ", databaseName, e); - return null; + DatabaseMetaData databaseMetaData = conn.getMetaData(); + ResultSet rs = + databaseMetaData.getColumns( + databaseName, relationalMeta.getSchemaPattern(), tableName, columnNamePattern); + List columnFields = new ArrayList<>(); + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String columnType = rs.getString("TYPE_NAME"); + String columnTable = rs.getString("TABLE_NAME"); + columnFields.add(new ColumnField(columnTable, columnName, columnType)); + } + rs.close(); + conn.close(); + return columnFields; + } catch (SQLException | RelationalTaskExecuteFailureException e) { + LOGGER.error("unexpected error: ", e); + return new ArrayList<>(); } } @@ -189,63 +335,44 @@ private boolean filterContainsType(List types, Filter filter) { } @Override - public List getColumns() throws PostgreSQLException { + public List getColumns() throws RelationalTaskExecuteFailureException { List columns = new ArrayList<>(); Map extraParams = meta.getExtraParams(); - try (Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT)) { - while (databaseSet.next()) { - String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 + try { + for (String databaseName : getDatabaseNames()) { if ((extraParams.get("has_data") == null || extraParams.get("has_data").equals("false")) && !databaseName.startsWith(DATABASE_PREFIX)) { continue; } - Connection connection = getConnection(databaseName); - if (connection == null) { - continue; - } - try (Connection conn = connection; - ResultSet tableSet = - conn.getMetaData().getTables(databaseName, "public", "%", new String[] {"TABLE"})) { - while (tableSet.next()) { - String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 - try (ResultSet columnSet = - conn.getMetaData().getColumns(databaseName, "public", tableName, "%")) { - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 - String typeName = columnSet.getString("TYPE_NAME"); // 列字段类型 - if (columnName.equals(KEY_NAME)) { // key 列不显示 - continue; - } - Pair> nameAndTags = splitFullName(columnName); - if (databaseName.startsWith(DATABASE_PREFIX)) { - columns.add( - new Column( - tableName + SEPARATOR + nameAndTags.k, - fromPostgreSQL(typeName), - nameAndTags.v)); - } else { - columns.add( - new Column( - databaseName + SEPARATOR + tableName + SEPARATOR + nameAndTags.k, - fromPostgreSQL(typeName), - nameAndTags.v)); - } - } - } catch (SQLException e) { - throw new PostgreSQLException( - String.format( - "failed to get columns for table: %s in %s ", tableName, databaseName), - e); + + List tables = getTables(databaseName, "%"); + for (String tableName : tables) { + List columnFieldList = getColumns(databaseName, tableName, "%"); + for (ColumnField columnField : columnFieldList) { + String columnName = columnField.columnName; + String typeName = columnField.columnType; + if (columnName.equals(KEY_NAME)) { // key 列不显示 + continue; + } + Pair> nameAndTags = splitFullName(columnName); + if (databaseName.startsWith(DATABASE_PREFIX)) { + columns.add( + new Column( + tableName + SEPARATOR + nameAndTags.k, + relationalMeta.getDataTypeTransformer().fromEngineType(typeName), + nameAndTags.v)); + } else { + columns.add( + new Column( + databaseName + SEPARATOR + tableName + SEPARATOR + nameAndTags.k, + relationalMeta.getDataTypeTransformer().fromEngineType(typeName), + nameAndTags.v)); } } - } catch (SQLException e) { - throw new PostgreSQLException( - String.format("failed to get tables of database: %s", databaseName), e); } } } catch (SQLException e) { - throw new PostgreSQLException(String.format("failed to get databases of ", meta), e); + throw new RelationalTaskExecuteFailureException("failed to get columns ", e); } return columns; } @@ -285,22 +412,21 @@ public TaskExecuteResult executeProjectWithSelect( private TaskExecuteResult executeProjectWithFilter( Project project, Filter filter, DataArea dataArea) { + try { + String databaseName = dataArea.getStorageUnit(); + Connection conn = getConnection(databaseName); + if (conn == null) { + return new TaskExecuteResult( + new RelationalTaskExecuteFailureException( + String.format("cannot connect to database %s", databaseName))); + } - String databaseName = dataArea.getStorageUnit(); - Connection connection = getConnection(databaseName); - if (connection == null) { - return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( - String.format("cannot connect to database %s", databaseName))); - } - - try (Connection conn = connection) { List databaseNameList = new ArrayList<>(); List resultSets = new ArrayList<>(); - Statement stmt = null; + Statement stmt; Map tableNameToColumnNames = - splitAndMergeQueryPatterns(databaseName, conn, project.getPatterns()); + splitAndMergeQueryPatterns(databaseName, project.getPatterns()); String statement; // 如果table>1的情况下存在Value或Path Filter，说明filter的匹配需要跨table，此时需要将所有table join到一起进行查询 @@ -310,10 +436,10 @@ && filterContainsType(Arrays.asList(FilterType.Value, FilterType.Path), filter)) for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); String quotColumnNames = getQuotColumnNames(entry.getValue()); - String filterStr = FilterTransformer.toString(filter); + String filterStr = filterTransformer.toString(filter); statement = String.format( - QUERY_STATEMENT, + relationalMeta.getQueryStatement(), quotColumnNames, getQuotName(tableName), filterStr.isEmpty() ? "" : "WHERE " + filterStr); @@ -344,42 +470,25 @@ else if (!tableNameToColumnNames.isEmpty()) { new ArrayList<>(Arrays.asList(entry.getValue().split(", "))); // 将columnNames中的列名加上tableName前缀 - fullColumnNames.replaceAll(s -> PostgreSQLSchema.getQuotFullName(tableName, s)); + fullColumnNames.replaceAll( + s -> RelationSchema.getQuotFullName(tableName, s, relationalMeta.getQuote())); fullColumnNamesList.add(fullColumnNames); } StringBuilder fullColumnNames = new StringBuilder(); - fullColumnNames.append(PostgreSQLSchema.getQuotFullName(tableNames.get(0), KEY_NAME)); + fullColumnNames.append( + RelationSchema.getQuotFullName(tableNames.get(0), KEY_NAME, relationalMeta.getQuote())); for (List columnNames : fullColumnNamesList) { for (String columnName : columnNames) { fullColumnNames.append(", ").append(columnName); } } - // table之间用FULL OUTER JOIN ON table1.⺅= table2.⺅ 连接，超过2个table的情况下，需要多次嵌套join - StringBuilder fullTableName = new StringBuilder(); - fullTableName.append(getQuotName(tableNames.get(0))); - for (int i = 1; i < tableNames.size(); i++) { - fullTableName.insert(0, "("); - fullTableName - .append(" FULL OUTER JOIN ") - .append(getQuotName(tableNames.get(i))) - .append(" ON "); - for (int j = 0; j < i; j++) { - fullTableName - .append(PostgreSQLSchema.getQuotFullName(tableNames.get(i), KEY_NAME)) - .append(" = ") - .append(PostgreSQLSchema.getQuotFullName(tableNames.get(j), KEY_NAME)); - - if (j != i - 1) { - fullTableName.append(" AND "); - } - } - fullTableName.append(")"); - } + // 将所有表进行full join + String fullTableName = getFullJoinTables(tableNames, fullColumnNamesList); // 对通配符做处理，将通配符替换成对应的列名 - if (FilterTransformer.toString(filter).contains("*")) { + if (filterTransformer.toString(filter).contains("*")) { // 把fullColumnNamesList中的列名全部用removeFullColumnNameQuote去掉引号 fullColumnNamesList.replaceAll( columnNames -> { @@ -393,14 +502,26 @@ else if (!tableNameToColumnNames.isEmpty()) { filter = LogicalFilterUtils.mergeTrue(filter); } - String filterStr = FilterTransformer.toString(filter); + String fullColumnNamesStr = fullColumnNames.toString(); + String filterStr = filterTransformer.toString(filter); + String orderByKey = + RelationSchema.getQuotFullName(tableNames.get(0), KEY_NAME, relationalMeta.getQuote()); + if (!relationalMeta.isSupportFullJoin()) { + // 如果不支持full join,需要为left join + union模拟的full join表起别名，同时select、where、order by的部分都要调整 + fullColumnNamesStr = fullColumnNamesStr.replaceAll("`\\.`", "."); + filterStr = filterStr.replaceAll("`\\.`", "."); + filterStr = + filterStr.replace( + getQuotName(KEY_NAME), getQuotName(tableNames.get(0) + SEPARATOR + KEY_NAME)); + orderByKey = orderByKey.replaceAll("`\\.`", "."); + } statement = String.format( QUERY_STATEMENT_WITHOUT_KEYNAME, - fullColumnNames, + fullColumnNamesStr, fullTableName, filterStr.isEmpty() ? "" : "WHERE " + filterStr, - PostgreSQLSchema.getQuotFullName(tableNames.get(0), KEY_NAME)); + orderByKey); ResultSet rs = null; try { @@ -418,19 +539,166 @@ else if (!tableNameToColumnNames.isEmpty()) { RowStream rowStream = new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream( + new RelationQueryRowStream( databaseNameList, resultSets, false, filter, project.getTagFilter(), - Collections.singletonList(conn))); + Collections.singletonList(conn), + relationalMeta)); return new TaskExecuteResult(rowStream); } catch (SQLException e) { + LOGGER.error("unexpected error: ", e); return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( - "execute project task in postgresql failure", e)); + new RelationalTaskExecuteFailureException( + String.format("execute project task in %s failure", engineName), e)); + } + } + + private String getFullJoinTables(List tableNames, List> fullColumnList) { + StringBuilder fullTableName = new StringBuilder(); + if (relationalMeta.isSupportFullJoin()) { + // 支持全连接，就直接用全连接连接各个表 + fullTableName.append(getQuotName(tableNames.get(0))); + for (int i = 1; i < tableNames.size(); i++) { + fullTableName.insert(0, "("); + fullTableName + .append(" FULL OUTER JOIN ") + .append(getQuotName(tableNames.get(i))) + .append(" ON "); + for (int j = 0; j < i; j++) { + fullTableName + .append( + RelationSchema.getQuotFullName( + tableNames.get(i), KEY_NAME, relationalMeta.getQuote())) + .append(" = ") + .append( + RelationSchema.getQuotFullName( + tableNames.get(j), KEY_NAME, relationalMeta.getQuote())); + + if (j != i - 1) { + fullTableName.append(" AND "); + } + } + fullTableName.append(")"); + } + } else { + // 不支持全连接，就要用Left Join+Union来模拟全连接 + StringBuilder allColumns = new StringBuilder(); + fullColumnList.forEach( + columnList -> + columnList.forEach( + column -> + allColumns.append( + String.format("%s AS %s, ", column, column.replaceAll("`\\.`", "."))))); + allColumns.delete(allColumns.length() - 2, allColumns.length()); + + fullTableName.append("("); + for (int i = 0; i < tableNames.size(); i++) { + + String keyStr = + String.format( + "%s.%s AS %s", + getQuotName(tableNames.get(i)), + getQuotName(KEY_NAME), + getQuotName(tableNames.get(i) + SEPARATOR + KEY_NAME)); + fullTableName.append( + String.format( + "SELECT %s FROM %s", keyStr + ", " + allColumns, getQuotName(tableNames.get(i)))); + for (int j = 0; j < tableNames.size(); j++) { + if (i != j) { + fullTableName.append( + String.format( + " LEFT JOIN %s ON %s.%s = %s.%s", + getQuotName(tableNames.get(j)), + getQuotName(tableNames.get(i)), + getQuotName(KEY_NAME), + getQuotName(tableNames.get(j)), + getQuotName(KEY_NAME))); + } + } + if (i != tableNames.size() - 1) { + fullTableName.append(" UNION "); + } + } + fullTableName.append(")"); + + fullTableName.append(" AS derived "); } + + return fullTableName.toString(); + } + + private String getDummyFullJoinTables( + List tableNames, + Map allColumnNameForTable, + List> fullColumnList) { + StringBuilder fullTableName = new StringBuilder(); + if (relationalMeta.isSupportFullJoin()) { + // table之间用FULL OUTER JOIN ON concat(table1所有列) = concat(table2所有列) + // 连接，超过2个table的情况下，需要多次嵌套join + fullTableName.append(getQuotName(tableNames.get(0))); + for (int i = 1; i < tableNames.size(); i++) { + fullTableName.insert(0, "("); + fullTableName + .append(" FULL OUTER JOIN ") + .append(getQuotName(tableNames.get(i))) + .append(" ON "); + for (int j = 0; j < i; j++) { + fullTableName + .append("CONCAT(") + .append(allColumnNameForTable.get(tableNames.get(i))) + .append(")") + .append(" = ") + .append("CONCAT(") + .append(allColumnNameForTable.get(tableNames.get(j))) + .append(")"); + if (j != i - 1) { + fullTableName.append(" AND "); + } + } + fullTableName.append(")"); + } + } else { + // 不支持全连接，就要用Left Join+Union来模拟全连接 + StringBuilder allColumns = new StringBuilder(); + fullColumnList.forEach( + columnList -> + columnList.forEach( + column -> + allColumns.append( + String.format("%s AS %s, ", column, column.replaceAll("`\\.`", "."))))); + allColumns.delete(allColumns.length() - 2, allColumns.length()); + + fullTableName.append("("); + for (int i = 0; i < tableNames.size(); i++) { + String keyStr = + String.format( + "%s AS %s", + getQuotName(KEY_NAME), getQuotName(tableNames.get(i) + SEPARATOR + KEY_NAME)); + fullTableName.append( + String.format( + "SELECT %s FROM %s", keyStr + ", " + allColumns, getQuotName(tableNames.get(i)))); + for (int j = 0; j < tableNames.size(); j++) { + if (i != j) { + fullTableName.append( + String.format( + " LEFT JOIN %s ON CONCAT(%s) = CONCAT(%s)", + getQuotName(tableNames.get(j)), + allColumnNameForTable.get(tableNames.get(i)), + allColumnNameForTable.get(tableNames.get(j)))); + } + } + if (i != tableNames.size() - 1) { + fullTableName.append(" UNION "); + } + } + fullTableName.append(")"); + fullTableName.append(" AS derived "); + } + + return fullTableName.toString(); } private Filter generateWildCardsFilter(Filter filter, List> columnNamesList) { @@ -561,7 +829,9 @@ private List getMatchedPath(String path, List> columnNamesL } private String removeFullColumnNameQuote(String fullColumnName) { - return fullColumnName.substring(1, fullColumnName.length() - 1).replace("\".\"", "."); + return fullColumnName + .substring(1, fullColumnName.length() - 1) + .replace(relationalMeta.getQuote() + "." + relationalMeta.getQuote(), "."); } private Filter cutFilterDatabaseNameForDummy(Filter filter, String databaseName) { @@ -614,20 +884,17 @@ private Map getAllColumnNameForTable( for (Map.Entry entry : tableNameToColumnNames.entrySet()) { String tableName = entry.getKey(); String columnNames = ""; - Connection conn = getConnection(databaseName); - if (conn == null) { - continue; - } - ResultSet rs = conn.getMetaData().getColumns(databaseName, "public", tableName, "%"); - while (rs.next()) { - tableName = rs.getString("TABLE_NAME"); - columnNames = PostgreSQLSchema.getQuotFullName(tableName, rs.getString("COLUMN_NAME")); + + List columnFieldList = getColumns(databaseName, tableName, "%"); + for (ColumnField columnField : columnFieldList) { + String columnName = columnField.columnName; + columnNames = + RelationSchema.getQuotFullName(tableName, columnName, relationalMeta.getQuote()); if (allColumnNameForTable.containsKey(tableName)) { columnNames = allColumnNameForTable.get(tableName) + ", " + columnNames; } allColumnNameForTable.put(tableName, columnNames); } - conn.close(); } return allColumnNameForTable; } @@ -669,19 +936,20 @@ && filterContainsType(Arrays.asList(FilterType.Value, FilterType.Path), filter)) String tableName = entry.getKey(); String fullQuotColumnNames = getQuotColumnNames(entry.getValue()); List fullPathList = Arrays.asList(entry.getValue().split(", ")); - fullPathList.replaceAll(s -> PostgreSQLSchema.getQuotFullName(tableName, s)); + fullPathList.replaceAll( + s -> RelationSchema.getQuotFullName(tableName, s, relationalMeta.getQuote())); String filterStr = - FilterTransformer.toString( + filterTransformer.toString( dummyFilterSetTrueByColumnNames( cutFilterDatabaseNameForDummy(filter.copy(), databaseName), fullPathList)); statement = String.format( - CONCAT_QUERY_STATEMENT_AND_CONCAT_KEY, + relationalMeta.getConcatQueryStatement(), allColumnNameForTable.get(tableName), fullQuotColumnNames, getQuotName(tableName), filterStr.isEmpty() ? "" : "WHERE " + filterStr, - allColumnNameForTable.get(tableName)); + "Concat(" + allColumnNameForTable.get(tableName) + ")"); try { stmt = conn.createStatement(); @@ -706,12 +974,13 @@ else if (!tableNameToColumnNames.isEmpty()) { String tableName = entry.getKey(); tableNames.add(tableName); List columnNames = new ArrayList<>(Arrays.asList(entry.getValue().split(", "))); - columnNames.replaceAll(s -> PostgreSQLSchema.getFullName(tableName, s)); + columnNames.replaceAll(s -> RelationSchema.getFullName(tableName, s)); fullColumnNamesList.add(columnNames); List fullColumnNames = new ArrayList<>(Arrays.asList(entry.getValue().split(", "))); - fullColumnNames.replaceAll(s -> PostgreSQLSchema.getQuotFullName(tableName, s)); + fullColumnNames.replaceAll( + s -> RelationSchema.getQuotFullName(tableName, s, relationalMeta.getQuote())); fullQuotColumnNamesList.add(fullColumnNames); } @@ -732,38 +1001,22 @@ else if (!tableNameToColumnNames.isEmpty()) { } allColumnNames.delete(allColumnNames.length() - 2, allColumnNames.length()); - // table之间用FULL OUTER JOIN ON concat(table1所有列) = concat(table2所有列) - // 连接，超过2个table的情况下，需要多次嵌套join - StringBuilder fullTableName = new StringBuilder(); - fullTableName.append(tableNames.get(0)); - for (int i = 1; i < tableNames.size(); i++) { - fullTableName.insert(0, "("); - fullTableName.append(" FULL OUTER JOIN ").append(tableNames.get(i)).append(" ON "); - for (int j = 0; j < i; j++) { - fullTableName - .append("CONCAT(") - .append(allColumnNameForTable.get(tableNames.get(i))) - .append(")") - .append(" = ") - .append("CONCAT(") - .append(allColumnNameForTable.get(tableNames.get(j))) - .append(")"); - if (j != i - 1) { - fullTableName.append(" AND "); - } - } - fullTableName.append(")"); - } + String fullTableName = + getDummyFullJoinTables(tableNames, allColumnNameForTable, fullColumnNamesList); Filter copyFilter = cutFilterDatabaseNameForDummy(filter.copy(), databaseName); copyFilter = dummyFilterSetTrueByColumnNames( copyFilter, - Arrays.asList(fullColumnNames.toString().replace("\"", "").split(", "))); + Arrays.asList( + fullColumnNames + .toString() + .replace(String.valueOf(relationalMeta.getQuote()), "") + .split(", "))); // 对通配符做处理，将通配符替换成对应的列名 - if (FilterTransformer.toString(copyFilter).contains("*")) { + if (filterTransformer.toString(copyFilter).contains("*")) { // 把fullColumnNamesList中的列名全部用removeFullColumnNameQuote去掉引号 fullColumnNamesList.replaceAll( columnNames -> { @@ -777,15 +1030,27 @@ else if (!tableNameToColumnNames.isEmpty()) { copyFilter = LogicalFilterUtils.mergeTrue(copyFilter); } - String filterStr = FilterTransformer.toString(copyFilter); + String fullColumnNamesStr = fullColumnNames.toString(); + String filterStr = filterTransformer.toString(copyFilter); + String orderByKey = String.format("Concat(%s)", allColumnNames); + if (!relationalMeta.isSupportFullJoin()) { + // 如果不支持full join,需要为left join + union模拟的full join表起别名，同时select、where、order by的部分都要调整 + fullColumnNamesStr = fullColumnNamesStr.replaceAll("`\\.`", "."); + filterStr = filterStr.replaceAll("`\\.`", "."); + filterStr = + filterStr.replace( + getQuotName(KEY_NAME), getQuotName(tableNames.get(0) + SEPARATOR + KEY_NAME)); + orderByKey = getQuotName(tableNames.get(0) + SEPARATOR + KEY_NAME); + } + statement = String.format( - CONCAT_QUERY_STATEMENT_AND_CONCAT_KEY, + relationalMeta.getConcatQueryStatement(), allColumnNames, - fullColumnNames, + fullColumnNamesStr, fullTableName, filterStr.isEmpty() ? "" : "WHERE " + filterStr, - allColumnNames); + orderByKey); try { stmt = conn.createStatement(); @@ -803,13 +1068,20 @@ else if (!tableNameToColumnNames.isEmpty()) { RowStream rowStream = new ClearEmptyRowStreamWrapper( - new PostgreSQLQueryRowStream( - databaseNameList, resultSets, true, filter, project.getTagFilter(), connList)); + new RelationQueryRowStream( + databaseNameList, + resultSets, + true, + filter, + project.getTagFilter(), + connList, + relationalMeta)); return new TaskExecuteResult(rowStream); } catch (SQLException e) { + LOGGER.error("unexpected error: ", e); return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( - "execute project task in postgresql failure", e)); + new RelationalTaskExecuteFailureException( + String.format("execute project task in %s failure", engineName), e)); } } @@ -855,7 +1127,7 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { Connection conn = getConnection(databaseName); if (conn == null) { return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( + new RelationalTaskExecuteFailureException( String.format("cannot connect to database %s", databaseName))); } @@ -865,42 +1137,44 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { List> deletedPaths; // table name -> column name String tableName; String columnName; - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = null; - ResultSet columnSet = null; + List tables; if (delete.getKeyRanges() == null || delete.getKeyRanges().isEmpty()) { if (paths.size() == 1 && paths.get(0).equals("*") && delete.getTagFilter() == null) { - conn.close(); - Connection postgresConn = - getConnection("postgres"); // 正在使用的数据库无法被删除，因此需要切换到名为 postgres 的默认数据库 - if (postgresConn != null) { - stmt = postgresConn.createStatement(); - statement = String.format(DROP_DATABASE_STATEMENT, databaseName); + closeConnection(databaseName); + Connection defaultConn = + getConnection(relationalMeta.getDefaultDatabaseName()); // 正在使用的数据库无法被删除，因此需要切换到默认数据库 + if (defaultConn != null) { + stmt = defaultConn.createStatement(); + statement = String.format(relationalMeta.getDropDatabaseStatement(), databaseName); LOGGER.info("[Delete] execute delete: {}", statement); stmt.execute(statement); // 删除数据库 stmt.close(); - postgresConn.close(); + defaultConn.close(); return new TaskExecuteResult(null, null); } else { return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( - "cannot connect to database postgres", new SQLException())); + new RelationalTaskExecuteFailureException( + String.format( + "cannot connect to database %s", relationalMeta.getDefaultDatabaseName()), + new SQLException())); } } else { deletedPaths = determineDeletedPaths(paths, delete.getTagFilter()); for (Pair pair : deletedPaths) { tableName = pair.k; columnName = pair.v; - tableSet = - databaseMetaData.getTables( - databaseName, "public", tableName, new String[] {"TABLE"}); - if (tableSet.next()) { + tables = getTables(databaseName, tableName); + if (!tables.isEmpty()) { statement = String.format( DROP_COLUMN_STATEMENT, getQuotName(tableName), getQuotName(columnName)); LOGGER.info("[Delete] execute delete: {}", statement); - stmt.execute(statement); // 删除列 + try { + stmt.execute(statement); // 删除列 + } catch (SQLException e) { + // 可能会出现该列不存在的问题，此时不做处理 + } } } } @@ -909,12 +1183,11 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { for (Pair pair : deletedPaths) { tableName = pair.k; columnName = pair.v; - columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, columnName); - if (columnSet.next()) { + if (!getColumns(databaseName, tableName, columnName).isEmpty()) { for (KeyRange keyRange : delete.getKeyRanges()) { statement = String.format( - UPDATE_STATEMENT, + relationalMeta.getUpdateStatement(), getQuotName(tableName), getQuotName(columnName), keyRange.getBeginKey(), @@ -925,19 +1198,14 @@ public TaskExecuteResult executeDelete(Delete delete, DataArea dataArea) { } } } - if (tableSet != null) { - tableSet.close(); - } - if (columnSet != null) { - columnSet.close(); - } stmt.close(); conn.close(); return new TaskExecuteResult(null, null); - } catch (SQLException | PhysicalException e) { + } catch (SQLException e) { + LOGGER.error("unexpected error: ", e); return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( - "execute delete task in postgresql failure", e)); + new RelationalTaskExecuteFailureException( + String.format("execute delete task in %s failure", engineName), e)); } } @@ -948,7 +1216,7 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { Connection conn = getConnection(databaseName); if (conn == null) { return new TaskExecuteResult( - new PostgresqlTaskExecuteFailureException( + new RelationalTaskExecuteFailureException( String.format("cannot connect to database %s", databaseName))); } Exception e = null; @@ -969,36 +1237,25 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { } if (e != null) { return new TaskExecuteResult( - null, new PostgreSQLException("execute insert task in postgresql failure", e)); + null, + new RelationalException( + String.format("execute insert task in %s failure", engineName), e)); } return new TaskExecuteResult(null, null); } @Override public Pair getBoundaryOfStorage(String dataPrefix) - throws PostgreSQLException { + throws PhysicalException { ColumnsInterval columnsInterval; List paths = new ArrayList<>(); try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); - while (databaseSet.next()) { - String databaseName = databaseSet.getString("DATNAME"); // 获取数据库名称 - Connection conn = getConnection(databaseName); - if (conn == null) { - continue; - } - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = - databaseMetaData.getTables(databaseName, "public", "%", new String[] {"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString("TABLE_NAME"); // 获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(databaseName, "public", tableName, "%"); - StringBuilder columnNames = new StringBuilder(); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME"); // 获取列名称 - columnNames.append(columnName); - columnNames.append(", "); // c1, c2, c3, + for (String databaseName : getDatabaseNames()) { + List tables = getTables(databaseName, "%"); + for (String tableName : tables) { + List columnFieldList = getColumns(databaseName, tableName, "%"); + for (ColumnField columnField : columnFieldList) { + String columnName = columnField.columnName; // 获取列名称 String path = databaseName + SEPARATOR + tableName + SEPARATOR + columnName; if (dataPrefix != null && !path.startsWith(dataPrefix)) { @@ -1007,18 +1264,14 @@ public Pair getBoundaryOfStorage(String dataPrefix paths.add(path); } } - tableSet.close(); - conn.close(); } - databaseSet.close(); - stmt.close(); } catch (SQLException e) { LOGGER.error("encounter error when getting boundary of storage: ", e); } paths.sort(String::compareTo); if (paths.isEmpty()) { - throw new PostgreSQLException("no data!"); + throw new RelationalTaskExecuteFailureException("no data!"); } if (dataPrefix != null) { @@ -1028,7 +1281,7 @@ public Pair getBoundaryOfStorage(String dataPrefix new ColumnsInterval(paths.get(0), StringUtils.nextString(paths.get(paths.size() - 1))); } - return new Pair<>(columnsInterval, new KeyInterval(Long.MIN_VALUE, Long.MAX_VALUE)); + return new Pair<>(columnsInterval, KeyInterval.getDefaultKeyInterval()); } private List getRegexPatternByName( @@ -1063,8 +1316,8 @@ private List getRegexPatternByName( return Arrays.asList(tableNamePattern, columnNamePattern); } - private Map splitAndMergeQueryPatterns( - String databaseName, Connection conn, List patterns) throws SQLException { + private Map splitAndMergeQueryPatterns(String databaseName, List patterns) + throws SQLException { // table name -> column names // 1 -> n Map tableNameToColumnNames = new HashMap<>(); @@ -1080,7 +1333,7 @@ private Map splitAndMergeQueryPatterns( tableName = pattern; columnNames = "%"; } else { - PostgreSQLSchema schema = new PostgreSQLSchema(pattern); + RelationSchema schema = new RelationSchema(pattern, relationalMeta.getQuote()); tableName = schema.getTableName(); columnNames = schema.getColumnName(); boolean columnEqualsStar = columnNames.startsWith("*"); @@ -1100,20 +1353,21 @@ private Map splitAndMergeQueryPatterns( if (!columnNames.endsWith("%")) { columnNames += "%"; // 匹配 tagKV } - ResultSet rs = - conn.getMetaData() - .getColumns( - databaseName, - "public", - StringUtils.reformatPath(tableName), - StringUtils.reformatPath(columnNames)); + + List columnFieldList; + if (relationalMeta.jdbcSupportSpecialChar()) { + columnFieldList = + getColumns(databaseName, reformatForJDBC(tableName), reformatForJDBC(columnNames)); + } else { + columnFieldList = getColumns(databaseName, "%", "%"); + } List patternList = getRegexPatternByName(tableName, columnNames, false); Pattern tableNamePattern = patternList.get(0), columnNamePattern = patternList.get(1); - while (rs.next()) { - String curTableName = rs.getString("TABLE_NAME"); - String curColumnNames = rs.getString("COLUMN_NAME"); + for (ColumnField columnField : columnFieldList) { + String curTableName = columnField.tableName; + String curColumnNames = columnField.columnName; if (curColumnNames.equals(KEY_NAME)) { continue; } @@ -1140,12 +1394,16 @@ private Map splitAndMergeQueryPatterns( } tableNameToColumnNames.put(curTableName, curColumnNames); } - rs.close(); } return tableNameToColumnNames; } + /** JDBC中的路径中的 . 不需要转义 */ + private String reformatForJDBC(String path) { + return StringUtils.reformatPath(path).replace("\\.", "."); + } + private Map> splitAndMergeHistoryQueryPatterns(List patterns) throws SQLException { // > @@ -1170,7 +1428,7 @@ private Map> splitAndMergeHistoryQueryPatterns(List< tableName = "%"; columnNames = parts[1].equals("*") ? "%" : parts[1]; } else { - PostgreSQLSchema schema = new PostgreSQLSchema(pattern, true); + RelationSchema schema = new RelationSchema(pattern, true, relationalMeta.getQuote()); tableName = schema.getTableName().replace("*", "%"); columnNames = schema.getColumnName(); if (columnNames.startsWith("*")) { @@ -1184,30 +1442,19 @@ private Map> splitAndMergeHistoryQueryPatterns(List< Pattern tableNamePattern = patternList.get(0), columnNamePattern = patternList.get(1); if (databaseName.equals("%")) { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); - while (databaseSet.next()) { - String tempDatabaseName = databaseSet.getString("DATNAME"); + for (String tempDatabaseName : getDatabaseNames()) { if (tempDatabaseName.startsWith(DATABASE_PREFIX)) { continue; } - Connection conn = getConnection(tempDatabaseName); - if (conn == null) { - continue; - } - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = - databaseMetaData.getTables( - tempDatabaseName, "public", tableName, new String[] {"TABLE"}); - while (tableSet.next()) { - String tempTableName = tableSet.getString("TABLE_NAME"); + List tables = getTables(tempDatabaseName, tableName); + for (String tempTableName : tables) { if (!tableNamePattern.matcher(tempTableName).find()) { continue; } - ResultSet columnSet = - databaseMetaData.getColumns(tempDatabaseName, "public", tempTableName, columnNames); - while (columnSet.next()) { - String tempColumnNames = columnSet.getString("COLUMN_NAME"); + List columnFieldList = + getColumns(tempDatabaseName, tempTableName, columnNames); + for (ColumnField columnField : columnFieldList) { + String tempColumnNames = columnField.columnName; if (!columnNamePattern.matcher(tempColumnNames).find()) { continue; } @@ -1221,19 +1468,13 @@ private Map> splitAndMergeHistoryQueryPatterns(List< splitResults.put(tempDatabaseName, tableNameToColumnNames); } } - conn.close(); } } else { - Connection conn = getConnection(databaseName); - if (conn == null) { - continue; - } - ResultSet rs = - conn.getMetaData().getColumns(databaseName, "public", tableName, columnNames); + List columnFieldList = getColumns(databaseName, tableName, columnNames); Map tableNameToColumnNames = new HashMap<>(); - while (rs.next()) { - tableName = rs.getString("TABLE_NAME"); - columnNames = rs.getString("COLUMN_NAME"); + for (ColumnField columnField : columnFieldList) { + tableName = columnField.tableName; + columnNames = columnField.columnName; if (!tableNamePattern.matcher(tableName).find() || !columnNamePattern.matcher(columnNames).find()) { continue; @@ -1265,7 +1506,6 @@ private Map> splitAndMergeHistoryQueryPatterns(List< tableNameToColumnNames = oldTableNameToColumnNames; } splitResults.put(databaseName, tableNameToColumnNames); - conn.close(); } } @@ -1285,41 +1525,36 @@ private void createOrAlterTables( tags = tagsList.get(i); } DataType dataType = dataTypeList.get(i); - PostgreSQLSchema schema = new PostgreSQLSchema(path); + RelationSchema schema = new RelationSchema(path, relationalMeta.getQuote()); String tableName = schema.getTableName(); String columnName = schema.getColumnName(); try { Statement stmt = conn.createStatement(); - DatabaseMetaData databaseMetaData = conn.getMetaData(); - ResultSet tableSet = - databaseMetaData.getTables(storageUnit, "public", tableName, new String[] {"TABLE"}); + + List tables = getTables(storageUnit, tableName); columnName = toFullName(columnName, tags); - if (!tableSet.next()) { + if (tables.isEmpty()) { String statement = String.format( - CREATE_TABLE_STATEMENT, + relationalMeta.getCreateTableStatement(), getQuotName(tableName), getQuotName(columnName), - DataTypeTransformer.toPostgreSQL(dataType)); + relationalMeta.getDataTypeTransformer().toEngineType(dataType)); LOGGER.info("[Create] execute create: {}", statement); stmt.execute(statement); } else { - ResultSet columnSet = - databaseMetaData.getColumns(storageUnit, "public", tableName, columnName); - if (!columnSet.next()) { + if (getColumns(storageUnit, tableName, columnName).isEmpty()) { String statement = String.format( ADD_COLUMN_STATEMENT, getQuotName(tableName), getQuotName(columnName), - DataTypeTransformer.toPostgreSQL(dataType)); + relationalMeta.getDataTypeTransformer().toEngineType(dataType)); LOGGER.info("[Create] execute create: {}", statement); stmt.execute(statement); } - columnSet.close(); } - tableSet.close(); stmt.close(); } catch (SQLException e) { LOGGER.error("create or alter table {} field {} error: ", tableName, columnName, e); @@ -1351,7 +1586,7 @@ private Exception insertNonAlignedRowRecords( for (int j = 0; j < data.getPathNum(); j++) { String path = data.getPath(j); DataType dataType = data.getDataType(j); - PostgreSQLSchema schema = new PostgreSQLSchema(path); + RelationSchema schema = new RelationSchema(path, relationalMeta.getQuote()); String tableName = schema.getTableName(); String columnName = schema.getColumnName(); Map tags = data.getTags(j); @@ -1455,7 +1690,7 @@ private Exception insertNonAlignedColumnRecords( for (int i = 0; i < data.getPathNum(); i++) { String path = data.getPath(i); DataType dataType = data.getDataType(i); - PostgreSQLSchema schema = new PostgreSQLSchema(path); + RelationSchema schema = new RelationSchema(path, relationalMeta.getQuote()); String tableName = schema.getTableName(); String columnName = schema.getColumnName(); Map tags = data.getTags(i); @@ -1537,7 +1772,8 @@ private Exception insertNonAlignedColumnRecords( } stmt.close(); } catch (SQLException e) { - return new SQLException("encounter error when inserting non-aligned column records", e); + LOGGER.error("unexpected error: ", e); + return e; } return null; @@ -1553,9 +1789,9 @@ private void executeBatchInsert( String[] parts = columnNames.split(", "); boolean hasMultipleRows = parts.length != 1; - // INSERT INTO XXX ("\u2E85", XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., + // INSERT INTO XXX ("key", XXX, ...) VALUES (XXX, XXX, ...), (XXX, XXX, ...), ..., // (XXX, - // XXX, ...) ON CONFLICT ("\u2E85") DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); + // XXX, ...) ON CONFLICT ("key") DO UPDATE SET (XXX, ...) = (excluded.XXX, ...); StringBuilder statement = new StringBuilder(); statement.append("INSERT INTO "); statement.append(getQuotName(tableName)); @@ -1571,66 +1807,59 @@ private void executeBatchInsert( statement.append(value, 0, value.length() - 2); statement.append("), "); } - statement = new StringBuilder(statement.substring(0, statement.length() - 2)); + statement.delete(statement.length() - 2, statement.length()); + + statement.append(relationalMeta.getUpsertStatement()); - statement.append(" ON CONFLICT ("); - statement.append(getQuotName(KEY_NAME)); - statement.append(") DO UPDATE SET "); - if (hasMultipleRows) { - statement.append("("); // 只有一列不加括号 - } - statement.append(fullColumnNames); - if (hasMultipleRows) { - statement.append(")"); // 只有一列不加括号 - } - statement.append(" = "); - if (hasMultipleRows) { - statement.append("("); // 只有一列不加括号 - } for (String part : parts) { - statement.append("excluded."); - statement.append(getQuotName(part)); + if (part.equals(KEY_NAME)) { + continue; + } + statement.append( + String.format( + relationalMeta.getUpsertConflictStatement(), getQuotName(part), getQuotName(part))); statement.append(", "); } - statement = new StringBuilder(statement.substring(0, statement.length() - 2)); - if (hasMultipleRows) { - statement.append(")"); // 只有一列不加括号 - } + + statement.delete(statement.length() - 2, statement.length()); + statement.append(";"); - // LOGGER.info("[Insert] execute insert: {}", statement); stmt.addBatch(statement.toString()); } stmt.executeBatch(); } - private List> determineDeletedPaths(List paths, TagFilter tagFilter) - throws PostgreSQLException { - List columns = getColumns(); - List> deletedPaths = new ArrayList<>(); + private List> determineDeletedPaths( + List paths, TagFilter tagFilter) { + try { + List columns = getColumns(); + List> deletedPaths = new ArrayList<>(); - for (Column column : columns) { - for (String path : paths) { - if (Pattern.matches(StringUtils.reformatPath(path), column.getPath())) { - if (tagFilter != null && !TagKVUtils.match(column.getTags(), tagFilter)) { - continue; + for (Column column : columns) { + for (String path : paths) { + if (Pattern.matches(StringUtils.reformatPath(path), column.getPath())) { + if (tagFilter != null && !TagKVUtils.match(column.getTags(), tagFilter)) { + continue; + } + String fullPath = column.getPath(); + RelationSchema schema = new RelationSchema(fullPath, relationalMeta.getQuote()); + String tableName = schema.getTableName(); + String columnName = toFullName(schema.getColumnName(), column.getTags()); + deletedPaths.add(new Pair<>(tableName, columnName)); + break; } - String fullPath = column.getPath(); - PostgreSQLSchema schema = new PostgreSQLSchema(fullPath); - String tableName = schema.getTableName(); - String columnName = toFullName(schema.getColumnName(), column.getTags()); - deletedPaths.add(new Pair<>(tableName, columnName)); - break; } } + return deletedPaths; + } catch (RelationalTaskExecuteFailureException e) { + LOGGER.error(e.getMessage(), e); + return new ArrayList<>(); } - - return deletedPaths; } private String getQuotName(String name) { - return "\"" + name + "\""; - // return Character.isDigit(name.charAt(0)) ? "\"" + name + "\"" : name; + return relationalMeta.getQuote() + name + relationalMeta.getQuote(); } private String getQuotColumnNames(String columnNames) { @@ -1644,11 +1873,11 @@ private String getQuotColumnNames(String columnNames) { } @Override - public void release() throws PostgreSQLException { + public void release() throws PhysicalException { try { connection.close(); } catch (SQLException e) { - throw new PostgreSQLException("failed to close connection", e); + throw new RelationalException(e); } } } diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java new file mode 100644 index 0000000000..bdf0530397 --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/IDataTypeTransformer.java @@ -0,0 +1,10 @@ +package cn.edu.tsinghua.iginx.relational.datatype.transformer; + +import cn.edu.tsinghua.iginx.thrift.DataType; + +public interface IDataTypeTransformer { + + public DataType fromEngineType(String dataType); + + public String toEngineType(DataType dataType); +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java new file mode 100644 index 0000000000..be6b23cc35 --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/JDBCDataTypeTransformer.java @@ -0,0 +1,45 @@ +package cn.edu.tsinghua.iginx.relational.datatype.transformer; + +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class JDBCDataTypeTransformer implements IDataTypeTransformer { + private final Properties typeMappings; + + private static final String prefix = "IGinX-"; + + public JDBCDataTypeTransformer(Properties properties) { + typeMappings = properties; + } + + @Override + public DataType fromEngineType(String dataType) { + String mappedType = typeMappings.getProperty(dataType); + if (mappedType != null) { + return str2DataType(mappedType); + } + return DataType.BINARY; + } + + @Override + public String toEngineType(DataType dataType) { + return typeMappings.getProperty(prefix + dataType.name(), "TEXT"); + } + + private static final Map dataTypeMap = new HashMap<>(); + + static { + dataTypeMap.put(prefix + "BOOLEAN", DataType.BOOLEAN); + dataTypeMap.put(prefix + "INTEGER", DataType.INTEGER); + dataTypeMap.put(prefix + "LONG", DataType.LONG); + dataTypeMap.put(prefix + "FLOAT", DataType.FLOAT); + dataTypeMap.put(prefix + "DOUBLE", DataType.DOUBLE); + dataTypeMap.put(prefix + "BINARY", DataType.BINARY); + } + + private static DataType str2DataType(String typeStr) { + return dataTypeMap.getOrDefault(typeStr, DataType.BINARY); + } +} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java similarity index 71% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java index 6fe42067e1..33b512390a 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/DataTypeTransformer.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/datatype/transformer/PostgreSQLDataTypeTransformer.java @@ -1,12 +1,18 @@ -package cn.edu.tsinghua.iginx.postgresql.tools; +package cn.edu.tsinghua.iginx.relational.datatype.transformer; import static cn.edu.tsinghua.iginx.thrift.DataType.*; import cn.edu.tsinghua.iginx.thrift.DataType; -public class DataTypeTransformer { +public class PostgreSQLDataTypeTransformer implements IDataTypeTransformer { - public static DataType fromPostgreSQL(String dataType) { + private static final PostgreSQLDataTypeTransformer INSTANCE = new PostgreSQLDataTypeTransformer(); + + public static PostgreSQLDataTypeTransformer getInstance() { + return INSTANCE; + } + + public DataType fromEngineType(String dataType) { if (dataType.equalsIgnoreCase("bool")) { return BOOLEAN; } else if (dataType.equalsIgnoreCase("int") @@ -26,7 +32,7 @@ public static DataType fromPostgreSQL(String dataType) { } } - public static String toPostgreSQL(DataType dataType) { + public String toEngineType(DataType dataType) { switch (dataType) { case BOOLEAN: return "BOOLEAN"; diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java new file mode 100644 index 0000000000..2b3bd398f3 --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalException.java @@ -0,0 +1,18 @@ +package cn.edu.tsinghua.iginx.relational.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; + +public class RelationalException extends PhysicalException { + + public RelationalException(Throwable cause) { + super(cause); + } + + public RelationalException(String message) { + super(message); + } + + public RelationalException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java new file mode 100644 index 0000000000..3c588a5e4c --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/exception/RelationalTaskExecuteFailureException.java @@ -0,0 +1,14 @@ +package cn.edu.tsinghua.iginx.relational.exception; + +import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; + +public class RelationalTaskExecuteFailureException extends PhysicalTaskExecuteFailureException { + + public RelationalTaskExecuteFailureException(String message) { + super(message); + } + + public RelationalTaskExecuteFailureException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java new file mode 100644 index 0000000000..8225037585 --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/AbstractRelationalMeta.java @@ -0,0 +1,162 @@ +package cn.edu.tsinghua.iginx.relational.meta; + +import static cn.edu.tsinghua.iginx.relational.tools.Constants.KEY_NAME; + +import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; +import cn.edu.tsinghua.iginx.relational.datatype.transformer.IDataTypeTransformer; +import java.util.List; + +public abstract class AbstractRelationalMeta { + + protected StorageEngineMeta meta; + + protected static final String DROP_DATABASE_STATEMENT = "DROP DATABASE IF EXISTS %s;"; + + public AbstractRelationalMeta(StorageEngineMeta meta) { + this.meta = meta; + } + + /** + * 获取ENGINE的默认数据库名称 + * + * @return ENGINE的默认数据库名称 + */ + public abstract String getDefaultDatabaseName(); + + /** + * 获取ENGINE的驱动类 + * + * @return ENGINE的驱动类 + */ + public abstract String getDriverClass(); + + /** + * 获取ENGINE的数据类型转换器 + * + * @return ENGINE的数据类型转换器 + */ + public abstract IDataTypeTransformer getDataTypeTransformer(); + + /** + * 获取系统数据库名称，用于忽略，如pg的template0,template1 + * + * @return 系统数据库名称 + */ + public abstract List getSystemDatabaseName(); + + /** + * 获取数据库列表查询SQL + * + * @return 数据库列表查询SQL + */ + public abstract String getDatabaseQuerySql(); + + /** + * 获取引号，在SQL中，不同的数据库引号不同，PG是双引号，MYSQL是反引号 + * + * @return 引号 + */ + public abstract char getQuote(); + + /** + * 获取Update的SQL语句 + * + * @return Update的SQL语句 + */ + public String getUpdateStatement() { + return "UPDATE %s SET %s = null WHERE (" + + getQuote() + + KEY_NAME + + getQuote() + + " >= %d AND " + + getQuote() + + KEY_NAME + + getQuote() + + " < %d);"; + } + + /** + * 获取query的SQL语句 + * + * @return query的SQL语句 + */ + public String getQueryStatement() { + return "SELECT " + + getQuote() + + KEY_NAME + + getQuote() + + ", %s FROM %s %s ORDER BY " + + getQuote() + + KEY_NAME + + getQuote() + + ";"; + } + + /** + * 获取通过concat生成key的query的SQL语句 + * + * @return 通过concat生成key的query的SQL语句 + */ + public String getConcatQueryStatement() { + return "SELECT concat(%s) AS " + + getQuote() + + KEY_NAME + + getQuote() + + ", %s FROM %s %s ORDER BY %s"; + } + + public String getCreateTableStatement() { + return "CREATE TABLE %s (" + + getQuote() + + KEY_NAME + + getQuote() + + " BIGINT NOT NULL, %s %s, PRIMARY KEY(" + + getQuote() + + KEY_NAME + + getQuote() + + "));"; + } + + public abstract String getDropDatabaseStatement(); + + /** + * 在使用JDBC时元数据查询时，是否需要引号 + * + * @return 是否需要引号 + */ + public abstract boolean jdbcNeedQuote(); + + /** + * 获取数据源在使用JDBC获取元数据时的schemaPattern + * + * @return schemaPattern + */ + public abstract String getSchemaPattern(); + + /** + * 获取upsert中间那段SQL语句 + * + * @return 获取upsert中间那段SQL语句 + */ + public abstract String getUpsertStatement(); + + /** + * 获取upsert冲突后段SQL语句格式 + * + * @return 获取upsert冲突后段SQL语句格式 + */ + public abstract String getUpsertConflictStatement(); + + /** 是否支持Full Join */ + public abstract boolean isSupportFullJoin(); + + /** + * 获取正则表达式的操作符 + * + * @return 正则表达式的操作符 + */ + public abstract String getRegexpOp(); + + /** jdbc获取元数据是否支持反斜杠的识别 */ + public abstract boolean jdbcSupportSpecialChar(); +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java new file mode 100644 index 0000000000..66204a301c --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/JDBCMeta.java @@ -0,0 +1,143 @@ +package cn.edu.tsinghua.iginx.relational.meta; + +import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; +import cn.edu.tsinghua.iginx.relational.datatype.transformer.IDataTypeTransformer; +import cn.edu.tsinghua.iginx.relational.datatype.transformer.JDBCDataTypeTransformer; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +public class JDBCMeta extends AbstractRelationalMeta { + + private final Properties properties; + + private final char quote; + private String defaultDatabaseName; + + private String driverClass; + + private JDBCDataTypeTransformer dataTypeTransformer; + + private List systemDatabaseName; + + private String databaseQuerySql; + + private String databaseDropStatement; + + private boolean needQuote; + + private String schemaPattern; + + private String upsertStatement; + + private String upsertConflictStatement; + + private boolean isSupportFullJoin; + + private String regexpOp; + + private boolean jdbcSupportBackslash; + + public JDBCMeta(StorageEngineMeta meta, String propertiesPath) throws IOException { + super(meta); + properties = new Properties(); + File file = new File(propertiesPath); + if (!file.exists()) { + throw new IOException(String.format("Properties file %s not found", file.getAbsolutePath())); + } + try (InputStream inputStream = Files.newInputStream(Paths.get(propertiesPath))) { + properties.load(inputStream); + } + + quote = properties.getProperty("quote").charAt(0); + driverClass = properties.getProperty("driver_class"); + defaultDatabaseName = properties.getProperty("default_database"); + dataTypeTransformer = new JDBCDataTypeTransformer(properties); + systemDatabaseName = Arrays.asList(properties.getProperty("system_databases").split(",")); + databaseQuerySql = properties.getProperty("database_query_sql"); + databaseDropStatement = properties.getProperty("drop_database_statement"); + needQuote = Boolean.parseBoolean(properties.getProperty("jdbc_need_quote")); + schemaPattern = properties.getProperty("schema_pattern"); + upsertStatement = properties.getProperty("upsert_statement"); + upsertConflictStatement = properties.getProperty("upsert_conflict_statement"); + isSupportFullJoin = Boolean.parseBoolean(properties.getProperty("is_support_full_join")); + regexpOp = properties.getProperty("regex_like_symbol"); + jdbcSupportBackslash = + Boolean.parseBoolean(properties.getProperty("jdbc_support_special_char")); + } + + @Override + public char getQuote() { + return quote; + } + + @Override + public String getDefaultDatabaseName() { + return defaultDatabaseName; + } + + @Override + public String getDriverClass() { + return driverClass; + } + + @Override + public IDataTypeTransformer getDataTypeTransformer() { + return dataTypeTransformer; + } + + @Override + public List getSystemDatabaseName() { + return systemDatabaseName; + } + + @Override + public String getDatabaseQuerySql() { + return databaseQuerySql; + } + + @Override + public String getDropDatabaseStatement() { + return databaseDropStatement; + } + + @Override + public boolean jdbcNeedQuote() { + return needQuote; + } + + @Override + public String getSchemaPattern() { + return schemaPattern; + } + + @Override + public String getUpsertStatement() { + return upsertStatement; + } + + @Override + public String getUpsertConflictStatement() { + return upsertConflictStatement; + } + + @Override + public boolean isSupportFullJoin() { + return isSupportFullJoin; + } + + @Override + public String getRegexpOp() { + return regexpOp; + } + + @Override + public boolean jdbcSupportSpecialChar() { + return jdbcSupportBackslash; + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java new file mode 100644 index 0000000000..108001d1cb --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/meta/PostgreSQLMeta.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package cn.edu.tsinghua.iginx.relational.meta; + +import static cn.edu.tsinghua.iginx.relational.tools.Constants.KEY_NAME; + +import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; +import cn.edu.tsinghua.iginx.relational.datatype.transformer.IDataTypeTransformer; +import cn.edu.tsinghua.iginx.relational.datatype.transformer.PostgreSQLDataTypeTransformer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.postgresql.ds.PGConnectionPoolDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgreSQLMeta extends AbstractRelationalMeta { + + private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLMeta.class); + + private final Map connectionPoolMap = + new ConcurrentHashMap<>(); + + private static final String TIMEOUT_SETTING_SQL = + "alter system set idle_in_transaction_session_timeout='1min'"; + + private static final String DRIVER_CLASS = "org.postgresql.Driver"; + + private static final String DEFAULT_DATABASE_NAME = "postgres"; + + private static final PostgreSQLDataTypeTransformer dataTypeTransformer = + PostgreSQLDataTypeTransformer.getInstance(); + + private static final List SYSTEM_DATABASE_NAME = + new ArrayList<>(Arrays.asList("template0", "template1", "readme_to_recover")); + + public PostgreSQLMeta(StorageEngineMeta meta) { + super(meta); + } + + @Override + public String getDefaultDatabaseName() { + return DEFAULT_DATABASE_NAME; + } + + @Override + public String getDriverClass() { + return DRIVER_CLASS; + } + + @Override + public IDataTypeTransformer getDataTypeTransformer() { + return dataTypeTransformer; + } + + @Override + public List getSystemDatabaseName() { + return SYSTEM_DATABASE_NAME; + } + + @Override + public String getDatabaseQuerySql() { + return "SELECT datname FROM pg_database;"; + } + + @Override + public char getQuote() { + return '"'; + } + + public String getDropDatabaseStatement() { + return "DROP DATABASE IF EXISTS %s WITH (FORCE);"; + } + + @Override + public boolean jdbcNeedQuote() { + return false; + } + + @Override + public String getSchemaPattern() { + return "public"; + } + + @Override + public String getUpsertStatement() { + return " ON CONFLICT (" + getQuote() + KEY_NAME + getQuote() + ") DO UPDATE SET "; + } + + @Override + public String getUpsertConflictStatement() { + return "%s = EXCLUDED.%s"; + } + + @Override + public boolean isSupportFullJoin() { + return true; + } + + @Override + public String getRegexpOp() { + return "~"; + } + + @Override + public boolean jdbcSupportSpecialChar() { + return true; + } +} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java similarity index 80% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java index 67a43abed4..b7d7a3a0f5 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/query/entity/PostgreSQLQueryRowStream.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/query/entity/RelationQueryRowStream.java @@ -1,10 +1,10 @@ -package cn.edu.tsinghua.iginx.postgresql.query.entity; +package cn.edu.tsinghua.iginx.relational.query.entity; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.SEPARATOR; import static cn.edu.tsinghua.iginx.engine.physical.memory.execute.utils.FilterUtils.validate; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.KEY_NAME; -import static cn.edu.tsinghua.iginx.postgresql.tools.HashUtils.toHash; -import static cn.edu.tsinghua.iginx.postgresql.tools.TagKVUtils.splitFullName; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.*; +import static cn.edu.tsinghua.iginx.relational.tools.HashUtils.toHash; +import static cn.edu.tsinghua.iginx.relational.tools.TagKVUtils.splitFullName; import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; @@ -15,8 +15,8 @@ import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; -import cn.edu.tsinghua.iginx.postgresql.tools.DataTypeTransformer; -import cn.edu.tsinghua.iginx.postgresql.tools.PostgreSQLSchema; +import cn.edu.tsinghua.iginx.relational.meta.AbstractRelationalMeta; +import cn.edu.tsinghua.iginx.relational.tools.RelationSchema; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; import java.sql.Connection; @@ -27,9 +27,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PostgreSQLQueryRowStream implements RowStream { +public class RelationQueryRowStream implements RowStream { - private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLQueryRowStream.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RelationQueryRowStream.class); private final List resultSets; @@ -57,18 +57,26 @@ public class PostgreSQLQueryRowStream implements RowStream { private List connList; - public PostgreSQLQueryRowStream( + private AbstractRelationalMeta relationalMeta; + + private String fullKeyName = KEY_NAME; + + private boolean isPushDown = false; + + public RelationQueryRowStream( List databaseNameList, List resultSets, boolean isDummy, Filter filter, TagFilter tagFilter, - List connList) + List connList, + AbstractRelationalMeta relationalMeta) throws SQLException { this.resultSets = resultSets; this.isDummy = isDummy; this.filter = filter; this.connList = connList; + this.relationalMeta = relationalMeta; if (resultSets.isEmpty()) { this.header = new Header(Field.KEY, Collections.emptyList()); @@ -94,10 +102,23 @@ public PostgreSQLQueryRowStream( String columnName = resultSetMetaData.getColumnName(j); String typeName = resultSetMetaData.getColumnTypeName(j); + if (j == 1 && columnName.contains(KEY_NAME) && columnName.contains(SEPARATOR)) { + isPushDown = true; + } + + if (!relationalMeta.isSupportFullJoin() && isPushDown) { + System.out.println(columnName); + RelationSchema relationSchema = + new RelationSchema(columnName, isDummy, relationalMeta.getQuote()); + tableName = relationSchema.getTableName(); + columnName = relationSchema.getColumnName(); + } + columnNameSet.add(columnName); if (j == 1 && columnName.equals(KEY_NAME)) { key = Field.KEY; + this.fullKeyName = resultSetMetaData.getColumnName(j); continue; } @@ -107,13 +128,13 @@ public PostgreSQLQueryRowStream( field = new Field( databaseNameList.get(i) + SEPARATOR + tableName + SEPARATOR + namesAndTags.k, - DataTypeTransformer.fromPostgreSQL(typeName), + relationalMeta.getDataTypeTransformer().fromEngineType(typeName), namesAndTags.v); } else { field = new Field( tableName + SEPARATOR + namesAndTags.k, - DataTypeTransformer.fromPostgreSQL(typeName), + relationalMeta.getDataTypeTransformer().fromEngineType(typeName), namesAndTags.v); } @@ -224,8 +245,11 @@ private void cacheOneRow() throws SQLException, PhysicalException { for (int j = 0; j < resultSetSizes[i]; j++) { String columnName = fieldToColumnName.get(header.getField(startIndex + j)); - PostgreSQLSchema schema = - new PostgreSQLSchema(header.getField(startIndex + j).getName(), isDummy); + RelationSchema schema = + new RelationSchema( + header.getField(startIndex + j).getName(), + isDummy, + relationalMeta.getQuote()); String tableName = schema.getTableName(); tableNameSet.add(tableName); @@ -233,6 +257,13 @@ private void cacheOneRow() throws SQLException, PhysicalException { Object value = getResultSetObject(resultSet, columnName, tableName); if (header.getField(startIndex + j).getType() == DataType.BINARY && value != null) { tempValue = value.toString().getBytes(); + } else if (header.getField(startIndex + j).getType() == DataType.BOOLEAN + && value != null) { + if (value instanceof Boolean) { + tempValue = value; + } else { + tempValue = ((int) value) == 1; + } } else { tempValue = value; } @@ -243,11 +274,11 @@ private void cacheOneRow() throws SQLException, PhysicalException { // 在Dummy查询的Join操作中，key列的值是由多个Join表的所有列的值拼接而成的，但实际上的Key列仅由一个表的所有列的值拼接而成 // 所以在这里需要将key列的值截断为一个表的所有列的值，因为能合并在一行里的不同表的数据一定是key相同的 // 所以查询出来的KEY值一定是（我们需要的KEY值 * 表的数量），因此只需要裁剪取第一个表的key列的值即可 - String keyString = resultSet.getString(KEY_NAME); + String keyString = resultSet.getString(fullKeyName); keyString = keyString.substring(0, keyString.length() / tableNameSet.size()); tempKey = toHash(keyString); } else { - tempKey = resultSet.getLong(KEY_NAME); + tempKey = resultSet.getLong(fullKeyName); } cachedKeys[i] = tempKey; @@ -301,6 +332,10 @@ private void cacheOneRow() throws SQLException, PhysicalException { */ private Object getResultSetObject(ResultSet resultSet, String columnName, String tableName) throws SQLException { + if (!relationalMeta.isSupportFullJoin() && isPushDown) { + return resultSet.getObject(tableName + SEPARATOR + columnName); + } + if (!resultSetHasColumnWithTheSameName.get(resultSets.indexOf(resultSet))) { return resultSet.getObject(columnName); } diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java new file mode 100644 index 0000000000..ca8bc0e0cc --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/ColumnField.java @@ -0,0 +1,13 @@ +package cn.edu.tsinghua.iginx.relational.tools; + +public class ColumnField { + public String tableName; + public String columnName; + public String columnType; + + public ColumnField(String tableName, String columnName, String columnType) { + this.tableName = tableName; + this.columnName = columnName; + this.columnType = columnType; + } +} diff --git a/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java new file mode 100644 index 0000000000..e7e7cca55f --- /dev/null +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/Constants.java @@ -0,0 +1,34 @@ +package cn.edu.tsinghua.iginx.relational.tools; + +import java.util.HashMap; +import java.util.Map; + +public abstract class Constants { + public static final String TAGKV_EQUAL = "="; + + public static final String TAGKV_SEPARATOR = "-"; + + public static final int BATCH_SIZE = 10000; + + public static final String USERNAME = "username"; + + public static final String PASSWORD = "password"; + + public static final String KEY_NAME = "RELATIONAL+KEY"; + + public static final String DATABASE_PREFIX = "unit"; + + public static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE %s;"; + + public static final String QUERY_STATEMENT_WITHOUT_KEYNAME = "SELECT %s FROM %s %s ORDER BY %s;"; + + public static final String ADD_COLUMN_STATEMENT = "ALTER TABLE %s ADD COLUMN %s %s;"; + + public static final String DROP_COLUMN_STATEMENT = "ALTER TABLE %s DROP COLUMN %s;"; + + public static final Map classMap = new HashMap<>(); + + static { + classMap.put("postgresql", "cn.edu.tsinghua.iginx.relational.meta.PostgreSQLMeta"); + } +} diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java similarity index 75% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java index ca27b59cf6..19a3fc800f 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/FilterTransformer.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/FilterTransformer.java @@ -16,17 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package cn.edu.tsinghua.iginx.postgresql.tools; +package cn.edu.tsinghua.iginx.relational.tools; import static cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op.isLikeOp; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.*; import cn.edu.tsinghua.iginx.engine.shared.operator.filter.*; +import cn.edu.tsinghua.iginx.relational.meta.AbstractRelationalMeta; import cn.edu.tsinghua.iginx.thrift.DataType; public class FilterTransformer { - public static String toString(Filter filter) { + AbstractRelationalMeta relationalMeta; + + public FilterTransformer(AbstractRelationalMeta relationalMeta) { + this.relationalMeta = relationalMeta; + } + + public String toString(Filter filter) { if (filter == null) { return ""; } @@ -50,7 +57,7 @@ public static String toString(Filter filter) { } } - private static String toString(AndFilter filter) { + private String toString(AndFilter filter) { StringBuilder sb = new StringBuilder(); for (Filter child : filter.getChildren()) { String filterStr = toString(child); @@ -66,28 +73,28 @@ private static String toString(AndFilter filter) { return "(" + sb.substring(0, sb.length() - 4) + ")"; } - private static String toString(BoolFilter filter) { + private String toString(BoolFilter filter) { return filter.isTrue() ? "true" : "false"; } - private static String toString(NotFilter filter) { + private String toString(NotFilter filter) { return "not " + toString(filter.getChild()); } - private static String toString(KeyFilter filter) { + private String toString(KeyFilter filter) { String op = Op.op2StrWithoutAndOr(filter.getOp()) .replace("==", "="); // postgresql does not support "==" but uses "=" instead return (getQuotName(KEY_NAME) + " " + op + " " + filter.getValue()); } - private static String toString(ValueFilter filter) { - PostgreSQLSchema schema = new PostgreSQLSchema(filter.getPath()); + private String toString(ValueFilter filter) { + RelationSchema schema = new RelationSchema(filter.getPath(), relationalMeta.getQuote()); String path = schema.getQuotFullName(); String op = isLikeOp(filter.getOp()) - ? "~" + ? relationalMeta.getRegexpOp() : Op.op2StrWithoutAndOr(filter.getOp()) .replace("==", "="); // postgresql does not support "==" but uses "=" instead @@ -101,7 +108,7 @@ private static String toString(ValueFilter filter) { return path + " " + op + " " + value; } - private static String toString(OrFilter filter) { + private String toString(OrFilter filter) { StringBuilder sb = new StringBuilder(); for (Filter child : filter.getChildren()) { String filterStr = toString(child); @@ -117,9 +124,9 @@ private static String toString(OrFilter filter) { return "(" + sb.substring(0, sb.length() - 4) + ")"; } - private static String toString(PathFilter filter) { - PostgreSQLSchema schemaA = new PostgreSQLSchema(filter.getPathA()); - PostgreSQLSchema schemaB = new PostgreSQLSchema(filter.getPathB()); + private String toString(PathFilter filter) { + RelationSchema schemaA = new RelationSchema(filter.getPathA(), relationalMeta.getQuote()); + RelationSchema schemaB = new RelationSchema(filter.getPathB(), relationalMeta.getQuote()); String pathA = schemaA.getQuotFullName(); String pathB = schemaB.getQuotFullName(); @@ -130,7 +137,7 @@ private static String toString(PathFilter filter) { return pathA + " " + op + " " + pathB; } - private static String getQuotName(String name) { - return "\"" + name + "\""; + private String getQuotName(String name) { + return relationalMeta.getQuote() + name + relationalMeta.getQuote(); } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java similarity index 88% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java index b4c0543a62..45c3845a60 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/HashUtils.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/HashUtils.java @@ -1,4 +1,4 @@ -package cn.edu.tsinghua.iginx.postgresql.tools; +package cn.edu.tsinghua.iginx.relational.tools; public class HashUtils { diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/PostgreSQLSchema.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java similarity index 59% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/PostgreSQLSchema.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java index 59ff16be62..052c245cf1 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/PostgreSQLSchema.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/RelationSchema.java @@ -1,9 +1,8 @@ -package cn.edu.tsinghua.iginx.postgresql.tools; +package cn.edu.tsinghua.iginx.relational.tools; import static cn.edu.tsinghua.iginx.constant.GlobalConstant.SEPARATOR; -import static cn.edu.tsinghua.iginx.engine.shared.Constants.*; -public class PostgreSQLSchema { +public class RelationSchema { private final String databaseName; @@ -11,11 +10,13 @@ public class PostgreSQLSchema { private final String columnName; - public PostgreSQLSchema(String path) { - this(path, false); + private final char quote; + + public RelationSchema(String path, char quote) { + this(path, false, quote); } - public PostgreSQLSchema(String path, boolean isDummy) { + public RelationSchema(String path, boolean isDummy, char quote) { int firstSeparator = path.indexOf("."); if (isDummy) { databaseName = path.substring(0, firstSeparator); @@ -26,16 +27,11 @@ public PostgreSQLSchema(String path, boolean isDummy) { int lastSeparator = path.lastIndexOf("."); tableName = path.substring(0, lastSeparator); columnName = path.substring(lastSeparator + 1); + this.quote = quote; } - public PostgreSQLSchema(String tableName, String columnName) { - this.databaseName = ""; - this.tableName = tableName; - this.columnName = columnName; - } - - public static String getQuotFullName(String tableName, String columnName) { - return getQuotName(tableName) + SEPARATOR + getQuotName(columnName); + public static String getQuotFullName(String tableName, String columnName, char quote) { + return getQuotName(tableName, quote) + SEPARATOR + getQuotName(columnName, quote); } public static String getFullName(String tableName, String columnName) { @@ -55,18 +51,18 @@ public String getColumnName() { } public String getQuotFullName() { - return getQuotName(tableName) + SEPARATOR + getQuotName(columnName); + return getQuotName(tableName, quote) + SEPARATOR + getQuotName(columnName, quote); } public String getQuotTableName() { - return getQuotName(tableName); + return getQuotName(tableName, quote); } public String getQuotColumnName() { - return getQuotName(columnName); + return getQuotName(columnName, quote); } - private static String getQuotName(String name) { - return "\"" + name + "\""; + private static String getQuotName(String name, char quote) { + return quote + name + quote; } } diff --git a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java similarity index 88% rename from dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java rename to dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java index b30443f9ed..d84b3c8b43 100644 --- a/dataSources/postgresql/src/main/java/cn/edu/tsinghua/iginx/postgresql/tools/TagKVUtils.java +++ b/dataSources/relational/src/main/java/cn/edu/tsinghua/iginx/relational/tools/TagKVUtils.java @@ -1,9 +1,11 @@ -package cn.edu.tsinghua.iginx.postgresql.tools; +package cn.edu.tsinghua.iginx.relational.tools; -import static cn.edu.tsinghua.iginx.postgresql.tools.Constants.*; +import static cn.edu.tsinghua.iginx.relational.tools.Constants.*; import cn.edu.tsinghua.iginx.utils.Pair; -import java.util.*; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; // TODO: 由于IGinX支持除了反引号和换行符之外的所有字符，因此需要TagKVUtils的实现 public class TagKVUtils { diff --git a/dataSources/relational/src/main/resources/mysql-meta-template.properties b/dataSources/relational/src/main/resources/mysql-meta-template.properties new file mode 100644 index 0000000000..118ecaaa88 --- /dev/null +++ b/dataSources/relational/src/main/resources/mysql-meta-template.properties @@ -0,0 +1,45 @@ +# 配置MySQL META以供JDBCMeta读取 + +# 驱动类 +driver_class=com.mysql.cj.jdbc.Driver +# 默认数据库 +default_database=mysql +# 系统数据库，用于过滤 +system_databases=performance_schema,information_schema,sys +# 用于包裹表名和字段名的引号 +quote=` +# 删除数据库的SQL语句 +drop_database_statement=DROP DATABASE IF EXISTS %s; +# 在JDBC使用getTables时是否需要加引号 +jdbc_need_quote=true +# upsert语句中间部分 +upsert_statement= ON DUPLICATE KEY UPDATE +# upsert语句后面部分格式 +upsert_conflict_statement= %s = VALUES(%s) +# 获取数据库列表的SQL语句 +database_query_sql=SELECT SCHEMA_NAME AS datname FROM information_schema.schemata; +# 是否支持full join +is_support_full_join=false +# filter中正则匹配的符号 +regex_like_symbol=REGEXP +# jdbc元数据获取是否支持特殊字符识别 +jdbc_support_special_char=true + +# 配置MySQL DataTypeTransformer +TINYINT=IGinX-BOOLEAN +SMALLINT=IGinX-INTEGER +MEDIUMINT=IGinX-INTEGER +INT=IGinX-INTEGER +BIGINT=IGinX-LONG +INTEGER=IGinX-INTEGER +FLOAT=IGinX-FLOAT +DOUBLE=IGinX-DOUBLE +DECIMAL=IGinX-DOUBLE + + +IGinX-INTEGER=INTEGER +IGinX-FLOAT=FLOAT(12,6) +IGinX-DOUBLE=DOUBLE(22,11) +IGinX-BOOLEAN=TINYINT +IGinX-BINARY=TEXT +IGinX-LONG=BIGINT diff --git a/dataSources/relational/src/main/resources/oceanbase-meta.properties b/dataSources/relational/src/main/resources/oceanbase-meta.properties new file mode 100644 index 0000000000..f7c63c7f93 --- /dev/null +++ b/dataSources/relational/src/main/resources/oceanbase-meta.properties @@ -0,0 +1,45 @@ +# 配置MySQL META以供JDBCMeta读取 + +# 驱动类 +driver_class=com.oceanbase.jdbc.Driver +# 默认数据库 +default_database=oceanbase +# 系统数据库，用于过滤 +system_databases=test,mysql,information_schema +# 用于包裹表名和字段名的引号 +quote=` +# 删除数据库的SQL语句 +drop_database_statement=DROP DATABASE IF EXISTS %s; +# 在JDBC使用getTables时是否需要加引号 +jdbc_need_quote=false +# upsert语句中间部分 +upsert_statement= ON DUPLICATE KEY UPDATE +# upsert语句后面部分格式 +upsert_conflict_statement= %s = VALUES(%s) +# 获取数据库列表的SQL语句 +database_query_sql=SELECT SCHEMA_NAME AS datname FROM information_schema.schemata; +# 是否支持full join +is_support_full_join=false +# filter中正则匹配的符号 +regex_like_symbol=REGEXP +# jdbc元数据是否支持特殊字符的识别 +jdbc_support_special_char=false + +# 配置MySQL DataTypeTransformer +TINYINT=IGinX-BOOLEAN +SMALLINT=IGinX-INTEGER +MEDIUMINT=IGinX-INTEGER +INT=IGinX-INTEGER +BIGINT=IGinX-LONG +INTEGER=IGinX-INTEGER +FLOAT=IGinX-FLOAT +DOUBLE=IGinX-DOUBLE +DECIMAL=IGinX-DOUBLE + + +IGinX-INTEGER=INTEGER +IGinX-FLOAT=FLOAT(12,6) +IGinX-DOUBLE=DOUBLE(22,11) +IGinX-BOOLEAN=TINYINT +IGinX-BINARY=TEXT +IGinX-LONG=BIGINT diff --git a/dataSources/timescaledb/pom.xml b/dataSources/timescaledb/pom.xml deleted file mode 100644 index 20e98daa92..0000000000 --- a/dataSources/timescaledb/pom.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - 4.0.0 - - - iginx - cn.edu.tsinghua - ${revision} - ../../pom.xml - - - timescaledb - IGinX TimescaleDB - - - 8 - 8 - - - - - cn.edu.tsinghua - iginx-core - ${project.version} - provided - - - org.postgresql - postgresql - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.10 - - - copy-dependencies - package - - copy-dependencies - - - ../../core/target/iginx-core-${project.version}/driver/timescaledb/ - provided - - - - - - org.apache.maven.plugins - maven-antrun-plugin - 1.7 - - - copy-native-libraries - package - - run - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/TimescaleDBStorage.java b/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/TimescaleDBStorage.java deleted file mode 100644 index 8fa7017094..0000000000 --- a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/TimescaleDBStorage.java +++ /dev/null @@ -1,526 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.timescaledb; - -import static cn.edu.tsinghua.iginx.timescaledb.tools.FilterTransformer.MAX_TIMESTAMP; - -import cn.edu.tsinghua.iginx.engine.physical.exception.NonExecutablePhysicalTaskException; -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalTaskExecuteFailureException; -import cn.edu.tsinghua.iginx.engine.physical.exception.StorageInitializationException; -import cn.edu.tsinghua.iginx.engine.physical.storage.IStorage; -import cn.edu.tsinghua.iginx.engine.physical.storage.domain.Timeseries; -import cn.edu.tsinghua.iginx.engine.physical.task.StoragePhysicalTask; -import cn.edu.tsinghua.iginx.engine.physical.task.TaskExecuteResult; -import cn.edu.tsinghua.iginx.engine.shared.TimeRange; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import cn.edu.tsinghua.iginx.engine.shared.data.write.BitmapView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.ColumnDataView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.DataView; -import cn.edu.tsinghua.iginx.engine.shared.data.write.RowDataView; -import cn.edu.tsinghua.iginx.engine.shared.operator.Delete; -import cn.edu.tsinghua.iginx.engine.shared.operator.Insert; -import cn.edu.tsinghua.iginx.engine.shared.operator.Operator; -import cn.edu.tsinghua.iginx.engine.shared.operator.OperatorType; -import cn.edu.tsinghua.iginx.engine.shared.operator.Project; -import cn.edu.tsinghua.iginx.engine.shared.operator.Select; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.TimeFilter; -import cn.edu.tsinghua.iginx.metadata.entity.FragmentMeta; -import cn.edu.tsinghua.iginx.metadata.entity.StorageEngineMeta; -import cn.edu.tsinghua.iginx.metadata.entity.TimeInterval; -import cn.edu.tsinghua.iginx.metadata.entity.TimeSeriesInterval; -import cn.edu.tsinghua.iginx.thrift.DataType; -import cn.edu.tsinghua.iginx.timescaledb.entity.TimescaleDBQueryRowStream; -import cn.edu.tsinghua.iginx.timescaledb.tools.DataTypeTransformer; -import cn.edu.tsinghua.iginx.timescaledb.tools.FilterTransformer; -import cn.edu.tsinghua.iginx.timescaledb.tools.TagFilterUtils; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import cn.edu.tsinghua.iginx.utils.Pair; -import java.util.Map.Entry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TimescaleDBStorage implements IStorage { - - private static final Logger LOGGER = LoggerFactory.getLogger(TimescaleDBStorage.class); - - private static final int BATCH_SIZE = 10000; - - private static final String STORAGE_ENGINE = "timescaledb"; - - private static final String USERNAME = "username"; - - private static final String PASSWORD = "password"; - - private static final String DEFAULT_USERNAME = "postgres"; - - private static final String DEFAULT_PASSWORD = "123456"; - - private static final String QUERY_DATABASES = "SELECT datname FROM pg_database"; - - private static final String FIRST_QUERY = "select first(%s, time) from %s"; - - private static final String LAST_QUERY = "select last(%s, time) from %s"; - - private static final String QUERY_DATA = "SELECT time, %s FROM %s WHERE %s and %s"; - - private static final String DELETE_DATA = "DELETE FROM %s WHERE time >= to_timestamp(%d) and time < to_timestamp(%d)"; - - private static final String IGINX_SEPARATOR = "."; - - private static final String TIMESCALEDB_SEPARATOR = "$"; - - private static final String DATABASE_PREFIX = "unit"; - - private final StorageEngineMeta meta; - - private Connection connection; - - public TimescaleDBStorage(StorageEngineMeta meta) throws StorageInitializationException { - this.meta = meta; - if (!testConnection()) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - connection = DriverManager.getConnection(connUrl); - } catch (SQLException e) { - throw new StorageInitializationException("cannot connect to " + meta.toString()); - } - } - - private boolean testConnection() { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/?user=%s&password=%s", meta.getIp(), meta.getPort(), - username, password); - try { - Class.forName("org.postgresql.Driver"); - DriverManager.getConnection(connUrl); - return true; - } catch (SQLException | ClassNotFoundException e) { - return false; - } - } - - @Override - public TaskExecuteResult execute(StoragePhysicalTask task) { - List operators = task.getOperators(); - if (operators.size() != 1) { - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - FragmentMeta fragment = task.getTargetFragment(); - Operator op = operators.get(0); - String storageUnit = task.getStorageUnit(); - // 先切换数据库 - useDatabase(storageUnit); - - if (op.getType() == OperatorType.Project) { // 目前只实现 project 操作符 - Project project = (Project) op; - Filter filter; - if (operators.size() == 2) { - filter = ((Select) operators.get(1)).getFilter(); - } else { - filter = new AndFilter(Arrays - .asList(new TimeFilter(Op.GE, fragment.getTimeInterval().getStartTime()), - new TimeFilter(Op.L, fragment.getTimeInterval().getEndTime()))); - } - return executeProjectTask(project, filter); - } else if (op.getType() == OperatorType.Insert) { - Insert insert = (Insert) op; - return executeInsertTask(insert); - } else if (op.getType() == OperatorType.Delete) { - Delete delete = (Delete) op; - return executeDeleteTask(delete); - } - return new TaskExecuteResult( - new NonExecutablePhysicalTaskException("unsupported physical task")); - } - - @Override - public List getTimeSeries() throws PhysicalException { - List timeseries = new ArrayList<>(); - try { - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - if (tableName.startsWith("unit")) { - tableName = tableName.substring(tableName.indexOf(SEPARATOR) + 1); - } - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - timeseries.add(new Timeseries( - tableName.replace(TIMESCALEDB_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(TIMESCALEDB_SEPARATOR, IGINX_SEPARATOR), - DataTypeTransformer.fromTimescaleDB(typeName))); - } - } - } catch (SQLException e) { - throw new PhysicalException(e); - } - return timeseries; - } - - @Override - public Pair getBoundaryOfStorage() throws PhysicalException { - long minTime = Long.MAX_VALUE, maxTime = 0; - List paths = new ArrayList<>(); - try { - Statement stmt = connection.createStatement(); - ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES); - while (databaseSet.next()) { - String databaseName = databaseSet.getString(1);//获取表名称 - if (databaseName.startsWith(DATABASE_PREFIX)) { - useDatabase(databaseName); - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", "%", new String[]{"TABLE"}); - while (tableSet.next()) { - String tableName = tableSet.getString(3);//获取表名称 - ResultSet columnSet = databaseMetaData.getColumns(null, "%", tableName, "%"); - while (columnSet.next()) { - String columnName = columnSet.getString("COLUMN_NAME");//获取列名称 - paths.add(tableName.replace(TIMESCALEDB_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + columnName.replace(TIMESCALEDB_SEPARATOR, IGINX_SEPARATOR)); - // 获取first - String firstQueryStatement = String.format(FIRST_QUERY, columnName, tableName); - Statement firstQueryStmt = connection.createStatement(); - ResultSet firstQuerySet = firstQueryStmt.executeQuery(firstQueryStatement); - if (firstQuerySet.next()) { - long currMinTime = firstQuerySet.getLong(1); - minTime = Math.min(currMinTime, minTime); - } - // 获取last - String lastQueryStatement = String.format(LAST_QUERY, columnName, tableName); - Statement lastQueryStmt = connection.createStatement(); - ResultSet lastQuerySet = lastQueryStmt.executeQuery(lastQueryStatement); - if (lastQuerySet.next()) { - long currMaxTime = lastQuerySet.getLong(1); - maxTime = Math.max(currMaxTime, maxTime); - } - } - } - } - } - } catch (SQLException e) { - throw new PhysicalException(e); - } - paths.sort(String::compareTo); - - return new Pair<>(new TimeSeriesInterval(paths.get(0), paths.get(paths.size() - 1)), - new TimeInterval(minTime, maxTime + 1)); - } - - private TaskExecuteResult executeProjectTask(Project project, - Filter filter) { // 未来可能要用 columnsInterval 对查询出来的数据进行过滤 - try { - List resultSets = new ArrayList<>(); - List fields = new ArrayList<>(); - for (String path : project.getPatterns()) { - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - // 查询序列类型 - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (columnSet.next()) { - String typeName = columnSet.getString("TYPE_NAME");//列字段类型 - fields - .add(new Field(table.replace(TIMESCALEDB_SEPARATOR, IGINX_SEPARATOR) + IGINX_SEPARATOR - + field.replace(TIMESCALEDB_SEPARATOR, IGINX_SEPARATOR) - , DataTypeTransformer.fromTimescaleDB(typeName))); - String statement = String - .format(QUERY_DATA, field, table, - TagFilterUtils.transformToFilterStr(project.getTagFilter()), - FilterTransformer.toString(filter)); - Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(statement); - resultSets.add(rs); - } - } - RowStream rowStream = new TimescaleDBQueryRowStream(resultSets, fields); - return new TaskExecuteResult(rowStream); - } catch (SQLException e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute project task in timescaledb failure", - e)); - } - } - - private TaskExecuteResult executeInsertTask(Insert insert) { - DataView dataView = insert.getData(); - Exception e = null; - switch (dataView.getRawDataType()) { - case Row: - case NonAlignedRow: - e = insertRowRecords((RowDataView) dataView); - break; - case Column: - case NonAlignedColumn: - e = insertColumnRecords((ColumnDataView) dataView); - break; - } - if (e != null) { - return new TaskExecuteResult(null, - new PhysicalException("execute insert task in iotdb12 failure", e)); - } - return new TaskExecuteResult(null, null); - } - - private void createTimeSeriesIfNotExists(String table, String field, - Map tags, DataType dataType) { - try { - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet tableSet = databaseMetaData.getTables(null, "%", table, new String[]{"TABLE"}); - if (!tableSet.next()) { - Statement stmt = connection.createStatement(); - StringBuilder stringBuilder = new StringBuilder(); - for (Entry tagsEntry : tags.entrySet()) { - stringBuilder.append(tagsEntry.getKey()).append(" TEXT,"); - } - stringBuilder.append(field).append(" ").append(DataTypeTransformer.toTimescaleDB(dataType)); - stmt.execute(String - .format("CREATE TABLE %s (time TIMESTAMPTZ NOT NULL,%s NULL)", table, - stringBuilder.toString())); - stmt.execute(String.format("SELECT create_hypertable('%s', 'time')", table)); - } else { - for (String tag : tags.keySet()) { - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, tag); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s TEXT NULL", table, tag)); - } - } - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (!columnSet.next()) { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("ALTER TABLE %s ADD COLUMN %s %s NULL", table, field, - DataTypeTransformer.toTimescaleDB(dataType))); - } - } - } catch (SQLException e) { - LOGGER.error("create timeseries error", e); - } - } - - private void useDatabase(String dbname) { - try { - Statement stmt = connection.createStatement(); - stmt.execute(String.format("create database %s", dbname)); - } catch (SQLException e) { - LOGGER.info("create database error", e); - } - try { - Map extraParams = meta.getExtraParams(); - String username = extraParams.getOrDefault(USERNAME, DEFAULT_USERNAME); - String password = extraParams.getOrDefault(PASSWORD, DEFAULT_PASSWORD); - String connUrl = String - .format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", meta.getIp(), meta.getPort(), - dbname, username, password); - connection = DriverManager.getConnection(connUrl); - Statement stmt = connection.createStatement(); - stmt.execute("CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE"); - } catch (SQLException e) { - LOGGER.info("change database error", e); - } - } - - private Exception insertRowRecords(RowDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - for (int i = 0; i < data.getTimeSize(); i++) { - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getPathNum(); j++) { - if (bitmapView.get(j)) { - String path = data.getPath(j); - DataType dataType = data.getDataType(j); - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - Map tags = data.getTags(i); - if (tags == null) { - tags = new HashMap<>(); - } - createTimeSeriesIfNotExists(table, field, tags, dataType); - - long time = data.getTimestamp(i) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(j) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getValue()).append(", "); - columnValues.append("'").append(tagEntry.getValue()).append("'").append(", "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, time, columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); - } - - index++; - } - } - } - stmt.executeBatch(); - } catch (SQLException e) { - return e; - } - - return null; - } - - private Exception insertColumnRecords(ColumnDataView data) { - int batchSize = Math.min(data.getTimeSize(), BATCH_SIZE); - try { - Statement stmt = connection.createStatement(); - for (int i = 0; i < data.getPathNum(); i++) { - String path = data.getPath(i); - DataType dataType = data.getDataType(i); - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - Map tags = data.getTags(i); - if (tags == null) { - tags = new HashMap<>(); - } - createTimeSeriesIfNotExists(table, field, tags, dataType); - BitmapView bitmapView = data.getBitmapView(i); - int index = 0; - for (int j = 0; j < data.getTimeSize(); j++) { - if (bitmapView.get(j)) { - long time = data.getTimestamp(j) / 1000; // timescaledb存10位时间戳，java为13位时间戳 - String value; - if (data.getDataType(i) == DataType.BINARY) { - value = "'" + new String((byte[]) data.getValue(i, index), StandardCharsets.UTF_8) - + "'"; - } else { - value = data.getValue(i, index).toString(); - } - - StringBuilder columnsKeys = new StringBuilder(); - StringBuilder columnValues = new StringBuilder(); - for (Entry tagEntry : tags.entrySet()) { - columnsKeys.append(tagEntry.getKey()).append(", "); - columnValues.append("'").append(tagEntry.getValue()).append("'").append(", "); - } - columnsKeys.append(field); - columnValues.append(value); - - stmt.addBatch(String - .format("INSERT INTO %s (time, %s) values (to_timestamp(%d), %s)", table, - columnsKeys, - time, - columnValues)); - if (index > 0 && (index + 1) % batchSize == 0) { - stmt.executeBatch(); - } - index++; - } - } - } - stmt.executeBatch(); - } catch (SQLException e) { - LOGGER.error("unexpected error: ", e); - return e; - } - - return null; - } - - private TaskExecuteResult executeDeleteTask(Delete delete) { - // only support to the level of device now - // TODO support the delete to the level of sensor - try { - for (int i = 0; i < delete.getPatterns().size(); i++) { - String path = delete.getPatterns().get(i); - TimeRange timeRange = delete.getTimeRanges().get(i); - String table = path.substring(0, path.lastIndexOf('.')); - table = table.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - String field = path.substring(path.lastIndexOf('.') + 1); - field = field.replace(IGINX_SEPARATOR, TIMESCALEDB_SEPARATOR); - // 查询序列类型 - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet columnSet = databaseMetaData.getColumns(null, "%", table, field); - if (columnSet.next()) { - String statement = String - .format(DELETE_DATA, table, - timeRange.getBeginTime(), Math.min(timeRange.getEndTime(), MAX_TIMESTAMP)); - Statement stmt = connection.createStatement(); - stmt.execute(statement); - } - } - return new TaskExecuteResult(null, null); - } catch (SQLException e) { - return new TaskExecuteResult( - new PhysicalTaskExecuteFailureException("execute delete task in timescaledb failure", - e)); - } - } - - @Override - public void release() throws PhysicalException { - try { - connection.close(); - } catch (SQLException e) { - throw new PhysicalException(e); - } - } -} \ No newline at end of file diff --git a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/entity/TimescaleDBQueryRowStream.java b/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/entity/TimescaleDBQueryRowStream.java deleted file mode 100644 index ca0211d4b2..0000000000 --- a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/entity/TimescaleDBQueryRowStream.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.timescaledb.entity; - -import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException; -import cn.edu.tsinghua.iginx.engine.physical.exception.RowFetchException; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Field; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Header; -import cn.edu.tsinghua.iginx.engine.shared.data.read.Row; -import cn.edu.tsinghua.iginx.engine.shared.data.read.RowStream; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -public class TimescaleDBQueryRowStream implements RowStream { - - private final List resultSets; - - private final long[] currTimestamps; - - private final Object[] currValues; - - private final Header header; - - public TimescaleDBQueryRowStream(List resultSets, List fields) { - this.resultSets = resultSets; - this.header = new Header(Field.TIME, fields); - this.currTimestamps = new long[resultSets.size()]; - this.currValues = new Object[resultSets.size()]; - // 默认填充一下timestamp列表 - try { - for (int i = 0; i < this.currTimestamps.length; i++) { - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - this.currValues[i] = resultSet.getObject(2); - } - } - } catch (SQLException e) { - LOGGER.error("unexpected error: ", e); - // pass - } - } - - @Override - public Header getHeader() { - return this.header; - } - - @Override - public void close() { - try { - for (ResultSet resultSet : resultSets) { - resultSet.close(); - } - } catch (SQLException e) { - // pass - } - } - - @Override - public boolean hasNext() throws PhysicalException { - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - return true; - } - } - return false; - } - - @Override - public Row next() throws PhysicalException { - try { - long timestamp = Long.MAX_VALUE; - Object[] values = new Object[this.resultSets.size()]; - for (long currTimestamp : this.currTimestamps) { - if (currTimestamp != Long.MIN_VALUE) { - timestamp = Math.min(timestamp, currTimestamp); - } - } - - for (int i = 0; i < this.currTimestamps.length; i++) { - if (this.currTimestamps[i] == timestamp) { - values[i] = this.currValues[i]; - ResultSet resultSet = this.resultSets.get(i); - if (resultSet.next()) { - this.currTimestamps[i] = resultSet.getTimestamp(1).getTime(); - this.currValues[i] = resultSet.getObject(2); - } else { - // 值已经取完 - this.currTimestamps[i] = Long.MIN_VALUE; - this.currValues[i] = null; - } - } - } - return new Row(header, timestamp, values); - } catch (SQLException e) { - throw new RowFetchException(e); - } - } -} diff --git a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/DataTypeTransformer.java b/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/DataTypeTransformer.java deleted file mode 100644 index 76ced58689..0000000000 --- a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/DataTypeTransformer.java +++ /dev/null @@ -1,44 +0,0 @@ -package cn.edu.tsinghua.iginx.timescaledb.tools; - -import static cn.edu.tsinghua.iginx.thrift.DataType.BINARY; -import static cn.edu.tsinghua.iginx.thrift.DataType.BOOLEAN; -import static cn.edu.tsinghua.iginx.thrift.DataType.DOUBLE; -import static cn.edu.tsinghua.iginx.thrift.DataType.FLOAT; -import static cn.edu.tsinghua.iginx.thrift.DataType.INTEGER; -import static cn.edu.tsinghua.iginx.thrift.DataType.LONG; - -import cn.edu.tsinghua.iginx.exception.UnsupportedDataTypeException; -import cn.edu.tsinghua.iginx.thrift.DataType; - -public class DataTypeTransformer { - - public static DataType fromTimescaleDB(String dataType) { - if (dataType.contains("int") || dataType.contains("timestamptz") || dataType.contains("serial")) { - return LONG; - } else if (dataType.contains("bool")) { - return BOOLEAN; - } else if (dataType.contains("float")) { - return DOUBLE; - } else { - return BINARY; - } - } - - public static String toTimescaleDB(DataType dataType) { - switch (dataType){ - case BOOLEAN: - return "BOOLEAN"; - case INTEGER: - return "INTEGER"; - case LONG: - return "BIGINT"; - case FLOAT: - return "REAL"; - case DOUBLE: - return "DOUBLE PRECISION"; - case BINARY: - default: - return "TEXT"; - } - } -} diff --git a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/FilterTransformer.java b/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/FilterTransformer.java deleted file mode 100644 index 41decc17f1..0000000000 --- a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/FilterTransformer.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package cn.edu.tsinghua.iginx.timescaledb.tools; - -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.AndFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Filter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.NotFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.OrFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.TimeFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.filter.ValueFilter; -import java.util.stream.Collectors; - -public class FilterTransformer { - - public static final long MAX_TIMESTAMP = Integer.MAX_VALUE; - - public static String toString(Filter filter) { - if (filter == null) { - return ""; - } - switch (filter.getType()) { - case And: - return toString((AndFilter) filter); - case Or: - return toString((OrFilter) filter); - case Not: - return toString((NotFilter) filter); - case Value: - return toString((ValueFilter) filter); - case Time: - return toString((TimeFilter) filter); - default: - return ""; - } - } - - private static String toString(AndFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" and ", "(", ")")); - } - - private static String toString(NotFilter filter) { - return "not " + filter.toString(); - } - - private static String toString(TimeFilter filter) { - return "time " + Op.op2Str(filter.getOp()) + " to_timestamp(" + Math.min(filter.getValue(), MAX_TIMESTAMP) + ")"; - } - - private static String toString(ValueFilter filter) { - return filter.getPath() + " " + Op.op2Str(filter.getOp()) + " " + filter.getValue().getValue(); - } - - private static String toString(OrFilter filter) { - return filter.getChildren().stream().map(FilterTransformer::toString).collect(Collectors.joining(" or ", "(", ")")); - } - - - -} diff --git a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/TagFilterUtils.java b/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/TagFilterUtils.java deleted file mode 100644 index b27e0e5a8a..0000000000 --- a/dataSources/timescaledb/src/main/java/cn/edu/tsinghua/iginx/timescaledb/tools/TagFilterUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -package cn.edu.tsinghua.iginx.timescaledb.tools; - -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.AndTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.BaseTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.OrTagFilter; -import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; - -public class TagFilterUtils { - - public static String transformToFilterStr(TagFilter filter) { - StringBuilder builder = new StringBuilder(); - transformToFilterStr(filter, builder); - return builder.toString(); - } - - private static void transformToFilterStr(TagFilter filter, StringBuilder builder) { - switch (filter.getType()) { - case And: - AndTagFilter andFilter = (AndTagFilter) filter; - for (int i = 0; i < andFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(andFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != andFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" and "); - } - } - break; - case Or: - OrTagFilter orFilter = (OrTagFilter) filter; - for (int i = 0; i < orFilter.getChildren().size(); i++) { - builder.append('('); - transformToFilterStr(orFilter.getChildren().get(i), builder); - builder.append(')'); - if (i != orFilter.getChildren().size() - 1) { // 还不是最后一个 - builder.append(" or "); - } - } - break; - case Base: - BaseTagFilter baseFilter = (BaseTagFilter) filter; - builder.append(baseFilter.getTagKey()); - builder.append("="); - builder.append(baseFilter.getTagValue()); - break; - } - } -} \ No newline at end of file diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile new file mode 100644 index 0000000000..98c47d9b62 --- /dev/null +++ b/docker/client/Dockerfile @@ -0,0 +1,13 @@ +FROM maven:3-amazoncorretto-8 AS builder +COPY . /root/iginx +WORKDIR /root/iginx +RUN mvn clean package -pl client -am -Dmaven.test.skip=true -P-format -e + +FROM openjdk:11-jre-slim +COPY --from=builder /root/iginx/client/target/iginx-client-0.6.0-SNAPSHOT/ /iginx_client/ + +RUN mkdir -p /iginx_client/logs +RUN mkdir -p /iginx_client/data +VOLUME /iginx_client/logs +ENV PATH="/iginx_client/sbin/:${PATH}" +ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/docker/client/Dockerfile-no-maven b/docker/client/Dockerfile-no-maven new file mode 100644 index 0000000000..af0681b03d --- /dev/null +++ b/docker/client/Dockerfile-no-maven @@ -0,0 +1,8 @@ +FROM openjdk:11-jre-slim +COPY ./target/iginx-client-0.6.0-SNAPSHOT/ /iginx_client/ + +RUN mkdir -p /iginx_client/logs +RUN mkdir -p /iginx_client/data +VOLUME /iginx_client/logs +ENV PATH="/iginx_client/sbin/:${PATH}" +ENTRYPOINT ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/docker/client/Dockerfile-no-maven-windows b/docker/client/Dockerfile-no-maven-windows new file mode 100644 index 0000000000..251678cf37 --- /dev/null +++ b/docker/client/Dockerfile-no-maven-windows @@ -0,0 +1,22 @@ +# Build client package with mvn before building the image +FROM mcr.microsoft.com/windows/servercore:ltsc2022 +COPY . C:/iginx +WORKDIR C:/iginx +COPY ./target/iginx-client-0.6.0-SNAPSHOT/ C:/iginx_client + +# 设置环境变量 +USER ContainerAdministrator +ENV JAVA_HOME "C:\jdk\jdk-11.0.2" +RUN setx /M PATH "%JAVA_HOME%\bin;%PATH%" +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +# 下载 JDK +ADD https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_windows-x64_bin.zip C:/temp/jdk.zip + +# 解压 JDK +RUN Expand-Archive -Path C:/temp/jdk.zip -DestinationPath C:/jdk + +RUN New-Item -Path 'C:/iginx_client/logs' -ItemType Directory +RUN New-Item -Path 'C:/iginx_client/data' -ItemType Directory + +ENTRYPOINT ["cmd", "/c", "ping -t localhost > NUL"] \ No newline at end of file diff --git a/docker/client/README.md b/docker/client/README.md new file mode 100644 index 0000000000..7f15a2ea54 --- /dev/null +++ b/docker/client/README.md @@ -0,0 +1,47 @@ +# build image + +```bash +build.sh(unix) +# or +build-no-maven.bat(windows) +# or +build.bat(linux container on WindowsOS) +``` + +On Windows, user need to run `mvn clean package -Dmaven.test.skip=true -P-format` in root directory before +running the script. + +If user can use linux container on their windowsOS, they can run `build.bat` without `mvn package` + +# run container + +Run the container: + +```bash +run_docker.bat/.sh -n --datadir +``` + +- will be mounted as `/iginx_client/data` in container(default: %SCRIPT_PATH%/data) + +# run client and get the terminal + +```bash +# minimum config(if IGinX server runs on the host machine) +docker exec -it iginx-client /iginx_client/sbin/start_cli.sh -h host.docker.internal + +# full config +docker exec -it iginx-client /iginx_client/sbin/start_cli.sh -h host.docker.internal -p 6888 -u root -pw root +``` + +- **-h host.docker.internal**: to access IGinX that runs on host. Replace it with real ip address if necessary. +- **iginx-client** should be replaced by customized container name. +- **-p 6888 -u root -pw root**: optional. Default values are set. + +# run client and execute statements + +```bash +# example +docker exec iginx-client /iginx_client/sbin/start_cli.sh -h host.docker.internal -e "show cluster info; show functions;" +``` + +Don't forget the DOUBLE-QUOTES around statements. Statements are separated and ended with semicolon. diff --git a/docker/client/build-no-maven.bat b/docker/client/build-no-maven.bat new file mode 100644 index 0000000000..02015e96a7 --- /dev/null +++ b/docker/client/build-no-maven.bat @@ -0,0 +1 @@ +docker build --file Dockerfile-no-maven-windows -t iginx-client:0.6.0 ../../client \ No newline at end of file diff --git a/docker/client/build-no-maven.sh b/docker/client/build-no-maven.sh new file mode 100644 index 0000000000..961ecc07e8 --- /dev/null +++ b/docker/client/build-no-maven.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker build --file Dockerfile-no-maven -t iginx-client:0.6.0 ../../client \ No newline at end of file diff --git a/docker/client/build.bat b/docker/client/build.bat new file mode 100644 index 0000000000..2861d26462 --- /dev/null +++ b/docker/client/build.bat @@ -0,0 +1 @@ +docker build --file Dockerfile -t iginx-client:0.6.0 ../../client \ No newline at end of file diff --git a/docker/client/build.sh b/docker/client/build.sh new file mode 100644 index 0000000000..9f0a707b1a --- /dev/null +++ b/docker/client/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker build --file Dockerfile -t iginx-client:0.6.0 ../../client \ No newline at end of file diff --git a/docker/client/run_docker.bat b/docker/client/run_docker.bat new file mode 100644 index 0000000000..ea633324b5 --- /dev/null +++ b/docker/client/run_docker.bat @@ -0,0 +1,41 @@ +@echo off +setlocal enabledelayedexpansion + +set current_dir=%CD% +set double_slash_dir=%current_dir:\=/% + +set name=iginx-client +set datadir=%double_slash_dir%/data + +:parse_args +if "%~1"=="" goto end_parse_args +if "%~1"=="-n" ( + set name=%~2 + shift + ) +if "%~1"=="--datadir" ( + set datadir=%~2 + shift + ) +if "%~1"=="--ip" ( + set ip=%~2 + shift + ) +if "%~1"=="--net" ( + set net=%~2 + shift + ) +shift +goto parse_args +:end_parse_args + + +if not exist "%datadir%" ( + mkdir "%datadir%" +) + +set command=docker run --name="%name%" -dit --add-host=host.docker.internal:host-gateway --mount type=bind,source=!datadir!,target=C:/iginx_client/data iginx-client:0.6.0 +echo %command% +%command% + +endlocal \ No newline at end of file diff --git a/docker/client/run_docker.sh b/docker/client/run_docker.sh new file mode 100644 index 0000000000..5384ea95c4 --- /dev/null +++ b/docker/client/run_docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash +datadir="$(pwd)/data" +name="iginx-client" + +while [ "$1" != "" ]; do + case $1 in + -n ) shift + name=$1 + ;; + --datadir ) + shift + datadir="$1" + ;; + * ) + echo "Invalid option: $1" + exit 1 + ;; + esac + shift +done + +[ -d "$datadir" ] || mkdir -p "$datadir" + +command="docker run --name=\"$name\" --privileged -dit --add-host=host.docker.internal:host-gateway --mount type=bind,source=${datadir},target=/iginx_client/data iginx-client:0.6.0" +echo $command +eval $command diff --git a/docker/onlyIginx-parquet/Dockerfile-iginx b/docker/onlyIginx-parquet/Dockerfile-iginx index 1565f4231c..df05c79099 100644 --- a/docker/onlyIginx-parquet/Dockerfile-iginx +++ b/docker/onlyIginx-parquet/Dockerfile-iginx @@ -1,7 +1,7 @@ FROM maven:3-amazoncorretto-8 AS builder COPY . /root/iginx WORKDIR /root/iginx -RUN mvn clean package -DskipTests -P passFormat -e +RUN mvn clean package -DskipTests -P-format -e FROM openjdk:11-jre-slim COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx diff --git a/docker/onlyIginx/.dockerignore b/docker/onlyIginx/.dockerignore new file mode 100644 index 0000000000..40b878db5b --- /dev/null +++ b/docker/onlyIginx/.dockerignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file diff --git a/docker/onlyIginx/Dockerfile-iginx b/docker/onlyIginx/Dockerfile-iginx index 22ac79a858..6c37adca7c 100644 --- a/docker/onlyIginx/Dockerfile-iginx +++ b/docker/onlyIginx/Dockerfile-iginx @@ -1,12 +1,20 @@ FROM maven:3-amazoncorretto-8 AS builder COPY . /root/iginx WORKDIR /root/iginx -RUN mvn clean package -DskipTests +RUN mvn clean package -DskipTests -P-format FROM openjdk:11-jre-slim COPY --from=builder /root/iginx/core/target/iginx-core-0.6.0-SNAPSHOT /iginx +# 安装 Python 3.8, pip 并安装所需的 Python 包 + +RUN apt-get update && \ + apt-get install -y python3 python3-pip && \ + python3 -m pip install --upgrade pip && \ + python3 -m pip install --no-cache-dir pandas numpy pemja thrift EXPOSE 6888 VOLUME /iginx/logs ENV PATH="/iginx/sbin/:${PATH}" +RUN mkdir -p /usr/lib/python3.9/site-packages +RUN cp -r /usr/local/lib/python3.9/dist-packages/* /usr/lib/python3.9/site-packages/ ENTRYPOINT /iginx/sbin/start_iginx.sh > /iginx/logs/iginx.log 2>&1 \ No newline at end of file diff --git a/docker/onlyIginx/run_iginx_docker.bat b/docker/onlyIginx/run_iginx_docker.bat index bf5e90d1e8..6e275635fd 100644 --- a/docker/onlyIginx/run_iginx_docker.bat +++ b/docker/onlyIginx/run_iginx_docker.bat @@ -1,4 +1,14 @@ @echo off -set name=%1 -set port=%2 -docker run --name="%name%" --privileged -dit -p %port%:6888 iginx:0.6.0 \ No newline at end of file +set "current_dir=%CD%" +@REM 将路径中的单反斜线替换为双反斜线 +set "double_slash_dir=%current_dir:\=\\%" + +@REM 初始化变量 +set "name=iginx-client" +set "logdir=%double_slash_dir%\\..\\..\\logs\\docker_logs" + +set ip=%1 +set name=%2 +set port=%3 +mkdir -p logs/docker_logs +docker run --name="%name%" --privileged -dit --net docker-cluster-iginx --ip %ip% --add-host=host.docker.internal:host-gateway -v %logdir%:/iginx/logs/ -p %port%:6888 iginx:0.6.0 \ No newline at end of file diff --git a/docker/onlyIginx/run_iginx_docker.sh b/docker/onlyIginx/run_iginx_docker.sh index 95d9c104a0..c22cdebe2f 100755 --- a/docker/onlyIginx/run_iginx_docker.sh +++ b/docker/onlyIginx/run_iginx_docker.sh @@ -1,3 +1,6 @@ ip=$1 -port=$2 -docker run --name="iginx0" --privileged -dit --net docker-cluster-iginx --ip ${ip} -p ${port}:6888 iginx:0.6.0 \ No newline at end of file +name=$2 +port=$3 +logdir="$(pwd)/../../logs/docker_logs" +mkdir -p $logdir +docker run --name="${name}" --privileged -dit --net docker-cluster-iginx --ip ${ip} --add-host=host.docker.internal:host-gateway -v ${logdir}:/iginx/logs/ -p ${port}:6888 iginx:0.6.0 \ No newline at end of file diff --git a/docs/quickStarts/IGinXManual-EnglishVersion.md b/docs/quickStarts/IGinXManual-EnglishVersion.md index 0e1dab0dd6..56b3ebfbfd 100644 --- a/docs/quickStarts/IGinXManual-EnglishVersion.md +++ b/docs/quickStarts/IGinXManual-EnglishVersion.md @@ -305,21 +305,21 @@ In order to facilitate installation and management of IGinX, IGinX provides user #### IGinX Configuration -| Configuration Item | Description | Configuration | -|------------------------------|-----------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ip | iginx ip bounds | 0.0.0.0 | -| port | iginx back-end port | 6888 | -| username | iginx username | root | -| password | iginx password | root | -| storageEngineList | Time series database list, use ',' to separate different instances | 127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false | -| maxAsyncRetryTimes | The maximum number of repetitions of asynchronous requests | 3 | -| asyncExecuteThreadPool | Asynchronous execution concurrent number | 20 | -| syncExecuteThreadPool | The number of concurrent executions | 60 | -| replicaNum | number of copies written | 1 | -| databaseClassNames | The underlying database class name, use ',' to separate different databases | iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage | -| policyClassName | Policy class name | cn.edu.tsinghua.iginx.policy.naive.NaivePolicy | -| statisticsCollectorClassName | Statistics collection class | cn.edu.tsinghua.iginx.statistics.StatisticsCollector | -| statisticsLogInterval | Statistics print interval, in milliseconds | 1000 | +| Configuration Item | Description | Configuration | +|------------------------------|-----------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ip | iginx ip bounds | 0.0.0.0 | +| port | iginx back-end port | 6888 | +| username | iginx username | root | +| password | iginx password | root | +| storageEngineList | Time series database list, use ',' to separate different instances | 127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false | +| maxAsyncRetryTimes | The maximum number of repetitions of asynchronous requests | 3 | +| asyncExecuteThreadPool | Asynchronous execution concurrent number | 20 | +| syncExecuteThreadPool | The number of concurrent executions | 60 | +| replicaNum | number of copies written | 1 | +| databaseClassNames | The underlying database class name, use ',' to separate different databases | iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationAbstractStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage | +| policyClassName | Policy class name | cn.edu.tsinghua.iginx.policy.naive.NaivePolicy | +| statisticsCollectorClassName | Statistics collection class | cn.edu.tsinghua.iginx.statistics.StatisticsCollector | +| statisticsLogInterval | Statistics print interval, in milliseconds | 1000 | #### Rest Configuration diff --git a/docs/quickStarts/IGinXManual.md b/docs/quickStarts/IGinXManual.md index 7e79a5a095..12eeab3b7d 100644 --- a/docs/quickStarts/IGinXManual.md +++ b/docs/quickStarts/IGinXManual.md @@ -304,21 +304,21 @@ IGinX、Rest、元数据管理三方面配置。 #### IGinX 配置 -| 配置项 | 描述 | 默认值 | -|------------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ip | iginx 绑定的 ip | 0.0.0.0 | -| port | iginx 绑定的端口 | 6888 | -| username | iginx 本身的用户名 | root | -| password | iginx 本身的密码 | root | -| storageEngineList | 时序数据库列表，使用','分隔不同实例 | 127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false | -| maxAsyncRetryTimes | 异步请求最大重复次数 | 3 | -| asyncExecuteThreadPool | 异步执行并发数 | 20 | -| syncExecuteThreadPool | 同步执行并发数 | 60 | -| replicaNum | 写入的副本个数 | 1 | -| databaseClassNames | 底层数据库类名，使用','分隔不同数据库 | iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,postgresql=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage | -| policyClassName | 策略类名 | cn.edu.tsinghua.iginx.policy.naive.NaivePolicy | -| statisticsCollectorClassName | 统计信息收集类 | cn.edu.tsinghua.iginx.statistics.StatisticsCollector | -| statisticsLogInterval | 统计信息打印间隔，单位毫秒 | 1000 | +| 配置项 | 描述 | 默认值 | +|------------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ip | iginx 绑定的 ip | 0.0.0.0 | +| port | iginx 绑定的端口 | 6888 | +| username | iginx 本身的用户名 | root | +| password | iginx 本身的密码 | root | +| storageEngineList | 时序数据库列表，使用','分隔不同实例 | 127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false | +| maxAsyncRetryTimes | 异步请求最大重复次数 | 3 | +| asyncExecuteThreadPool | 异步执行并发数 | 20 | +| syncExecuteThreadPool | 同步执行并发数 | 60 | +| replicaNum | 写入的副本个数 | 1 | +| databaseClassNames | 底层数据库类名，使用','分隔不同数据库 | iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,parquet=cn.edu.tsinghua.iginx.parquet.ParquetStorage,relational=cn.edu.tsinghua.iginx.relational.RelationAbstractStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage | +| policyClassName | 策略类名 | cn.edu.tsinghua.iginx.policy.naive.NaivePolicy | +| statisticsCollectorClassName | 统计信息收集类 | cn.edu.tsinghua.iginx.statistics.StatisticsCollector | +| statisticsLogInterval | 统计信息打印间隔，单位毫秒 | 1000 | #### Rest 配置 diff --git a/pom.xml b/pom.xml index 69888d1fd5..bdca6a88c3 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,11 @@ postgresql ${project.version} + + cn.edu.tsinghua + relational + 0.6.0-SNAPSHOT + cn.edu.tsinghua redis diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java index 5c7cb3d573..c7215fc5fd 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/Session.java @@ -19,12 +19,15 @@ package cn.edu.tsinghua.iginx.session; import static cn.edu.tsinghua.iginx.utils.ByteUtils.getByteArrayFromLongArray; +import static cn.edu.tsinghua.iginx.utils.HostUtils.isLocalHost; import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.thrift.*; import cn.edu.tsinghua.iginx.utils.*; import java.io.File; +import java.io.IOException; import java.nio.ByteBuffer; +import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -34,8 +37,6 @@ import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.ArrayUtils; import org.apache.thrift.TException; @@ -922,33 +923,6 @@ public SessionExecuteSqlResult executeSql(String statement) throws SessionExcept return new SessionExecuteSqlResult(ref.resp); } - public SessionExecuteSqlResult executePythonRegister(String statement) throws SessionException { - Pattern pattern = Pattern.compile("\"([^\"]*)\""); - Matcher matcher = pattern.matcher(statement); - - // 1st "": sql name - if (!matcher.find()) { - throw new SessionException("Error: function name should be surrounded by DOUBLE-QUOTES"); - } - // 2nd "": class name - if (!matcher.find()) { - throw new SessionException("Error: python class name should be surrounded by DOUBLE-QUOTES"); - } - // 3rd "": script(s) file path - if (matcher.find()) { - // 提取python文件路径 - String filePathStr = matcher.group(1); - - File filePath = new File(filePathStr); - if (!filePath.isAbsolute()) { - statement = statement.replace(filePathStr, filePath.getAbsolutePath()); - } - return executeSql(statement); - } else { - throw new SessionException("Error: python file path should be surrounded by DOUBLE-QUOTES"); - } - } - public SessionQueryDataSet queryLast( List paths, long startKey, TimePrecision timePrecision) throws SessionException { return queryLast(paths, startKey, null, timePrecision); @@ -1104,6 +1078,55 @@ public Pair, Long> executeLoadCSV(String statement, ByteBuffer csvF return new Pair<>(ref.resp.getColumns(), ref.resp.getRecordsNum()); } + public LoadUDFResp executeRegisterTask(String statement) throws SessionException { + return executeRegisterTask(statement, !isLocalHost(host)); + } + + public LoadUDFResp executeRegisterTask(String statement, boolean isRemote) + throws SessionException { + LoadUDFReq req = new LoadUDFReq(sessionId, statement, isRemote); + Reference ref = new Reference<>(); + executeWithCheck(() -> (ref.resp = client.loadUDF(req)).status); + + LoadUDFResp res = ref.resp; + String parseErrorMsg = res.getParseErrorMsg(); + if (parseErrorMsg != null && !parseErrorMsg.equals("")) { + return new LoadUDFResp(RpcUtils.FAILURE.setMessage(parseErrorMsg)); + } + String path = res.getUDFModulePath(); + File file = new File(path); + if (!file.isAbsolute()) { + statement = statement.replace(path, file.getAbsolutePath()); + } else if (!isRemote) { + return new LoadUDFResp(RpcUtils.SUCCESS); + } + + if (!file.exists()) { + throw new InvalidParameterException(path + " does not exist!"); + } + + ByteBuffer moduleBuffer; + if (isRemote) { + try { + moduleBuffer = CompressionUtils.zipToByteBuffer(file); + } catch (IOException e) { + return new LoadUDFResp( + RpcUtils.FAILURE.setMessage( + String.format( + "Failed to compress module and load into buffer. %s", e.getMessage()))); + } + } else { + moduleBuffer = ByteBuffer.allocate(0); + } + + LoadUDFReq newReq = new LoadUDFReq(sessionId, statement, isRemote); + newReq.setUdfFile(moduleBuffer); + Reference newRef = new Reference<>(); + executeWithCheck(() -> (newRef.resp = client.loadUDF(newReq)).status); + + return newRef.resp; + } + void closeQuery(long queryId) throws SessionException { CloseStatementReq req = new CloseStatementReq(sessionId, queryId); executeWithCheck(() -> client.closeStatement(req)); diff --git a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java index 319801072c..453a5de818 100644 --- a/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java +++ b/session/src/main/java/cn/edu/tsinghua/iginx/session/SessionExecuteSqlResult.java @@ -45,6 +45,7 @@ public class SessionExecuteSqlResult { private List jobIdList; private Map configs; private String loadCsvPath; + private String UDFModulePath; private List sessionIDs; private Map rules; @@ -106,6 +107,8 @@ public SessionExecuteSqlResult(ExecuteSqlResp resp) { break; case LoadCsv: this.loadCsvPath = resp.getLoadCsvPath(); + case RegisterTask: + this.UDFModulePath = resp.getUDFModulePath(); case ShowSessionID: this.sessionIDs = resp.getSessionIDList(); case ShowRules: @@ -221,7 +224,7 @@ private List> cacheResult( } for (int i = 0; i < paths.size(); i++) { String path = paths.get(i); - if (!path.equals("TITLE.DESCRIPTION")) { // TODO 不展示系统级时间序列 + if (!path.equals("title.description")) { // TODO 不展示系统级时间序列 label.add(path); } else { annotationPathIndex = i; @@ -273,7 +276,7 @@ private String buildShowColumnsResult() { List> cache = new ArrayList<>(); cache.add(new ArrayList<>(Arrays.asList("Path", "DataType"))); for (int i = 0; i < paths.size(); i++) { - if (!paths.get(i).equals("TITLE.DESCRIPTION")) { // TODO 不展示系统级时间序列 + if (!paths.get(i).equals("title.description")) { // TODO 不展示系统级时间序列 cache.add(new ArrayList<>(Arrays.asList(paths.get(i), dataTypeList.get(i).toString()))); num++; } @@ -567,6 +570,10 @@ public String getLoadCsvPath() { return loadCsvPath; } + public String getUDFModulePath() { + return UDFModulePath; + } + public List getSessionIDs() { return sessionIDs; } diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java new file mode 100644 index 0000000000..da7fd16174 --- /dev/null +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/CompressionUtils.java @@ -0,0 +1,101 @@ +package cn.edu.tsinghua.iginx.utils; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.zip.*; + +/** tools for file compression and decompression. */ +public class CompressionUtils { + /** + * compress a file or a folder into ByteBuffer + * + * @param fileOrFolder a file or a folder to be compressed + * @return compressed ByteBuffer + */ + public static ByteBuffer zipToByteBuffer(File fileOrFolder) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + ZipOutputStream zipStream = new ZipOutputStream(byteStream); + + compressFile(fileOrFolder, fileOrFolder.getName(), zipStream); + zipStream.close(); + byteStream.close(); + + byte[] bytes = byteStream.toByteArray(); + return ByteBuffer.wrap(bytes); + } + + private static void compressFile(File file, String path, ZipOutputStream zipOut) + throws IOException { + if (file.isDirectory()) { + if (path.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(path)); + zipOut.closeEntry(); + } else { + zipOut.putNextEntry(new ZipEntry(path + "/")); + zipOut.closeEntry(); + } + File[] children = file.listFiles(); + if (children != null) { + for (File childFile : children) { + compressFile(childFile, path + "/" + childFile.getName(), zipOut); + } + } + return; + } + FileInputStream fis = new FileInputStream(file); + ZipEntry zipEntry = new ZipEntry(path); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + fis.close(); + zipOut.closeEntry(); + } + + /** + * decompress a file or a folder from ByteBuffer + * + * @param buffer ByteBuffer that holds the file content + * @param destination a folder to place the decompressed file or folder. The file or folder uses + * original name. The write permission of destination must be checked before this method is + * called + */ + public static void unzipFromByteBuffer(ByteBuffer buffer, File destination) throws IOException { + assert !destination.getPath().contains(".."); + InputStream inputStream = new ByteArrayInputStream(buffer.array()); + ZipInputStream zipIn = new ZipInputStream(inputStream); + + ZipEntry entry = zipIn.getNextEntry(); + while (entry != null) { + File fileDest = new File(destination, entry.getName()); + if (!fileDest.toPath().normalize().startsWith(destination.toPath())) + throw new IOException("Illegal file path found during unzipping operation."); + if (entry.isDirectory()) { + fileDest.mkdirs(); + } else { + fileDest.getParentFile().mkdirs(); + extractFile(zipIn, fileDest); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + zipIn.close(); + inputStream.close(); + } + + private static void extractFile(ZipInputStream zipIn, File file) throws IOException { + if (!file.exists() && !file.createNewFile()) { + throw new IOException("cannot create file: " + file.getPath()); + } + BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(file.toPath())); + byte[] bytesIn = new byte[1024]; + int read = 0; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + bos.close(); + } +} diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java new file mode 100644 index 0000000000..a86a777c25 --- /dev/null +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/FileCompareUtils.java @@ -0,0 +1,76 @@ +package cn.edu.tsinghua.iginx.utils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.io.IOUtils; + +public class FileCompareUtils { + + /** + * Compare contents of two files + * + * @return true if two files are identically same; false otherwise. + */ + public static boolean compareFile(File file1, File file2) throws IOException { + if (file1.exists() && file2.exists() && file1.isFile() && file2.isFile()) { + try (BufferedReader br1 = new BufferedReader(new java.io.FileReader(file1)); + BufferedReader br2 = new BufferedReader(new FileReader(file2))) { + return IOUtils.contentEqualsIgnoreEOL(br1, br2); + } + } else { + throw new IOException( + String.format( + "files not exist or are not files: %s & %s.", file1.getPath(), file2.getPath())); + } + } + + /** + * Compare contents of two folders + * + * @return true if two folders are identically same; false otherwise. + */ + public static boolean compareFolder(File folder1, File folder2) throws IOException { + if (folder1.exists() && folder1.isDirectory() && folder2.exists() && folder2.isDirectory()) { + try (Stream stream1 = Files.walk(folder1.toPath()); + Stream stream2 = Files.walk(folder2.toPath())) { + List list1 = stream1.collect(Collectors.toList()); + List list2 = stream2.collect(Collectors.toList()); + + if (list1.size() != list2.size()) { + return false; + } + + for (int i = 0; i < list1.size(); i++) { + Path path1 = list1.get(i); + Path path2 = list2.get(i); + Path relativePath1 = folder1.toPath().relativize(path1); + Path relativePath2 = folder2.toPath().relativize(path2); + + if (!relativePath1.equals(relativePath2) + || Files.isDirectory(path1) != Files.isDirectory(path2) + || Files.isRegularFile(path1) != Files.isRegularFile(path2)) { + return false; + } + + if (Files.isRegularFile(path1) && Files.isRegularFile(path2)) { + if (!compareFile(path1.toFile(), path2.toFile())) { + return false; + } + } + } + } + return true; + } else { + throw new IOException( + String.format( + "Dirs not exist or are not dirs: %s & %s.", folder1.getPath(), folder2.getPath())); + } + } +} diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java index b368d497c7..e714b8186d 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/HostUtils.java @@ -60,6 +60,10 @@ public static boolean isValidHost(String host) { if (host == null || host.trim().isEmpty()) { return false; } + if (host.equals("host.docker.internal")) { + // using docker + return true; + } if (!IPV4_PATTERN.matcher(host).matches()) { return false; } diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java new file mode 100644 index 0000000000..cf8a9c5283 --- /dev/null +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/YAMLWriter.java @@ -0,0 +1,48 @@ +package cn.edu.tsinghua.iginx.utils; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; + +public class YAMLWriter { + private final Yaml yaml; + + private static final Logger LOGGER = LoggerFactory.getLogger(YAMLWriter.class); + + private static final DumperOptions options = new DumperOptions(); + + public YAMLWriter() { + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + options.setPrettyFlow(true); + options.setIndent(2); + options.setExplicitStart(true); + LoaderOptions loaderOptions = new LoaderOptions(); + Representer representer = new Representer(options); + representer.addClassTag(JobFromYAML.class, Tag.MAP); + representer.setDefaultScalarStyle(DumperOptions.ScalarStyle.DOUBLE_QUOTED); + + this.yaml = new Yaml(new Constructor(JobFromYAML.class, loaderOptions), representer, options); + } + + /** + * write job information into given yaml file. + * + * @param file target file to write + * @param job job to dump + */ + public void writeJobIntoYAML(File file, JobFromYAML job) throws IOException { + assert file.exists() && file.isFile(); + try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) { + yaml.dump(job, bw); + } + } +} diff --git a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java new file mode 100644 index 0000000000..cdf145a76e --- /dev/null +++ b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/CompressionUtilsTest.java @@ -0,0 +1,131 @@ +package cn.edu.tsinghua.iginx.utils; + +import static org.junit.Assert.fail; + +import java.io.*; +import java.nio.ByteBuffer; +import org.junit.AfterClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CompressionUtilsTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(CompressionUtilsTest.class); + + private static final String resourcePath = + String.join( + File.separator, + System.getProperty("user.dir"), + "src", + "test", + "resources", + "compressionTest"); + + @AfterClass + public static void deleteTestFolder() { + try { + FileUtils.deleteFolder(new File(resourcePath)); + } catch (IOException e) { + LOGGER.error("failed to delete test folder:{}.", resourcePath, e); + } + } + + // test compress file and decompress + @Test + public void testCompressFile() throws IOException { + String testName = "compressionFileTest"; + File sourceFile = new File(String.join(File.separator, resourcePath, testName, "source")); + File destFolder = new File(String.join(File.separator, resourcePath, testName, "dest")); + + if (!sourceFile.exists() + && !sourceFile.getParentFile().mkdirs() + && !sourceFile.createNewFile()) { + fail(); + } + + FileUtils.writeFile(sourceFile, "testing...中文", "new行"); + ByteBuffer buffer = CompressionUtils.zipToByteBuffer(sourceFile); + CompressionUtils.unzipFromByteBuffer(buffer, destFolder); + + File destFile = new File(String.join(File.separator, resourcePath, testName, "dest", "source")); + try { + if (!FileCompareUtils.compareFile(sourceFile, destFile)) { + fail("unzipped file doesn't match original file"); + } + } catch (IOException e) { + LOGGER.error( + "Failed to compare contents of file: {} and file: {}", + sourceFile.getPath(), + destFile.getPath()); + } + + FileUtils.deleteFolder(new File(String.join(File.separator, resourcePath, testName))); + } + + // test compress folder and decompress + @Test + public void testCompressFolder() throws IOException { + String testName = "compressionFolderTest"; + String sourceFolderPath = String.join(File.separator, resourcePath, testName, "source"); + String destFolderPath = String.join(File.separator, resourcePath, testName, "dest"); + File sourceFolder = new File(sourceFolderPath); + File destFolder = new File(destFolderPath); + + File sourceFile1 = new File(String.join(File.separator, sourceFolderPath, "file1")); + File sourceFile2 = new File(String.join(File.separator, sourceFolderPath, "folder1", "file2")); + try { + if (!sourceFile1.exists() + && !sourceFile1.getParentFile().mkdirs() + && !sourceFile1.createNewFile()) { + fail(); + } + if (!sourceFile2.exists() + && !sourceFile2.getParentFile().mkdirs() + && !sourceFile2.createNewFile()) { + fail(); + } + } catch (IOException e) { + LOGGER.error("can't create file.", e); + fail(); + } + + FileUtils.writeFile(sourceFile1, "testing...中文111", "new行11111"); + FileUtils.writeFile(sourceFile2, "testing...中文222", "new行22222"); + ByteBuffer buffer = CompressionUtils.zipToByteBuffer(sourceFolder); + CompressionUtils.unzipFromByteBuffer(buffer, destFolder); + + if (!FileCompareUtils.compareFolder( + sourceFolder, new File(String.join(File.separator, destFolderPath, "source")))) { + fail("unzipped folder doesn't match original folder"); + } + + FileUtils.deleteFolder(new File(String.join(File.separator, resourcePath, testName))); + } + + // test compress empty folder and decompress + @Test + public void testCompressEmptyFolder() throws IOException { + String testName = "compressionFolderTest"; + String sourceFolderPath = String.join(File.separator, resourcePath, testName, "source"); + String destFolderPath = String.join(File.separator, resourcePath, testName, "dest"); + File sourceFolder = new File(sourceFolderPath); + File destFolder = new File(destFolderPath); + + if (!sourceFolder.exists() && !sourceFolder.mkdirs()) { + fail(); + } + if (!destFolder.exists() && !destFolder.mkdirs()) { + fail(); + } + ByteBuffer buffer = CompressionUtils.zipToByteBuffer(sourceFolder); + CompressionUtils.unzipFromByteBuffer(buffer, destFolder); + + if (!FileCompareUtils.compareFolder( + sourceFolder, new File(String.join(File.separator, destFolderPath, "source")))) { + fail("unzipped folder doesn't match original folder"); + } + + FileUtils.deleteFolder(new File(String.join(File.separator, resourcePath, testName))); + } +} diff --git a/test/pom.xml b/test/pom.xml index 5e98a815ac..d2d0c8dc6c 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -46,7 +46,7 @@ cn.edu.tsinghua - postgresql + relational cn.edu.tsinghua diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java index b1790ad0a4..b5231a1d69 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/controller/Controller.java @@ -205,7 +205,7 @@ public static void writeColumnsData( LOGGER.error("write data fail, caused by generator is null"); return; } - if (StorageEngineType.valueOf(conf.getStorageType().toLowerCase()) == parquet) { + if (StorageEngineType.valueOf(conf.getStorageType(false).toLowerCase()) == parquet) { ParquetHistoryDataGenerator parquetGenerator = (ParquetHistoryDataGenerator) generator; String path = pathList.get(i); String tableName = path.substring(0, path.indexOf(".")); @@ -249,7 +249,7 @@ public static void writeRowsDataToDummy( LOGGER.error("write data fail, caused by generator is null"); return; } - if (StorageEngineType.valueOf(conf.getStorageType().toLowerCase()) == parquet) { + if (StorageEngineType.valueOf(conf.getStorageType(false).toLowerCase()) == parquet) { ParquetHistoryDataGenerator parquetGenerator = (ParquetHistoryDataGenerator) generator; String path = pathList.get(0); String tableName = path.substring(0, path.indexOf(".")); @@ -463,7 +463,7 @@ public void testUnion() throws Exception { envir.setTestTasks( testConfLoader .getTaskMap() - .get(StorageEngineType.valueOf(testConfLoader.getStorageType().toLowerCase())), + .get(StorageEngineType.valueOf(testConfLoader.getStorageType(false).toLowerCase())), TEST_TASK_FILE); // run the test together shellRunner.runShellCommand(MVN_RUN_TEST); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java index 7df4214f9d..393f3d55b3 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseCapacityExpansionIT.java @@ -12,6 +12,7 @@ import cn.edu.tsinghua.iginx.integration.expansion.influxdb.InfluxDBCapacityExpansionIT; import cn.edu.tsinghua.iginx.integration.expansion.parquet.ParquetCapacityExpansionIT; import cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools; +import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; import cn.edu.tsinghua.iginx.session.ClusterInfo; import cn.edu.tsinghua.iginx.session.QueryDataSet; import cn.edu.tsinghua.iginx.session.Session; @@ -32,10 +33,14 @@ public abstract class BaseCapacityExpansionIT { protected static Session session; + private static final ConfLoader testConf = new ConfLoader(Controller.CONFIG_FILE); + protected StorageEngineType type; protected String extraParams; + protected List wrongExtraParams = new ArrayList<>(); + private final boolean IS_PARQUET_OR_FILE_SYSTEM = this instanceof FileSystemCapacityExpansionIT || this instanceof ParquetCapacityExpansionIT; @@ -45,13 +50,22 @@ public abstract class BaseCapacityExpansionIT { public static final String DBCE_PARQUET_FS_TEST_DIR = "test"; - public BaseCapacityExpansionIT(StorageEngineType type, String extraParams) { + protected static BaseHistoryDataGenerator generator; + + public BaseCapacityExpansionIT( + StorageEngineType type, String extraParams, BaseHistoryDataGenerator generator) { this.type = type; this.extraParams = extraParams; + BaseCapacityExpansionIT.generator = generator; } protected String addStorageEngine( - int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) { + int port, + boolean hasData, + boolean isReadOnly, + String dataPrefix, + String schemaPrefix, + String extraParams) { try { StringBuilder statement = new StringBuilder(); statement.append("ADD STORAGEENGINE (\"127.0.0.1\", "); @@ -94,13 +108,14 @@ protected String addStorageEngine( return null; } catch (SessionException e) { LOGGER.warn( - "add storage engine {} port {} hasData {} isReadOnly {} dataPrefix {} schemaPrefix {} failure: ", + "add storage engine:{} port:{} hasData:{} isReadOnly:{} dataPrefix:{} schemaPrefix:{} extraParams:{} failure: ", type.name(), port, hasData, isReadOnly, dataPrefix, schemaPrefix, + extraParams, e); return e.getMessage(); } @@ -137,7 +152,7 @@ private void addStorageEngineInProgress( startStorageEngineWithIginx(port, hasData, isReadOnly); } else { // 测试会添加初始数据，所以hasData=true - addStorageEngine(port, hasData, isReadOnly, dataPrefix, schemaPrefix); + addStorageEngine(port, hasData, isReadOnly, dataPrefix, schemaPrefix, extraParams); } } @@ -214,10 +229,14 @@ public void oriNoDataExpNoData() throws InterruptedException { public void testReadOnly() throws InterruptedException { // 查询原始只读节点的历史数据，结果不为空 testQueryHistoryDataOriHasData(); + // 测试参数错误的只读节点扩容 + testInvalidDummyParams(readOnlyPort, true, false, null, EXP_SCHEMA_PREFIX); // 扩容只读节点 addStorageEngineInProgress(readOnlyPort, true, true, null, READ_ONLY_SCHEMA_PREFIX); // 查询扩容只读节点的历史数据，结果不为空 testQueryHistoryDataReadOnly(); + // 测试参数错误的可写节点扩容 + testInvalidDummyParams(expPort, true, false, null, EXP_SCHEMA_PREFIX); // 扩容可写节点 addStorageEngineInProgress(expPort, true, false, null, EXP_SCHEMA_PREFIX); // 查询扩容可写节点的历史数据，结果不为空 @@ -232,11 +251,95 @@ public void testReadOnly() throws InterruptedException { if (this instanceof FileSystemCapacityExpansionIT) { // 仅用于扩容文件系统后查询文件 testQueryForFileSystem(); - // TODO 扩容后show columns测试 - testShowColumnsForFileSystem(); + } + + // 扩容后show columns测试 + testShowColumns(); + + // clear data first, because history generator cannot append. It can only write + clearData(); + generator.clearHistoryData(); + + // 向三个dummy数据库中追加dummy数据，数据的key和列名都在添加数据库时的范围之外 + generator.writeExtendDummyData(); + // 能查到key在初始范围外的数据，查不到列名在初始范围外的数据 + queryExtendedKeyDummy(); + queryExtendedColDummy(); + } + + protected void testInvalidDummyParams( + int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) { + // wrong params + String res; + for (String params : wrongExtraParams) { + res = addStorageEngine(port, hasData, isReadOnly, dataPrefix, schemaPrefix, params); + if (res != null) { + LOGGER.info( + "Successfully rejected dummy engine with wrong params: {}; {}. msg: {}", + port, + params, + res); + } else { + LOGGER.error("Dummy engine with wrong params {}; {} shouldn't be added.", port, params); + fail(); + } + } + + // wrong port + res = addStorageEngine(port + 999, hasData, isReadOnly, dataPrefix, schemaPrefix, extraParams); + if (res != null) { + LOGGER.info( + "Successfully rejected dummy engine with wrong port: {}; params: {}. msg: {}", + port + 999, + extraParams, + res); + } else { + LOGGER.error( + "Dummy engine with wrong port {} & params:{} shouldn't be added.", + port + 999, + extraParams); + fail(); } } + protected void queryExtendedKeyDummy() { + // ori + // extended key queryable + // NOTE: in some database(e.g. mongoDB), the key for dummy data is given randomly and cannot be + // controlled. Thus, when extended value can be queried without specifying key filter, + // we still assume that dummy key range is extended. + String statement = "select wf01.wt01.status, wf01.wt01.temperature from mn;"; + SQLTestTools.executeAndContainValue(session, statement, ORI_PATH_LIST, ORI_EXTEND_VALUES_LIST); + + // exp + statement = "select wf03.wt01.status2 from nt;"; + SQLTestTools.executeAndContainValue( + session, statement, EXP_PATH_LIST1, EXP_EXTEND_VALUES_LIST1); + statement = "select wf04.wt01.temperature from nt;"; + SQLTestTools.executeAndContainValue( + session, statement, EXP_PATH_LIST2, EXP_EXTEND_VALUES_LIST2); + + // ro + statement = "select wf05.wt01.status, wf05.wt01.temperature from tm;"; + SQLTestTools.executeAndContainValue( + session, statement, READ_ONLY_PATH_LIST, READ_ONLY_EXTEND_VALUES_LIST); + } + + protected void queryExtendedColDummy() { + // ori + // extended columns unreachable + String statement = "select * from a.a.a;"; + SQLTestTools.executeAndCompare(session, statement, new ArrayList<>(), new ArrayList<>()); + + // exp + statement = "select * from a.a.b;"; + SQLTestTools.executeAndCompare(session, statement, new ArrayList<>(), new ArrayList<>()); + + // ro + statement = "select * from a.a.c;"; + SQLTestTools.executeAndCompare(session, statement, new ArrayList<>(), new ArrayList<>()); + } + protected void testQuerySpecialHistoryData() {} private void testQueryHistoryDataOriHasData() { @@ -363,29 +466,29 @@ private void testAddAndRemoveStorageEngineWithPrefix() { List> valuesList = EXP_VALUES_LIST1; // 添加不同 schemaPrefix，相同 dataPrefix - addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix1); + addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix1, extraParams); // 添加节点 dataPrefix = dataPrefix1 && schemaPrefix = p1 后查询 String statement = "select status2 from *;"; List pathList = Arrays.asList("nt.wf03.wt01.status2", "p1.nt.wf03.wt01.status2"); SQLTestTools.executeAndCompare(session, statement, pathList, REPEAT_EXP_VALUES_LIST1); - addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix2); - addStorageEngine(expPort, true, true, dataPrefix1, null); + addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix2, extraParams); + addStorageEngine(expPort, true, true, dataPrefix1, null, extraParams); testShowClusterInfo(5); // 如果是重复添加，则报错 - String res = addStorageEngine(expPort, true, true, dataPrefix1, null); + String res = addStorageEngine(expPort, true, true, dataPrefix1, null, extraParams); if (res != null && !res.contains("repeatedly add storage engine")) { fail(); } testShowClusterInfo(5); - addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix3); + addStorageEngine(expPort, true, true, dataPrefix1, schemaPrefix3, extraParams); // 这里是之后待测试的点，如果添加包含关系的，应当报错。 // res = addStorageEngine(expPort, true, true, "nt.wf03.wt01", "p3"); // 添加相同 schemaPrefix，不同 dataPrefix - addStorageEngine(expPort, true, true, dataPrefix2, schemaPrefix3); + addStorageEngine(expPort, true, true, dataPrefix2, schemaPrefix3, extraParams); testShowClusterInfo(7); // 添加节点 dataPrefix = dataPrefix1 && schemaPrefix = p1 后查询 @@ -513,15 +616,17 @@ private void testQueryForFileSystem() { } } - private void testShowColumnsForFileSystem() { + // test dummy and non-dummy columns, in read only test + @Test + public void testShowColumns() { String statement = "SHOW COLUMNS mn.*;"; String expected = "Columns:\n" + "+------------------------+--------+\n" + "| Path|DataType|\n" + "+------------------------+--------+\n" - + "| mn.wf01.wt01.status| BINARY|\n" - + "|mn.wf01.wt01.temperature| BINARY|\n" + + "| mn.wf01.wt01.status| LONG|\n" + + "|mn.wf01.wt01.temperature| DOUBLE|\n" + "+------------------------+--------+\n" + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expected); @@ -532,8 +637,8 @@ private void testShowColumnsForFileSystem() { + "+------------------------+--------+\n" + "| Path|DataType|\n" + "+------------------------+--------+\n" - + "| nt.wf03.wt01.status2| BINARY|\n" - + "|nt.wf04.wt01.temperature| BINARY|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "|nt.wf04.wt01.temperature| DOUBLE|\n" + "+------------------------+--------+\n" + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expected); @@ -544,23 +649,26 @@ private void testShowColumnsForFileSystem() { + "+------------------------+--------+\n" + "| Path|DataType|\n" + "+------------------------+--------+\n" - + "| tm.wf05.wt01.status| BINARY|\n" - + "|tm.wf05.wt01.temperature| BINARY|\n" + + "| tm.wf05.wt01.status| LONG|\n" + + "|tm.wf05.wt01.temperature| DOUBLE|\n" + "+------------------------+--------+\n" + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expected); + } - statement = "SHOW COLUMNS a.*;"; - expected = + // test dummy query for data out of initial key range (should be visible) + protected void testDummyKeyRange() { + String statement; + statement = "select * from mn where key < 1;"; + String expected = "Columns:\n" - + "+-------------+--------+\n" - + "| Path|DataType|\n" - + "+-------------+--------+\n" - + "|a.b.c.d.1\\txt| BINARY|\n" - + "| a.e.2\\txt| BINARY|\n" - + "| a.f.g.3\\txt| BINARY|\n" - + "+-------------+--------+\n" - + "Total line number = 3\n"; + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| mn.wf01.wt01.status| BINARY|\n" + + "|mn.wf01.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expected); } @@ -568,12 +676,13 @@ private void testSameKeyWarning() { try { session.executeSql( "insert into mn.wf01.wt01 (key, status) values (0, 123),(1, 123),(2, 123),(3, 123);"); - String statement = "select * from mn.wf01.wt01"; + String statement = "select * from mn.wf01.wt01;"; QueryDataSet res = session.executeQuery(statement); - if ((res.getWarningMsg() == null || res.getWarningMsg().isEmpty()) - && !res.getWarningMsg().contains("The query results contain overlapped keys.") - && SUPPORT_KEY.get(type.name())) { + if ((res.getWarningMsg() == null + || res.getWarningMsg().isEmpty() + || !res.getWarningMsg().contains("The query results contain overlapped keys.")) + && SUPPORT_KEY.get(testConf.getStorageType())) { LOGGER.error("未抛出重叠key的警告"); fail(); } @@ -581,7 +690,7 @@ private void testSameKeyWarning() { clearData(); res = session.executeQuery(statement); - if (res.getWarningMsg() != null && SUPPORT_KEY.get(type.name())) { + if (res.getWarningMsg() != null && SUPPORT_KEY.get(testConf.getStorageType())) { LOGGER.error("不应抛出重叠key的警告"); fail(); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java index 8f2f31cf2a..b2660298d5 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/BaseHistoryDataGenerator.java @@ -3,6 +3,8 @@ import static cn.edu.tsinghua.iginx.integration.expansion.constant.Constant.*; import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.junit.Test; @@ -51,19 +53,57 @@ public void writeInitDataToDummy(int port) { writeHistoryData(port, INIT_PATH_LIST, INIT_DATA_TYPE_LIST, INIT_KEYS_LIST, INIT_VALUES_LIST); } + public void writeExtendDummyData() { + writeExtendedHistoryDataToOri(); + writeExtendedHistoryDataToExp(); + writeExtendedHistoryDataToReadOnly(); + } + public void writeHistoryDataToOri() { writeHistoryData(oriPort, ORI_PATH_LIST, ORI_DATA_TYPE_LIST, ORI_VALUES_LIST); } + // write dummy data that contains key & columns that are not in initial range + public void writeExtendedHistoryDataToOri() { + writeHistoryData(oriPort, ORI_EXTEND_PATH_LIST, ORI_DATA_TYPE_LIST, ORI_VALUES_LIST); + writeHistoryData( + oriPort, + ORI_PATH_LIST, + ORI_DATA_TYPE_LIST, + new ArrayList<>(Collections.singletonList(999999L)), + ORI_EXTEND_VALUES_LIST); + } + public void writeHistoryDataToExp() { writeHistoryData(expPort, EXP_PATH_LIST, EXP_DATA_TYPE_LIST, EXP_VALUES_LIST); } + public void writeExtendedHistoryDataToExp() { + writeHistoryData(expPort, EXP_EXTEND_PATH_LIST, EXP_DATA_TYPE_LIST, EXP_VALUES_LIST); + writeHistoryData( + expPort, + EXP_PATH_LIST, + EXP_DATA_TYPE_LIST, + new ArrayList<>(Collections.singletonList(999999L)), + EXP_EXTEND_VALUES_LIST); + } + public void writeHistoryDataToReadOnly() { writeHistoryData( readOnlyPort, READ_ONLY_PATH_LIST, READ_ONLY_DATA_TYPE_LIST, READ_ONLY_VALUES_LIST); } + public void writeExtendedHistoryDataToReadOnly() { + writeHistoryData( + readOnlyPort, READ_ONLY_EXTEND_PATH_LIST, READ_ONLY_DATA_TYPE_LIST, READ_ONLY_VALUES_LIST); + writeHistoryData( + readOnlyPort, + READ_ONLY_PATH_LIST, + READ_ONLY_DATA_TYPE_LIST, + new ArrayList<>(Collections.singletonList(999999L)), + READ_ONLY_EXTEND_VALUES_LIST); + } + public abstract void writeHistoryData( int port, List pathList, List dataTypeList, List> valuesList); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java index 2fd0854c14..7ef5adbc14 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/constant/Constant.java @@ -33,9 +33,15 @@ public class Constant { public static final List ORI_PATH_LIST = Arrays.asList("mn.wf01.wt01.status", "mn.wf01.wt01.temperature"); + public static final List ORI_EXTEND_PATH_LIST = + Arrays.asList("a.a.a.status", "a.a.a.temperature"); + public static final List EXP_PATH_LIST = Arrays.asList("nt.wf03.wt01.status2", "nt.wf04.wt01.temperature"); + public static final List EXP_EXTEND_PATH_LIST = + Arrays.asList("a.a.b.status2", "a.a.b.temperature"); + public static final List EXP_PATH_LIST1 = Collections.singletonList("nt.wf03.wt01.status2"); @@ -45,6 +51,9 @@ public class Constant { public static final List READ_ONLY_PATH_LIST = Arrays.asList("tm.wf05.wt01.status", "tm.wf05.wt01.temperature"); + public static final List READ_ONLY_EXTEND_PATH_LIST = + Arrays.asList("a.a.c.status", "a.a.c.temperature"); + // data type public static List INIT_DATA_TYPE_LIST = Arrays.asList(DataType.LONG, DataType.LONG); @@ -62,9 +71,21 @@ public class Constant { public static List> ORI_VALUES_LIST = Arrays.asList(Arrays.asList(11111111L, 15123.27), Arrays.asList(22222222L, 20123.71)); + public static List> ORI_EXTEND_VALUES_LIST = + Collections.singletonList(Arrays.asList(123456L, 654321.2)); + public static List> EXP_VALUES_LIST = Arrays.asList(Arrays.asList(33333333L, 66123.23), Arrays.asList(44444444L, 77123.71)); + public static List> EXP_EXTEND_VALUES_LIST = + Collections.singletonList(Arrays.asList(77777777L, 3498.1)); + + public static List> EXP_EXTEND_VALUES_LIST1 = + Collections.singletonList(Collections.singletonList(77777777L)); + + public static List> EXP_EXTEND_VALUES_LIST2 = + Collections.singletonList(Collections.singletonList(3498.1)); + public static List> EXP_VALUES_LIST1 = Arrays.asList(Collections.singletonList(33333333L), Collections.singletonList(44444444L)); @@ -74,6 +95,9 @@ public class Constant { public static List> READ_ONLY_VALUES_LIST = Arrays.asList(Arrays.asList(55555555L, 10012.01), Arrays.asList(66666666L, 99123.99)); + public static List> READ_ONLY_EXTEND_VALUES_LIST = + Collections.singletonList(Arrays.asList(9999999L, 152346.1)); + public static List> REPEAT_EXP_VALUES_LIST1 = Arrays.asList(Arrays.asList(33333333L, 33333333L), Arrays.asList(44444444L, 44444444L)); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java index 578d8f0020..657bf4de45 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/filesystem/FileSystemCapacityExpansionIT.java @@ -3,6 +3,7 @@ import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.filesystem; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; +import cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +12,86 @@ public class FileSystemCapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemCapacityExpansionIT.class); public FileSystemCapacityExpansionIT() { - super(filesystem, "chunk_size_in_bytes:8"); + super(filesystem, "chunk_size_in_bytes:8", new FileSystemHistoryDataGenerator()); + } + + // skip this test + @Override + protected void testInvalidDummyParams( + int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) { + LOGGER.info("filesystem skips test for wrong dummy engine params."); + } + + @Override + public void testShowColumns() { + String statement = "SHOW COLUMNS mn.*;"; + String expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| mn.wf01.wt01.status| BINARY|\n" + + "|mn.wf01.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| a.b.c.d.1\\txt| BINARY|\n" + + "| a.e.2\\txt| BINARY|\n" + + "| a.f.g.3\\txt| BINARY|\n" + + "| ln.wf02.status| BOOLEAN|\n" + + "| ln.wf02.version| BINARY|\n" + + "| mn.wf01.wt01.status| BINARY|\n" + + "|mn.wf01.wt01.temperature| BINARY|\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|nt.wf04.wt01.temperature| BINARY|\n" + + "| tm.wf05.wt01.status| BINARY|\n" + + "|tm.wf05.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 11\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS nt.*;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|nt.wf04.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS tm.*;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| tm.wf05.wt01.status| BINARY|\n" + + "|tm.wf05.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS a.*;"; + expected = + "Columns:\n" + + "+-------------+--------+\n" + + "| Path|DataType|\n" + + "+-------------+--------+\n" + + "|a.b.c.d.1\\txt| BINARY|\n" + + "| a.e.2\\txt| BINARY|\n" + + "| a.f.g.3\\txt| BINARY|\n" + + "+-------------+--------+\n" + + "Total line number = 3\n"; + SQLTestTools.executeAndCompare(session, statement, expected); } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java index 2dcfe6adbe..47d638e437 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/influxdb/InfluxDBCapacityExpansionIT.java @@ -15,11 +15,20 @@ public class InfluxDBCapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(InfluxDBCapacityExpansionIT.class); public InfluxDBCapacityExpansionIT() { - super(influxdb, "username:user, password:12345678, token:testToken, organization:testOrg"); + super( + influxdb, + "username:user, password:12345678, token:testToken, organization:testOrg", + new InfluxDBHistoryDataGenerator()); ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); DBConf dbConf = conf.loadDBConf(conf.getStorageType()); Constant.oriPort = dbConf.getDBCEPortMap().get(Constant.ORI_PORT_NAME); Constant.expPort = dbConf.getDBCEPortMap().get(Constant.EXP_PORT_NAME); Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); + wrongExtraParams.add( + "username:user, password:12345678, token:testToken, organization:wrongOrg"); } + + // dummy key range cannot be extended yet + @Override + protected void queryExtendedKeyDummy() {} } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java index 2874304c29..0361e3d49f 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/iotdb/IoTDB12CapacityExpansionIT.java @@ -11,6 +11,11 @@ public class IoTDB12CapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDB12CapacityExpansionIT.class); public IoTDB12CapacityExpansionIT() { - super(iotdb12, "username:root, password:root, sessionPoolSize:20"); + super( + iotdb12, + "username:root, password:root, sessionPoolSize:20", + new IoTDB12HistoryDataGenerator()); + wrongExtraParams.add("username:root, password:wrong, sessionPoolSize:20"); + wrongExtraParams.add("username:wrong, password:root, sessionPoolSize:20"); } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java index ca40a82259..d6c30250a9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mongodb/MongoDBCapacityExpansionIT.java @@ -15,7 +15,7 @@ public class MongoDBCapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(MongoDBCapacityExpansionIT.class); public MongoDBCapacityExpansionIT() { - super(mongodb, null); + super(mongodb, null, new MongoDBHistoryDataGenerator()); ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); DBConf dbConf = conf.loadDBConf(conf.getStorageType()); Constant.oriPort = dbConf.getDBCEPortMap().get(Constant.ORI_PORT_NAME); @@ -276,4 +276,48 @@ private void testFilter() { + "Empty set.\n"; SQLTestTools.executeAndCompare(session, statement, expect); } + + // mongoDB中，数据含有id + @Override + public void testShowColumns() { + String statement = "SHOW COLUMNS mn.*;"; + String expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| mn.wf01._id| INTEGER|\n" + + "| mn.wf01.wt01.status| LONG|\n" + + "|mn.wf01.wt01.temperature| DOUBLE|\n" + + "+------------------------+--------+\n" + + "Total line number = 3\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS nt.*;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| nt.wf03._id| INTEGER|\n" + + "| nt.wf03.wt01.status2| LONG|\n" + + "| nt.wf04._id| INTEGER|\n" + + "|nt.wf04.wt01.temperature| DOUBLE|\n" + + "+------------------------+--------+\n" + + "Total line number = 4\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS tm.*;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| tm.wf05._id| INTEGER|\n" + + "| tm.wf05.wt01.status| LONG|\n" + + "|tm.wf05.wt01.temperature| DOUBLE|\n" + + "+------------------------+--------+\n" + + "Total line number = 3\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java new file mode 100644 index 0000000000..f75fbea476 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLCapacityExpansionIT.java @@ -0,0 +1,28 @@ +package cn.edu.tsinghua.iginx.integration.expansion.mysql; + +import cn.edu.tsinghua.iginx.integration.controller.Controller; +import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; +import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; +import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; +import cn.edu.tsinghua.iginx.integration.tool.DBConf; +import cn.edu.tsinghua.iginx.thrift.StorageEngineType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySQLCapacityExpansionIT extends BaseCapacityExpansionIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySQLCapacityExpansionIT.class); + + public MySQLCapacityExpansionIT() { + super( + StorageEngineType.relational, + "engine:mysql, username:root," + + "meta_properties_path:dataSources/relational/src/main/resources/mysql-meta-template.properties", + new MySQLHistoryDataGenerator()); + ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); + DBConf dbConf = conf.loadDBConf(conf.getStorageType()); + Constant.oriPort = dbConf.getDBCEPortMap().get(Constant.ORI_PORT_NAME); + Constant.expPort = dbConf.getDBCEPortMap().get(Constant.EXP_PORT_NAME); + Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java new file mode 100644 index 0000000000..298b03100f --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/mysql/MySQLHistoryDataGenerator.java @@ -0,0 +1,220 @@ +package cn.edu.tsinghua.iginx.integration.expansion.mysql; + +import cn.edu.tsinghua.iginx.integration.expansion.BaseHistoryDataGenerator; +import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.sql.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySQLHistoryDataGenerator extends BaseHistoryDataGenerator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySQLHistoryDataGenerator.class); + + private static final char SEPARATOR = '.'; + + private static final String QUERY_DATABASES_STATEMENT = + "SELECT SCHEMA_NAME AS datname FROM information_schema.schemata;"; + + private static final String CREATE_DATABASE_STATEMENT = "CREATE DATABASE `%s`;"; + + private static final String CREATE_TABLE_STATEMENT = "CREATE TABLE %s (%s);"; + + private static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; + + private static final String DROP_DATABASE_STATEMENT = "DROP DATABASE IF EXISTS `%s`;"; + + public MySQLHistoryDataGenerator() { + Constant.oriPort = 3306; + Constant.expPort = 3307; + Constant.readOnlyPort = 3308; + } + + private Connection connect(int port, boolean useSystemDatabase, String databaseName) { + try { + String url; + if (useSystemDatabase) { + url = String.format("jdbc:mysql://127.0.0.1:%d/", port); + } else { + url = String.format("jdbc:mysql://127.0.0.1:%d/%s", port, databaseName); + } + Class.forName("com.mysql.cj.jdbc.Driver"); + return DriverManager.getConnection(url, "root", null); + } catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Override + public void writeHistoryData( + int port, + List pathList, + List dataTypeList, + List keyList, + List> valuesList) { + Connection connection = null; + try { + connection = connect(port, true, null); + if (connection == null) { + LOGGER.error("cannot connect to 127.0.0.1:{}!", port); + return; + } + + Map>> databaseToTablesToColumnIndexes = new HashMap<>(); + for (int i = 0; i < pathList.size(); i++) { + String path = pathList.get(i); + String databaseName = path.substring(0, path.indexOf(SEPARATOR)); + String tableName = path.substring(path.indexOf(SEPARATOR) + 1, path.lastIndexOf(SEPARATOR)); + + Map> tablesToColumnIndexes = + databaseToTablesToColumnIndexes.computeIfAbsent(databaseName, x -> new HashMap<>()); + List columnIndexes = + tablesToColumnIndexes.computeIfAbsent(tableName, x -> new ArrayList<>()); + columnIndexes.add(i); + tablesToColumnIndexes.put(tableName, columnIndexes); + } + + for (Map.Entry>> entry : + databaseToTablesToColumnIndexes.entrySet()) { + String databaseName = entry.getKey(); + Statement stmt = connection.createStatement(); + String createDatabaseSql = String.format(CREATE_DATABASE_STATEMENT, databaseName); + try { + LOGGER.info("create database with stmt: {}", createDatabaseSql); + stmt.execute(createDatabaseSql); + } catch (SQLException e) { + LOGGER.info("database {} exists!", databaseName); + } + stmt.close(); + + Connection conn = connect(port, false, databaseName); + stmt = conn.createStatement(); + for (Map.Entry> item : entry.getValue().entrySet()) { + String tableName = item.getKey(); + StringBuilder createTableStr = new StringBuilder(); + for (Integer index : item.getValue()) { + String path = pathList.get(index); + String columnName = path.substring(path.lastIndexOf(SEPARATOR) + 1); + DataType dataType = dataTypeList.get(index); + createTableStr.append(getQuotName(columnName)); + createTableStr.append(" "); + createTableStr.append(toPostgreSQL(dataType)); + createTableStr.append(", "); + } + stmt.execute( + String.format( + CREATE_TABLE_STATEMENT, + getQuotName(tableName), + createTableStr.substring(0, createTableStr.length() - 2))); + + StringBuilder insertStr = new StringBuilder(); + for (List values : valuesList) { + insertStr.append("("); + for (Integer index : item.getValue()) { + if (dataTypeList.get(index) == DataType.BINARY) { + insertStr.append("'").append(new String((byte[]) values.get(index))).append("'"); + } else { + insertStr.append(values.get(index)); + } + insertStr.append(", "); + } + insertStr = new StringBuilder(insertStr.substring(0, insertStr.length() - 2)); + insertStr.append("), "); + } + stmt.execute( + String.format( + INSERT_STATEMENT, + getQuotName(tableName), + insertStr.substring(0, insertStr.length() - 2))); + } + stmt.close(); + conn.close(); + } + connection.close(); + + LOGGER.info("write data to 127.0.0.1:{} success!", port); + } catch (RuntimeException | SQLException e) { + LOGGER.error("write data to 127.0.0.1:{} failure: ", port, e); + } finally { + try { + if (connection != null) { + connection.close(); + } + } catch (SQLException e) { + LOGGER.error("close connection failure: ", e); + } + } + } + + @Override + public void writeHistoryData( + int port, List pathList, List dataTypeList, List> valuesList) { + writeHistoryData(port, pathList, dataTypeList, new ArrayList<>(), valuesList); + } + + @Override + public void clearHistoryDataForGivenPort(int port) { + Connection conn = null; + try { + conn = connect(port, true, null); + Statement stmt = conn.createStatement(); + ResultSet databaseSet = stmt.executeQuery(QUERY_DATABASES_STATEMENT); + Statement dropDatabaseStatement = conn.createStatement(); + + while (databaseSet.next()) { + String databaseName = databaseSet.getString("DATNAME"); + if (databaseName.equalsIgnoreCase("performance_schema") + || databaseName.equalsIgnoreCase("information_schema") + || databaseName.equalsIgnoreCase("mysql") + || databaseName.equalsIgnoreCase("sys")) { + continue; + } + dropDatabaseStatement.addBatch(String.format(DROP_DATABASE_STATEMENT, databaseName)); + LOGGER.info("drop database {} on 127.0.0.1:{}: ", databaseName, port); + } + dropDatabaseStatement.executeBatch(); + + dropDatabaseStatement.close(); + databaseSet.close(); + stmt.close(); + conn.close(); + LOGGER.info("clear data on 127.0.0.1:{} success!", port); + } catch (SQLException e) { + LOGGER.warn("clear data on 127.0.0.1:{} failure: ", port, e); + } finally { + try { + if (conn != null) { + conn.close(); + } + } catch (SQLException e) { + LOGGER.error("close connection failure: ", e); + } + } + } + + private static String toPostgreSQL(DataType dataType) { + switch (dataType) { + case BOOLEAN: + return "BOOLEAN"; + case INTEGER: + return "INTEGER"; + case LONG: + return "BIGINT"; + case FLOAT: + return "REAL"; + case DOUBLE: + return "DOUBLE PRECISION"; + case BINARY: + default: + return "TEXT"; + } + } + + private static String getQuotName(String name) { + return "`" + name + "`"; + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java index 1f0ba4ebac..178847bcd6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/parquet/ParquetCapacityExpansionIT.java @@ -11,6 +11,13 @@ public class ParquetCapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(ParquetCapacityExpansionIT.class); public ParquetCapacityExpansionIT() { - super(parquet, null); + super(parquet, null, new ParquetHistoryDataGenerator()); + } + + // skip this test + @Override + protected void testInvalidDummyParams( + int port, boolean hasData, boolean isReadOnly, String dataPrefix, String schemaPrefix) { + LOGGER.info("parquet skips test for wrong dummy engine params."); } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java index 24fce59155..0ced1e22a1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLCapacityExpansionIT.java @@ -1,13 +1,12 @@ package cn.edu.tsinghua.iginx.integration.expansion.postgresql; -import static cn.edu.tsinghua.iginx.thrift.StorageEngineType.postgresql; - import cn.edu.tsinghua.iginx.integration.controller.Controller; import cn.edu.tsinghua.iginx.integration.expansion.BaseCapacityExpansionIT; import cn.edu.tsinghua.iginx.integration.expansion.constant.Constant; import cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools; import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; import cn.edu.tsinghua.iginx.integration.tool.DBConf; +import cn.edu.tsinghua.iginx.thrift.StorageEngineType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,12 +15,17 @@ public class PostgreSQLCapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLCapacityExpansionIT.class); public PostgreSQLCapacityExpansionIT() { - super(postgresql, "username:postgres, password:postgres"); + super( + StorageEngineType.relational, + "engine:postgresql, username:postgres, password:postgres", + new PostgreSQLHistoryDataGenerator()); ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); DBConf dbConf = conf.loadDBConf(conf.getStorageType()); Constant.oriPort = dbConf.getDBCEPortMap().get(Constant.ORI_PORT_NAME); Constant.expPort = dbConf.getDBCEPortMap().get(Constant.EXP_PORT_NAME); Constant.readOnlyPort = dbConf.getDBCEPortMap().get(Constant.READ_ONLY_PORT_NAME); + wrongExtraParams.add("username:wrong, password:postgres"); + // wrong password situation cannot be tested because trust mode is used } /** 执行一个简单查询1000次，测试是否会使连接池耗尽，来验证PG的dummy查询是否正确释放连接 */ diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java index def3cba35f..69d45ecc94 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/postgresql/PostgreSQLHistoryDataGenerator.java @@ -23,7 +23,7 @@ public class PostgreSQLHistoryDataGenerator extends BaseHistoryDataGenerator { private static final String INSERT_STATEMENT = "INSERT INTO %s VALUES %s;"; - private static final String DROP_DATABASE_STATEMENT = "DROP DATABASE \"%s\";"; + private static final String DROP_DATABASE_STATEMENT = "DROP DATABASE \"%s\" WITH (FORCE);"; private static final String USERNAME = "postgres"; diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java index 23e8474e39..fdfb57e0ad 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/redis/RedisCapacityExpansionIT.java @@ -16,7 +16,7 @@ public class RedisCapacityExpansionIT extends BaseCapacityExpansionIT { private static final Logger LOGGER = LoggerFactory.getLogger(RedisCapacityExpansionIT.class); public RedisCapacityExpansionIT() { - super(redis, null); + super(redis, null, new RedisHistoryDataGenerator()); ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); DBConf dbConf = conf.loadDBConf(conf.getStorageType()); Constant.oriPort = dbConf.getDBCEPortMap().get(Constant.ORI_PORT_NAME); @@ -38,4 +38,44 @@ protected void testQuerySpecialHistoryData() { + "Total line number = 2\n"; SQLTestTools.executeAndCompare(session, statement, expect); } + + // redis中，所有dummy数据都识别为BINARY + @Override + public void testShowColumns() { + String statement = "SHOW COLUMNS mn.*;"; + String expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| mn.wf01.wt01.status| BINARY|\n" + + "|mn.wf01.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS nt.*;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| nt.wf03.wt01.status2| BINARY|\n" + + "|nt.wf04.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + + statement = "SHOW COLUMNS tm.*;"; + expected = + "Columns:\n" + + "+------------------------+--------+\n" + + "| Path|DataType|\n" + + "+------------------------+--------+\n" + + "| tm.wf05.wt01.status| BINARY|\n" + + "|tm.wf05.wt01.temperature| BINARY|\n" + + "+------------------------+--------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, statement, expected); + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java index 970f6cf951..6085f6798c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/expansion/utils/SQLTestTools.java @@ -47,6 +47,18 @@ private static String execute(Session session, String statement) { private static void compareValuesList( List> expectedValuesList, List> actualValuesList) { + compareValuesList(expectedValuesList, actualValuesList, true); + } + + private static void containValuesList( + List> expectedValuesList, List> actualValuesList) { + compareValuesList(expectedValuesList, actualValuesList, false); + } + + private static void compareValuesList( + List> expectedValuesList, + List> actualValuesList, + boolean equalMode) { Set> expectedSet = expectedValuesList.stream() .map( @@ -81,9 +93,12 @@ private static void compareValuesList( }) .collect(Collectors.toSet()); - if (!expectedSet.equals(actualSet)) { + if (equalMode && !expectedSet.equals(actualSet)) { LOGGER.error("actual valuesList is {} and it should be {}", actualSet, expectedSet); fail(); + } else if (!equalMode && !actualSet.containsAll(expectedSet)) { + LOGGER.error("actual valuesList is {} and it should contain {}", actualSet, expectedSet); + fail(); } } @@ -106,6 +121,26 @@ public static void executeAndCompare( } } + /** execute query and result should contain expected values for specified paths. */ + public static void executeAndContainValue( + Session session, + String statement, + List pathListAns, + List> expectedValuesList) { + try { + SessionExecuteSqlResult res = session.executeSql(statement); + List pathList = res.getPaths(); + List> actualValuesList = res.getValues(); + + assertArrayEquals(new List[] {pathListAns}, new List[] {pathList}); + + containValuesList(expectedValuesList, actualValuesList); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", statement, e); + fail(); + } + } + public static int executeShellScript(String scriptPath, String... args) { try { // 构建shell命令，action中的windows runner需要使用绝对路径 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java index dcb3cdc342..7337e1a65e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/NewSessionIT.java @@ -122,7 +122,7 @@ private static TestDataSection buildBaseDataSection() { @BeforeClass public static void setUp() throws SessionException { ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); - if (StorageEngineType.valueOf(conf.getStorageType().toLowerCase()) == influxdb) { + if (StorageEngineType.valueOf(conf.getStorageType(false).toLowerCase()) == influxdb) { isInfluxdb = true; } if (!SUPPORT_KEY.get(conf.getStorageType()) && conf.isScaling()) { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java index 22f06b1238..2a3771bf83 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/session/SessionV2IT.java @@ -43,7 +43,7 @@ public class SessionV2IT { @BeforeClass public static void setUp() { ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); - if (StorageEngineType.valueOf(conf.getStorageType().toLowerCase()) == influxdb) { + if (StorageEngineType.valueOf(conf.getStorageType(false).toLowerCase()) == influxdb) { isInfluxdb = true; } if (!SUPPORT_KEY.get(conf.getStorageType()) && conf.isScaling()) { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java index 9d31ba5de2..accd4a9f3e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/SQLSessionIT.java @@ -219,6 +219,28 @@ private String generateDefaultInsertStatementByTimeRange(long start, long end) { return builder.toString(); } + private int getTimeCostFromExplainPhysicalResult(String explainPhysicalResult) { + String[] lines = explainPhysicalResult.split("\n"); + int executeTimeIndex = -1; + int timeCost = 0; + for (String line : lines) { + String[] split = line.split("\\|"); + if (split.length > 1) { + if (executeTimeIndex == -1) { + for (int i = 0; i < split.length; i++) { + if (split[i].trim().contains("Time")) { + executeTimeIndex = i; + break; + } + } + } else { + timeCost += Integer.parseInt(split[executeTimeIndex].trim().replace("ms", "")); + } + } + } + return timeCost; + } + @After public void clearData() { String clearData = "CLEAR DATA;"; @@ -7282,4 +7304,83 @@ public void testConstantFolding() { } } } + + @Test + public void testCorrelateSubqueryTime() { + // 测试IN和EXISTS的关联子查询时间是否相近，在某次修改前EXISTS的时间会比IN的时间长很多（约2个数量级） + String inQuery = + "select zipcode, city from us as a \n" + + "where zipcode not in ( \n" + + " SELECT zipcode \n" + + " FROM us \n" + + " where us.zipcode = a.zipcode && us.city <> a.city \n" + + ");"; + String existsQuery = + "SELECT zipcode, city\n" + + "FROM us AS a\n" + + "WHERE NOT EXISTS (\n" + + " SELECT *\n" + + " FROM us AS b\n" + + " WHERE a.zipcode = b.zipcode AND a.city <> b.city\n" + + ");"; + + // 插入数据 + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO us (key, zipcode, city) VALUES "); + int rows = 10000; + List cityList = + Arrays.asList( + "New York", + "Los Angeles", + "Chicago", + "Houston", + "Phoenix", + "Philadelphia", + "San Antonio", + "San Diego", + "Dallas"); + for (int i = 0; i < rows; i++) { + insert.append( + String.format( + "(%d, %d, \"%s\")", + i, (i % 9) + (i % 99) + (i % 999), cityList.get(i % cityList.size()))); + if (i != rows - 1) { + insert.append(","); + } + } + insert.append(";"); + executor.execute(insert.toString()); + + // 检查IN和EXISTS的查询结果是否一致 + String inResult = executor.execute(inQuery); + String existsResult = executor.execute(existsQuery); + String expect = + "ResultSets:\n" + + "+----+---------+-----------+\n" + + "| key|a.zipcode| a.city|\n" + + "+----+---------+-----------+\n" + + "| 0| 0| New York|\n" + + "| 1| 3|Los Angeles|\n" + + "| 2| 6| Chicago|\n" + + "| 987| 1089|San Antonio|\n" + + "| 988| 1092| San Diego|\n" + + "| 989| 1095| Dallas|\n" + + "|9987| 1089|San Antonio|\n" + + "|9988| 1092| San Diego|\n" + + "|9989| 1095| Dallas|\n" + + "+----+---------+-----------+\n" + + "Total line number = 9\n"; + assertEquals(expect, inResult); + assertEquals(expect, existsResult); + + // explain physical查询 + String inExplainResult = executor.execute("EXPLAIN PHYSICAL " + inQuery); + String existsExplainResult = executor.execute("EXPLAIN PHYSICAL " + existsQuery); + + // 考虑到波动，两者耗时相差不超过3倍 + int inTime = getTimeCostFromExplainPhysicalResult(inExplainResult); + int existsTime = getTimeCostFromExplainPhysicalResult(existsExplainResult); + System.out.println(String.format("IN COST: %dms, EXISTS COST: %dms", inTime, existsTime)); + assertTrue(inTime * 3 >= existsTime || existsTime * 3 >= inTime); + } } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java new file mode 100644 index 0000000000..1e4ebd6856 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/RemoteUDFIT.java @@ -0,0 +1,161 @@ +package cn.edu.tsinghua.iginx.integration.func.udf; + +import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; +import static org.junit.Assert.assertTrue; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.controller.Controller; +import cn.edu.tsinghua.iginx.integration.expansion.utils.SQLTestTools; +import cn.edu.tsinghua.iginx.integration.func.session.InsertAPIType; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.thrift.DataType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RemoteUDFIT { + private static final Logger LOGGER = LoggerFactory.getLogger(RemoteUDFIT.class); + + private static final String DROP_SQL = "DROP FUNCTION \"%s\";"; + + private static UDFTestTools tool; + private static Session session; + private static boolean dummyNoData = true; + private static List taskToBeRemoved; + + @BeforeClass + public static void setUp() throws SessionException { + session = new Session("127.0.0.1", 6888, "root", "root"); + session.openSession(); + tool = new UDFTestTools(session); + } + + @AfterClass + public static void tearDown() throws SessionException { + clearAllData(session); + session.closeSession(); + } + + @Before + public void insertData() { + long startKey = 0L; + long endKey = 15000L; + List pathList = + new ArrayList() { + { + add("us.d1.s1"); + add("us.d1.s2"); + add("us.d1.s3"); + add("us.d1.s4"); + } + }; + List dataTypeList = + new ArrayList() { + { + add(DataType.LONG); + add(DataType.LONG); + add(DataType.BINARY); + add(DataType.DOUBLE); + } + }; + List keyList = new ArrayList<>(); + List> valuesList = new ArrayList<>(); + int size = (int) (endKey - startKey); + for (int i = 0; i < size; i++) { + keyList.add(startKey + i); + valuesList.add( + Arrays.asList( + (long) i, + (long) i + 1, + ("\"" + RandomStringUtils.randomAlphanumeric(10) + "\"").getBytes(), + (i + 0.1d))); + } + Controller.writeRowsData( + session, + pathList, + keyList, + dataTypeList, + valuesList, + new ArrayList<>(), + InsertAPIType.Row, + dummyNoData); + dummyNoData = false; + Controller.after(session); + } + + @After + public void clearData() { + Controller.clearData(session); + } + + @Before + public void resetTaskToBeDropped() { + if (taskToBeRemoved == null) { + taskToBeRemoved = new ArrayList<>(); + } else { + taskToBeRemoved.clear(); + } + } + + // drop unwanted UDFs no matter what + @After + public void dropTasks() { + for (String name : taskToBeRemoved) { + tool.execute(String.format(DROP_SQL, name)); + } + taskToBeRemoved.clear(); + } + + // before this test, remote UDFs have been registered in test_remote_udf.sh + // after this test, we expect all functions are deleted + @Test + public void remoteRegisterTest() { + List udfList = + Arrays.asList( + "mock_udf", "udf_a", "udf_b", "udf_sub", "udf_a_file", "udf_b_file", "udf_c_file"); + assertTrue(tool.isUDFsRegistered(udfList)); + taskToBeRemoved.addAll(udfList); + + String sql = "insert into test(key,a) values(0,1), (1,2);", expect; + tool.execute(sql); + + // test each usage + // these UDFs are registered in 3 groups, for efficiency we only test one in each group. + sql = "select mock_udf(a,1) from test;"; + expect = + "ResultSets:\n" + + "+---+---+\n" + + "|key|col|\n" + + "+---+---+\n" + + "| 0| 1|\n" + + "| 1| 1|\n" + + "+---+---+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, sql, expect); + sql = "select udf_b(a,1) from test;"; + expect = + "ResultSets:\n" + + "+---+-----------+\n" + + "|key|col_outer_b|\n" + + "+---+-----------+\n" + + "| 0| 1|\n" + + "| 1| 1|\n" + + "+---+-----------+\n" + + "Total line number = 2\n"; + SQLTestTools.executeAndCompare(session, sql, expect); + sql = "select udf_b_file(a,1) from test;"; + expect = + "ResultSets:\n" + + "+-----------+\n" + + "|col_outer_b|\n" + + "+-----------+\n" + + "| 1|\n" + + "+-----------+\n" + + "Total line number = 1\n"; + SQLTestTools.executeAndCompare(session, sql, expect); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java index 01c87580e7..58078130ea 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/TransformIT.java @@ -33,7 +33,10 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import cn.edu.tsinghua.iginx.thrift.*; +import cn.edu.tsinghua.iginx.utils.JobFromYAML; import cn.edu.tsinghua.iginx.utils.RpcUtils; +import cn.edu.tsinghua.iginx.utils.YAMLReader; +import cn.edu.tsinghua.iginx.utils.YAMLWriter; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -41,6 +44,8 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.AfterClass; @@ -629,6 +634,12 @@ public void commitMixedPythonJobsByYamlWithRegisterTest() { OUTPUT_DIR_PREFIX + File.separator + "export_file_mixed_python_jobs_with_register_by_yaml.txt"; + YAMLReader reader = new YAMLReader(yamlFileName); + JobFromYAML job = reader.getJobFromYAML(); + replaceRelativePythonPathToAbsolute(job); + YAMLWriter writer = new YAMLWriter(); + writer.writeJobIntoYAML(new File(yamlFileName), job); + SessionExecuteSqlResult result = session.executeSql(String.format(COMMIT_SQL_FORMATTER, yamlFileName)); long jobId = result.getJobId(); @@ -641,6 +652,38 @@ public void commitMixedPythonJobsByYamlWithRegisterTest() { } } + // replace relative python filepath in sql with absolute paths for UDF registration. + private void replaceRelativePythonPathToAbsolute(JobFromYAML job) { + // match string between two double quotes + Pattern pattern = Pattern.compile("\"([^\"]*)\""); + job.getTaskList() + .forEach( + task -> { + if (task.getSqlList() == null) { + return; + } + List newSqlList = new ArrayList<>(); + task.getSqlList() + .forEach( + sql -> { + String oriPath; + Matcher matcher = pattern.matcher(sql); + while (matcher.find()) { + oriPath = matcher.group(1); + if (!oriPath.endsWith(".py")) { + continue; + } + File filePath = new File(oriPath); + if (!filePath.isAbsolute()) { + sql = sql.replace(oriPath, filePath.getAbsolutePath()); + } + } + newSqlList.add(sql); + }); + task.setSqlList(newSqlList); + }); + } + private void verifyMixedPythonJobs(String outputFileName) throws IOException { BufferedReader reader = new BufferedReader(new FileReader(outputFileName)); String line = reader.readLine(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java index 4f25b20261..428d1de1a9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFIT.java @@ -38,8 +38,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.AfterClass; @@ -83,6 +81,19 @@ public class UDFIT { "udf", "my_module"); + private static final String MODULE_FILE_PATH = + String.join( + File.separator, + System.getProperty("user.dir"), + "src", + "test", + "resources", + "udf", + "my_module", + "idle_classes.py"); + + private static UDFTestTools tool; + @BeforeClass public static void setUp() throws SessionException { ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); @@ -92,6 +103,7 @@ public static void setUp() throws SessionException { } session = new Session("127.0.0.1", 6888, "root", "root"); session.openSession(); + tool = new UDFTestTools(session); } @AfterClass @@ -165,147 +177,28 @@ public void resetTaskToBeDropped() { @After public void dropTasks() { for (String name : taskToBeRemoved) { - execute(String.format(DROP_SQL, name)); + tool.execute(String.format(DROP_SQL, name)); } taskToBeRemoved.clear(); } - private SessionExecuteSqlResult execute(String statement) { - LOGGER.info("Execute Statement: \"{}\"", statement); - - SessionExecuteSqlResult res = null; - try { - res = session.executeSql(statement); - } catch (SessionException e) { - LOGGER.error("Statement: \"{}\" execute fail. Caused by:", statement, e); - fail(); - } - - if (res.getParseErrorMsg() != null && !res.getParseErrorMsg().equals("")) { - LOGGER.error( - "Statement: \"{}\" execute fail. Caused by: {}.", statement, res.getParseErrorMsg()); - fail(); - } - - return res; - } - - // execute a statement and expect failure. - private void executeFail(String statement) { - LOGGER.info("Execute Statement: \"{}\"", statement); - - SessionExecuteSqlResult res = null; - try { - res = session.executeSql(statement); - } catch (SessionException e) { - // don't want to print e because it will be confusing - LOGGER.info( - "Statement: \"{}\" execute failed AS EXPECTED, with message: {}", - statement, - e.getMessage()); - return; - } - - if (res.getParseErrorMsg() != null && !res.getParseErrorMsg().equals("")) { - LOGGER.info( - "Statement: \"{}\" execute failed AS EXPECTED, with message: {}.", - statement, - res.getParseErrorMsg()); - return; - } - - fail("Statement: \"{}\" execute without failure, which was not expected."); - } - - private boolean isUDFRegistered(String udfName) { - SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); - List registerUDFs = - ret.getRegisterTaskInfos().stream() - .map(RegisterTaskInfo::getName) - .collect(Collectors.toList()); - return registerUDFs.contains(udfName); - } - - private boolean isUDFsRegistered(List names) { - SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); - List registerUDFs = - ret.getRegisterTaskInfos().stream() - .map(RegisterTaskInfo::getName) - .collect(Collectors.toList()); - for (String udfName : names) { - if (!registerUDFs.contains(udfName)) { - return false; - } - } - return true; - } - - // all udf shouldn't be registered. - private boolean isUDFsUnregistered(List names) { - SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); - List registerUDFs = - ret.getRegisterTaskInfos().stream() - .map(RegisterTaskInfo::getName) - .collect(Collectors.toList()); - for (String udfName : names) { - if (registerUDFs.contains(udfName)) { - return false; - } - } - return true; - } - - /** - * generate multiple UDFs' registration sql command - * - * @param types UDF types - * @param names UDF names that will be used in sql after - * @param classPaths UDF class path relative to module root - * @param modulePath module dir position - * @return a sql string - */ - private String concatMultiUDFReg( - List types, List names, List classPaths, String modulePath) { - assertEquals(types.size(), names.size()); - assertEquals(names.size(), classPaths.size()); - String udfs = - IntStream.range(0, types.size()) - .mapToObj( - i -> - String.format( - "%s \"%s\" FROM \"%s\"", types.get(i), names.get(i), classPaths.get(i))) - .collect(Collectors.joining(", ")); - return String.format(MULTI_UDF_REGISTER_SQL, udfs, modulePath); - } - - // all UDFs will be registered in one type - private String concatMultiUDFReg( - String type, List names, List classPaths, String modulePath) { - assertEquals(names.size(), classPaths.size()); - String udfs = - IntStream.range(0, names.size()) - .mapToObj(i -> String.format("\"%s\" FROM \"%s\"", names.get(i), classPaths.get(i))) - .collect(Collectors.joining(", ")); - return String.format(MULTI_UDF_REGISTER_SQL, type + " " + udfs, modulePath); - } - @Test public void baseTests() { String udtfSQLFormat = "SELECT %s(s1) FROM us.d1 WHERE key < 200;"; String udafSQLFormat = "SELECT %s(s1) FROM us.d1 OVER (RANGE 50 IN [0, 200));"; String udsfSQLFormat = "SELECT %s(s1) FROM us.d1 WHERE key < 50;"; - SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); + SessionExecuteSqlResult ret = tool.execute(SHOW_FUNCTION_SQL); List registerUDFs = ret.getRegisterTaskInfos(); for (RegisterTaskInfo info : registerUDFs) { // execute udf if (info.getType().equals(UDFType.UDTF)) { - execute(String.format(udtfSQLFormat, info.getName())); + tool.execute(String.format(udtfSQLFormat, info.getName())); } else if (info.getType().equals(UDFType.UDAF)) { - execute(String.format(udafSQLFormat, info.getName())); + tool.execute(String.format(udafSQLFormat, info.getName())); } else if (info.getType().equals(UDFType.UDSF)) { - execute(String.format(udsfSQLFormat, info.getName())); + tool.execute(String.format(udsfSQLFormat, info.getName())); } } } @@ -322,25 +215,25 @@ public void testDropTask() { "udf", "mock_udf.py"); String udfName = "mock_udf"; - execute(String.format(SINGLE_UDF_REGISTER_SQL, "UDAF", udfName, "MockUDF", filePath)); - assertTrue(isUDFRegistered(udfName)); + tool.executeReg(String.format(SINGLE_UDF_REGISTER_SQL, "UDAF", udfName, "MockUDF", filePath)); + assertTrue(tool.isUDFRegistered(udfName)); taskToBeRemoved.add(udfName); - execute(String.format(DROP_SQL, udfName)); + tool.execute(String.format(DROP_SQL, udfName)); // dropped udf cannot be queried - assertFalse(isUDFRegistered(udfName)); + assertFalse(tool.isUDFRegistered(udfName)); taskToBeRemoved.clear(); // make sure dropped udf cannot be used String statement = "SELECT " + udfName + "(s1,1) FROM us.d1 WHERE s1 < 10;"; - executeFail(statement); + tool.executeFail(statement); } @Test public void testCOS() { String statement = "SELECT cos(s1) FROM us.d1 WHERE s1 < 10;"; - SessionExecuteSqlResult ret = execute(statement); + SessionExecuteSqlResult ret = tool.execute(statement); compareResult(Collections.singletonList("cos(us.d1.s1)"), ret.getPaths()); compareResult(new long[] {0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L}, ret.getKeys()); @@ -368,11 +261,11 @@ public void testCOS() { public void testConcurrentCos() { String insert = "INSERT INTO test(key, s1, s2) VALUES (1, 2, 3), (2, 3, 1), (3, 4, 3), (4, 9, 7), (5, 3, 6), (6, 6, 4);"; - execute(insert); + tool.execute(insert); String query = "SELECT * FROM (SELECT cos(s1) AS cos_s1 FROM test) AS t1, (SELECT cos(s2) AS cos_s2 FROM test) AS t2 LIMIT 10;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); compareResult(4, ret.getPaths().size()); List cosS1ExpectedValues = @@ -416,10 +309,10 @@ public void testConcurrentCos() { public void testMultiColumns() { String insert = "INSERT INTO test(key, s1, s2, s3) VALUES (1, 2, 3, 2), (2, 3, 1, 3), (3, 4, 3, 1), (4, 9, 7, 5), (5, 3, 6, 2), (6, 6, 4, 2);"; - execute(insert); + tool.execute(insert); String query = "SELECT multiply(s1, s2) FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+--------------------------+\n" @@ -436,7 +329,7 @@ public void testMultiColumns() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT multiply(s1, s2, s3) FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-----------------------------------+\n" @@ -453,7 +346,7 @@ public void testMultiColumns() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT multiply(s1, s2, s3), s1, s2 + s3 FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-----------------------------------+-------+-----------------+\n" @@ -474,10 +367,10 @@ public void testMultiColumns() { public void testExprFilter() { String insert = "INSERT INTO test(key, s1, s2, s3) VALUES (1, 2, 3, 2), (2, 3, 1, 3), (3, 4, 3, 1), (4, 9, 7, 5), (5, 3, 6, 2), (6, 6, 4, 2);"; - execute(insert); + tool.execute(insert); String query = "SELECT * FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+-------+-------+-------+\n" @@ -494,7 +387,7 @@ public void testExprFilter() { assertEquals(expected, ret.getResultInString(false, "")); query = "SELECT multiply(s1, s2) FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+--------------------------+\n" @@ -511,7 +404,7 @@ public void testExprFilter() { assertEquals(expected, ret.getResultInString(false, "")); query = "SELECT * FROM test WHERE multiply(s1, s2) > 15;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-------+-------+-------+\n" @@ -525,7 +418,7 @@ public void testExprFilter() { assertEquals(expected, ret.getResultInString(false, "")); query = "SELECT * FROM test WHERE multiply(s1, s2) + 10 > 15;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-------+-------+-------+\n" @@ -541,7 +434,7 @@ public void testExprFilter() { assertEquals(expected, ret.getResultInString(false, "")); query = "SELECT * FROM test WHERE multiply(s1, s2) + 10 > s3 + 15;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-------+-------+-------+\n" @@ -556,7 +449,7 @@ public void testExprFilter() { assertEquals(expected, ret.getResultInString(false, "")); query = "SELECT * FROM test WHERE multiply(s1, s2) + 9 > cos(s3) + 15;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-------+-------+-------+\n" @@ -572,7 +465,7 @@ public void testExprFilter() { assertEquals(expected, ret.getResultInString(false, "")); query = "SELECT * FROM test WHERE pow(s1, 0.5) - 1 > cos(s2) + cos(s3);"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+-------+-------+-------+\n" @@ -593,7 +486,7 @@ public void testExprFilter() { public void testSelectFromUDF() { String insert = "INSERT INTO test(key, a, b) VALUES (1, 2, 3), (2, 3, 1), (3, 4, 3), (4, 9, 7), (5, 3, 6), (6, 6, 4);"; - execute(insert); + tool.execute(insert); List cosTestAExpectedValues = Arrays.asList( @@ -613,7 +506,7 @@ public void testSelectFromUDF() { -0.6536436208636119); String query = "SELECT `cos(test.a)` FROM(SELECT cos(*) FROM test);"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); compareResult(1, ret.getPaths().size()); compareResult("cos(test.a)", ret.getPaths().get(0)); @@ -626,7 +519,7 @@ public void testSelectFromUDF() { } query = "SELECT `cos(test.b)` AS cos_b FROM(SELECT cos(*) FROM test);"; - ret = execute(query); + ret = tool.execute(query); compareResult(1, ret.getPaths().size()); compareResult("cos_b", ret.getPaths().get(0)); @@ -643,10 +536,10 @@ public void testSelectFromUDF() { public void testTransposeRows() { String insert = "INSERT INTO test(key, a, b) VALUES (1, 2, 3), (2, 3, 1), (3, 4, 3), (4, 9, 7), (5, 3, 6), (6, 6, 4);"; - execute(insert); + tool.execute(insert); String query = "SELECT * FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+------+------+\n" @@ -663,7 +556,7 @@ public void testTransposeRows() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT transpose(*) FROM (SELECT * FROM test);"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+------------+------------+------------+------------+------------+------------+\n" @@ -677,7 +570,7 @@ public void testTransposeRows() { query = "SELECT `transpose(0)`, `transpose(1)`, `transpose(2)` FROM (SELECT transpose(*) FROM (SELECT * FROM test));"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+------------+------------+------------+\n" @@ -694,10 +587,10 @@ public void testTransposeRows() { public void testColumnExpand() { String insert = "INSERT INTO test(key, a) VALUES (1, 2), (2, 3), (3, 4), (4, 9), (5, 3), (6, 6);"; - execute(insert); + tool.execute(insert); String query = "SELECT a FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+------+\n" @@ -714,7 +607,7 @@ public void testColumnExpand() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT columnExpand(*) FROM (SELECT a FROM test);"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+--------------------+------------------------+----------------------+\n" @@ -732,7 +625,7 @@ public void testColumnExpand() { query = "SELECT `columnExpand(test.a+1.5)` FROM (SELECT columnExpand(*) FROM (SELECT a FROM test)) WHERE `columnExpand(test.a+1.5)` < 5;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+------------------------+\n" @@ -750,10 +643,10 @@ public void testColumnExpand() { public void testUDFWithArgs() { String insert = "INSERT INTO test(key, s1, s2) VALUES (1, 2, 3), (2, 3, 1), (3, 4, 3), (4, 9, 7), (5, 3, 6), (6, 6, 4);"; - execute(insert); + tool.execute(insert); String query = "SELECT pow(s1, 2) FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+---------------+\n" @@ -770,7 +663,7 @@ public void testUDFWithArgs() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT pow(s1, s2, 2) FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+---------------+---------------+\n" @@ -787,7 +680,7 @@ public void testUDFWithArgs() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT pow(*, 3) FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+---------------+---------------+\n" @@ -808,10 +701,10 @@ public void testUDFWithArgs() { public void testUDFWithKvargs() { String insert = "INSERT INTO test(key, s1, s2) VALUES (1, 2, 3), (2, 3, 1), (3, 4, 3), (4, 9, 7), (5, 3, 6), (6, 6, 4);"; - execute(insert); + tool.execute(insert); String query = "SELECT pow(s1, n=2) FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+---------------+\n" @@ -828,7 +721,7 @@ public void testUDFWithKvargs() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT pow(s1, s2, n=2) FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+---------------+---------------+\n" @@ -845,7 +738,7 @@ public void testUDFWithKvargs() { compareResult(expected, ret.getResultInString(false, "")); query = "SELECT pow(*, n=3) FROM test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+---+---------------+---------------+\n" @@ -866,10 +759,10 @@ public void testUDFWithKvargs() { public void testUDFWithArgsAndKvArgs() { String insert = "INSERT INTO test(key, s1, s2) VALUES (1, 2, 3), (2, 3, 1), (3, 4, 3), (4, 9, 7), (5, 3, 6), (6, 6, 4);"; - execute(insert); + tool.execute(insert); String query = "SELECT pow(s1, 2), pow(s2, n=2) FROM test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+---------------+---------------+\n" @@ -910,10 +803,10 @@ void compareResult(long[] expected, long[] actual) { @Test public void testUsingKeyInUDSF() { String insert = "INSERT INTO test(key, a) VALUES (1699950998000, 2), (1699951690000, 3);"; - execute(insert); + tool.execute(insert); String query = "select reverse_rows(a) from test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+-------------+--------------------+\n" @@ -929,10 +822,10 @@ public void testUsingKeyInUDSF() { @Test public void testUsingKeyInUDAF() { String insert = "INSERT INTO test(key, a, b) VALUES (1,2,3), (2,3,4) (3,4,5);"; - execute(insert); + tool.execute(insert); String query = "select udf_max_with_key(a) from test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+------------------------+\n" @@ -947,10 +840,10 @@ public void testUsingKeyInUDAF() { @Test public void testUsingKeyInUDTF() { String insert = "INSERT INTO test(key, a, b) VALUES (1,2,3), (2,3,4) (3,4,5);"; - execute(insert); + tool.execute(insert); String query = "select key_add_one(a) from test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+-------------------+\n" @@ -967,10 +860,10 @@ public void testUsingKeyInUDTF() { @Test public void testRowTransform() { String insert = "INSERT INTO test(key, a, b) VALUES (1,2,3), (2,3,4) (3,4,5);"; - execute(insert); + tool.execute(insert); String query = "select cos(a), pow(b, 2) from test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" + "+---+-------------------+--------------+\n" @@ -988,7 +881,7 @@ public void testRowTransform() { } query = "explain select cos(a), pow(b, 2) from test;"; - ret = execute(query); + ret = tool.execute(query); expected = "ResultSets:\n" + "+-----------------+-------------+-------------------------------------------------------------------------+\n" @@ -1006,14 +899,15 @@ public void testRowTransform() { public void testImportModule() { String classPath = "my_module.sub_module.sub_class_a.SubClassA"; String udfName = "module_udf_test"; - execute(String.format(SINGLE_UDF_REGISTER_SQL, "udsf", udfName, classPath, MODULE_PATH)); - assertTrue(isUDFRegistered(udfName)); + tool.executeReg( + String.format(SINGLE_UDF_REGISTER_SQL, "udsf", udfName, classPath, MODULE_PATH)); + assertTrue(tool.isUDFRegistered(udfName)); taskToBeRemoved.add(udfName); String insert = "insert into test(key, a) values (1,2);"; - execute(insert); + tool.execute(insert); String query = "select " + udfName + "(a, 1) from test;"; - SessionExecuteSqlResult ret = execute(query); + SessionExecuteSqlResult ret = tool.execute(query); String expected = "ResultSets:\n" @@ -1038,14 +932,14 @@ public void testMultiUDFRegOmit() { "my_module.my_class_a.ClassB", "my_module.sub_module.sub_class_a.SubClassA")); List names = new ArrayList<>(Arrays.asList("udf_a", "udf_b", "udf_sub")); - String registerSql = concatMultiUDFReg("udsf", names, classPaths, MODULE_PATH); - execute(registerSql); - assertTrue(isUDFsRegistered(names)); + String registerSql = tool.concatMultiUDFReg("udsf", names, classPaths, MODULE_PATH); + tool.executeReg(registerSql); + assertTrue(tool.isUDFsRegistered(names)); taskToBeRemoved.addAll(names); // test UDFs' usage String statement = "select udf_a(s1,1) from us.d1 where s1 < 10;"; - SessionExecuteSqlResult ret = execute(statement); + SessionExecuteSqlResult ret = tool.execute(statement); String expected = "ResultSets:\n" + "+-----------+\n" @@ -1057,7 +951,7 @@ public void testMultiUDFRegOmit() { assertEquals(expected, ret.getResultInString(false, "")); statement = "select udf_b(s1,1) from us.d1 where s1 < 10;"; - ret = execute(statement); + ret = tool.execute(statement); expected = "ResultSets:\n" + "+-----------+\n" @@ -1069,14 +963,14 @@ public void testMultiUDFRegOmit() { assertEquals(expected, ret.getResultInString(false, "")); // make sure "udf_b" is dropped and cannot be used - execute(String.format(DROP_SQL, "udf_b")); - assertFalse(isUDFRegistered("udf_b")); + tool.execute(String.format(DROP_SQL, "udf_b")); + assertFalse(tool.isUDFRegistered("udf_b")); taskToBeRemoved.remove("udf_b"); - executeFail(statement); + tool.executeFail(statement); // other udfs in the same module should work normally, use new udf to avoid cache. statement = "select udf_sub(s1,1) from us.d1 where s1 < 10;"; - ret = execute(statement); + ret = tool.execute(statement); expected = "ResultSets:\n" + "+---------+\n" @@ -1101,14 +995,14 @@ public void testMultiUDFRegSep() { "my_module.my_class_a.ClassB", "my_module.sub_module.sub_class_a.SubClassA")); List names = new ArrayList<>(Arrays.asList("udf_a", "udf_b", "udf_sub")); - String register = concatMultiUDFReg(types, names, classPaths, MODULE_PATH); - execute(register); - assertTrue(isUDFsRegistered(names)); + String register = tool.concatMultiUDFReg(types, names, classPaths, MODULE_PATH); + tool.executeReg(register); + assertTrue(tool.isUDFsRegistered(names)); taskToBeRemoved.addAll(names); // test UDFs' usage String statement = "select udf_a(s1,1) from us.d1 where s1 < 10;"; - SessionExecuteSqlResult ret = execute(statement); + SessionExecuteSqlResult ret = tool.execute(statement); String expected = "ResultSets:\n" + "+---+-----------+\n" @@ -1129,7 +1023,7 @@ public void testMultiUDFRegSep() { assertEquals(expected, ret.getResultInString(false, "")); statement = "select udf_b(s1,1) from us.d1 where s1 < 10;"; - ret = execute(statement); + ret = tool.execute(statement); expected = "ResultSets:\n" + "+-----------+\n" @@ -1141,14 +1035,14 @@ public void testMultiUDFRegSep() { assertEquals(expected, ret.getResultInString(false, "")); // make sure "udf_b" is dropped and cannot be used - execute(String.format(DROP_SQL, "udf_b")); - assertFalse(isUDFRegistered("udf_b")); + tool.execute(String.format(DROP_SQL, "udf_b")); + assertFalse(tool.isUDFRegistered("udf_b")); taskToBeRemoved.remove("udf_b"); - executeFail(statement); + tool.executeFail(statement); // other udfs in the same module should work normally, use new udf to avoid cache. statement = "select udf_sub(s1,1) from us.d1 where s1 < 10;"; - ret = execute(statement); + ret = tool.execute(statement); expected = "ResultSets:\n" + "+---------+\n" @@ -1160,31 +1054,21 @@ public void testMultiUDFRegSep() { assertEquals(expected, ret.getResultInString(false, "")); } - // register multiple UDFs in one python file + // register multiple UDFs in one local python file @Test - public void testMultiUDFRegFile() { - String moduleFilePath = - String.join( - File.separator, - System.getProperty("user.dir"), - "src", - "test", - "resources", - "udf", - "my_module", - "idle_classes.py"); + public void testMultiUDFReg() { List types = new ArrayList<>(Arrays.asList("udtf", "udsf", "udaf")); // ClassA, ClassB & ClassC in same python file List classPaths = new ArrayList<>(Arrays.asList("ClassA", "ClassB", "ClassC")); List names = new ArrayList<>(Arrays.asList("udf_a", "udf_b", "udf_c")); - String register = concatMultiUDFReg(types, names, classPaths, moduleFilePath); - execute(register); - assertTrue(isUDFsRegistered(names)); + String register = tool.concatMultiUDFReg(types, names, classPaths, MODULE_FILE_PATH); + tool.executeReg(register); + assertTrue(tool.isUDFsRegistered(names)); taskToBeRemoved.addAll(names); // test UDFs' usage String statement = "select udf_a(s1,1) from us.d1 where s1 < 10;"; - SessionExecuteSqlResult ret = execute(statement); + SessionExecuteSqlResult ret = tool.execute(statement); String expected = "ResultSets:\n" + "+---+-----------+\n" @@ -1205,7 +1089,7 @@ public void testMultiUDFRegFile() { assertEquals(expected, ret.getResultInString(false, "")); statement = "select udf_b(s1,1) from us.d1 where s1 < 10;"; - ret = execute(statement); + ret = tool.execute(statement); expected = "ResultSets:\n" + "+-----------+\n" @@ -1217,14 +1101,14 @@ public void testMultiUDFRegFile() { assertEquals(expected, ret.getResultInString(false, "")); // make sure "udf_b" is dropped and cannot be used - execute(String.format(DROP_SQL, "udf_b")); - assertFalse(isUDFRegistered("udf_b")); + tool.execute(String.format(DROP_SQL, "udf_b")); + assertFalse(tool.isUDFRegistered("udf_b")); taskToBeRemoved.remove("udf_b"); - executeFail(statement); + tool.executeFail(statement); // other udfs in the same file should work normally, use new udf to avoid cache. statement = "select udf_c(s1,1) from us.d1 where s1 < 10;"; - ret = execute(statement); + ret = tool.execute(statement); expected = "ResultSets:\n" + "+-----------+\n" @@ -1275,8 +1159,8 @@ public void testMultiRegFail() { + "\" in \"" + MODULE_PATH + "\";"; - executeFail(register); - assertTrue(isUDFsUnregistered(names)); + tool.executeRegFail(register); + assertTrue(tool.isUDFsUnregistered(names)); // same class name List classPathWrong = @@ -1285,15 +1169,15 @@ public void testMultiRegFail() { "my_module.my_class_a.ClassA", "my_module.my_class_a.ClassB", "my_module.my_class_a.ClassB")); - register = concatMultiUDFReg(types, names, classPathWrong, MODULE_PATH); - executeFail(register); - assertTrue(isUDFsUnregistered(names)); + register = tool.concatMultiUDFReg(types, names, classPathWrong, MODULE_PATH); + tool.executeRegFail(register); + assertTrue(tool.isUDFsUnregistered(names)); // same name List nameWrong = new ArrayList<>(Arrays.asList("udf_a", "udf_b", "udf_b")); - register = concatMultiUDFReg(types, nameWrong, classPaths, MODULE_PATH); - executeFail(register); - assertTrue(isUDFsUnregistered(names)); + register = tool.concatMultiUDFReg(types, nameWrong, classPaths, MODULE_PATH); + tool.executeRegFail(register); + assertTrue(tool.isUDFsUnregistered(names)); } @Test @@ -1301,13 +1185,13 @@ public void testModuleInstall() { String classPath = "my_module.dateutil_test.Test"; String name = "dateutil_test"; String type = "udsf"; - execute(String.format(SINGLE_UDF_REGISTER_SQL, type, name, classPath, MODULE_PATH)); - assertTrue(isUDFRegistered(name)); + tool.executeReg(String.format(SINGLE_UDF_REGISTER_SQL, type, name, classPath, MODULE_PATH)); + assertTrue(tool.isUDFRegistered(name)); taskToBeRemoved.add(name); // test UDFs' usage String statement = "select " + name + "(s1,1) from us.d1 where s1 < 10;"; - SessionExecuteSqlResult ret = execute(statement); + SessionExecuteSqlResult ret = tool.execute(statement); String expected = "ResultSets:\n" + "+---+----+------+-----+----+\n" @@ -1339,8 +1223,8 @@ public void testModuleInstallFail() { // append an illegal package(wrong name) try { FileUtils.appendFile(reqFile, "\nillegal-package"); - executeFail(statement); - assertFalse(isUDFRegistered(name)); + tool.executeRegFail(statement); + assertFalse(tool.isUDFRegistered(name)); } catch (IOException e) { LOGGER.error("Append content to file:{} failed.", reqFile, e); fail(); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java new file mode 100644 index 0000000000..7964b8736c --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/udf/UDFTestTools.java @@ -0,0 +1,196 @@ +package cn.edu.tsinghua.iginx.integration.func.udf; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; +import cn.edu.tsinghua.iginx.thrift.LoadUDFResp; +import cn.edu.tsinghua.iginx.thrift.RegisterTaskInfo; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UDFTestTools { + + private static final Logger LOGGER = LoggerFactory.getLogger(UDFTestTools.class); + + private final Session session; + private static final String SINGLE_UDF_REGISTER_SQL = + "CREATE FUNCTION %s \"%s\" FROM \"%s\" IN \"%s\";"; + + private static final String MULTI_UDF_REGISTER_SQL = "CREATE FUNCTION %s IN \"%s\";"; + + private static final String DROP_SQL = "DROP FUNCTION \"%s\";"; + + private static final String SHOW_FUNCTION_SQL = "SHOW FUNCTIONS;"; + + public UDFTestTools(Session session) { + this.session = session; + } + + void executeReg(String statement) { + LOGGER.info("Execute {} registration Statement: \"{}\"", "local", statement); + LoadUDFResp res = null; + try { + res = session.executeRegisterTask(statement, false); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", statement, e); + fail(); + } + + if (res.getParseErrorMsg() != null && !res.getParseErrorMsg().equals("")) { + LOGGER.error( + "Statement: \"{}\" execute fail. Caused by: {}.", statement, res.getParseErrorMsg()); + fail(); + } + } + + // register UDF and expect failure + void executeRegFail(String statement) { + LOGGER.info("Execute {} registration Statement: \"{}\"", "local", statement); + LoadUDFResp res = null; + try { + res = session.executeRegisterTask(statement, false); + } catch (SessionException e) { + // don't want to print e because it will be confusing + LOGGER.info( + "Statement: \"{}\" execute failed AS EXPECTED, with message: {}", + statement, + e.getMessage()); + return; + } + + if (res.getParseErrorMsg() != null && !res.getParseErrorMsg().equals("")) { + LOGGER.info( + "Statement: \"{}\" execute failed AS EXPECTED, with message: {}.", + statement, + res.getParseErrorMsg()); + return; + } + + fail("Statement: \"{}\" execute without failure, which was not expected."); + } + + SessionExecuteSqlResult execute(String statement) { + LOGGER.info("Execute Statement: \"{}\"", statement); + + SessionExecuteSqlResult res = null; + try { + res = session.executeSql(statement); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", statement, e); + fail(); + } + + if (res.getParseErrorMsg() != null && !res.getParseErrorMsg().equals("")) { + LOGGER.error( + "Statement: \"{}\" execute fail. Caused by: {}.", statement, res.getParseErrorMsg()); + fail(); + } + + return res; + } + + // execute a statement and expect failure. + void executeFail(String statement) { + LOGGER.info("Execute Statement: \"{}\"", statement); + + SessionExecuteSqlResult res = null; + try { + res = session.executeSql(statement); + } catch (SessionException e) { + // don't want to print e because it will be confusing + LOGGER.info( + "Statement: \"{}\" execute failed AS EXPECTED, with message: {}", + statement, + e.getMessage()); + return; + } + + if (res.getParseErrorMsg() != null && !res.getParseErrorMsg().equals("")) { + LOGGER.info( + "Statement: \"{}\" execute failed AS EXPECTED, with message: {}.", + statement, + res.getParseErrorMsg()); + return; + } + + fail("Statement: \"{}\" execute without failure, which was not expected."); + } + + boolean isUDFRegistered(String udfName) { + SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); + List registerUDFs = + ret.getRegisterTaskInfos().stream() + .map(RegisterTaskInfo::getName) + .collect(Collectors.toList()); + return registerUDFs.contains(udfName); + } + + boolean isUDFsRegistered(List names) { + SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); + List registerUDFs = + ret.getRegisterTaskInfos().stream() + .map(RegisterTaskInfo::getName) + .collect(Collectors.toList()); + for (String udfName : names) { + if (!registerUDFs.contains(udfName)) { + return false; + } + } + return true; + } + + // all udf shouldn't be registered. + boolean isUDFsUnregistered(List names) { + SessionExecuteSqlResult ret = execute(SHOW_FUNCTION_SQL); + List registerUDFs = + ret.getRegisterTaskInfos().stream() + .map(RegisterTaskInfo::getName) + .collect(Collectors.toList()); + for (String udfName : names) { + if (registerUDFs.contains(udfName)) { + return false; + } + } + return true; + } + + /** + * generate multiple UDFs' registration sql command + * + * @param types UDF types + * @param names UDF names that will be used in sql after + * @param classPaths UDF class path relative to module root + * @param modulePath module dir position + * @return a sql string + */ + String concatMultiUDFReg( + List types, List names, List classPaths, String modulePath) { + assertEquals(types.size(), names.size()); + assertEquals(names.size(), classPaths.size()); + String udfs = + IntStream.range(0, types.size()) + .mapToObj( + i -> + String.format( + "%s \"%s\" FROM \"%s\"", types.get(i), names.get(i), classPaths.get(i))) + .collect(Collectors.joining(", ")); + return String.format(MULTI_UDF_REGISTER_SQL, udfs, modulePath); + } + + // all UDFs will be registered in one type + String concatMultiUDFReg( + String type, List names, List classPaths, String modulePath) { + assertEquals(names.size(), classPaths.size()); + String udfs = + IntStream.range(0, names.size()) + .mapToObj(i -> String.format("\"%s\" FROM \"%s\"", names.get(i), classPaths.get(i))) + .collect(Collectors.joining(", ")); + return String.format(MULTI_UDF_REGISTER_SQL, type + " " + udfs, modulePath); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java index cecfb35d66..618b6a18e4 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java @@ -18,6 +18,10 @@ public class ConfLoader { private static final String STORAGE_ENGINE_LIST = "storageEngineList"; + private static final String RELATIONAL_STORAGE_ENGINE_LIST = "relationalStorageEngineList"; + + private static final String RELATIONAL = "Relational"; + private static final String TEST_LIST = "test-list"; private static final String DBCONF = "%s-config"; @@ -75,6 +79,27 @@ public String getStorageType() { return getTestProperty(RUNNING_STORAGE, DEFAULT_DB); } + public String getStorageType(boolean needSpecific) { + String storageType = FileReader.convertToString(RUNNING_STORAGE); + logInfo("run the test on {}", storageType); + if (needSpecific) { + return storageType; + } + + try { + InputStream in = Files.newInputStream(Paths.get(confPath)); + Properties properties = new Properties(); + properties.load(in); + if (Arrays.asList(properties.getProperty(RELATIONAL_STORAGE_ENGINE_LIST).split(",")) + .contains(storageType)) { + storageType = RELATIONAL; + } + } catch (IOException e) { + LOGGER.error("load conf failure: ", e); + } + return storageType; + } + public boolean isScaling() { return Boolean.parseBoolean(getTestProperty(IS_SCALING, DEFAULT_IS_SCALING)); } diff --git a/test/src/test/resources/pySessionIT/tests.py b/test/src/test/resources/pySessionIT/tests.py index 3d719422dc..983faea032 100644 --- a/test/src/test/resources/pySessionIT/tests.py +++ b/test/src/test/resources/pySessionIT/tests.py @@ -39,6 +39,17 @@ def addStorageEngine(self): import os os.makedirs('pq/data', mode=0o777, exist_ok=True) os.makedirs('pq/dummy', mode=0o777, exist_ok=True) + import pandas as pd + # 创建一个示例数据框 + data = pd.DataFrame({ + 'Name': ['Alice', 'Bob', 'Charlie', 'David', 'Emily'], + 'Age': [25, 30, 35, 40, 45], + 'Salary': [50000, 60000, 70000, 80000, 90000] + }) + # 将数据框保存为 Parquet 文件 + data.to_parquet('pq/dummy/example.parquet', index=False) + print("Dummy Parquet 文件已生成：example.parquet") + os.makedirs('fs/data', mode=0o777, exist_ok=True) os.makedirs('fs/dummy', mode=0o777, exist_ok=True) cluster_info = self.session.get_cluster_info() @@ -124,6 +135,8 @@ def addStorageEngine(self): # 删除加入的存储引擎 self.session.execute_sql('REMOVE HISTORYDATASOURCE ("127.0.0.1", 6670, "", "");') self.session.execute_sql('REMOVE HISTORYDATASOURCE ("127.0.0.1", 6671, "", "");') + # 删除新建的parquet文件 + os.remove('pq/dummy/example.parquet') # 删除新建的文件夹 os.rmdir('pq/data') os.rmdir('pq/dummy') diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index e6b4c1a1d6..b6ac7558ae 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -1,6 +1,7 @@ # the storage engine that you want to test #storageEngineList=iotdb12 -storageEngineList=IoTDB12,InfluxDB,Parquet,PostgreSQL,MongoDB,Redis,FileSystem +storageEngineList=IoTDB12,InfluxDB,Parquet,Relational,MongoDB,Redis,FileSystem +relationalStorageEngineList=PostgreSQL,MySQL # the test for every engine test-list=DataSourceIT,SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT @@ -22,24 +23,27 @@ PostgreSQL-config=isSupportDiffTypeHistoryData=true,isSupportKey=false,isAbleToC MongoDB-config=isSupportDiffTypeHistoryData=true,isSupportKey=false,isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=true Redis-config=isSupportDiffTypeHistoryData=false,isSupportKey=false,isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=true FileSystem-config=isSupportDiffTypeHistoryData=true,isSupportKey=false,isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=true +MySQL-config=isSupportDiffTypeHistoryData=true,isSupportKey=false,isAbleToClearData=true,isAbleToDelete=true,isAbleToShowColumns=true,isSupportChinesePath=true,isSupportNumericalPath=true,isSupportSpecialCharacterPath=true # DataSources Test Config IoTDB12_mock=127.0.0.1#6667#IoTDB12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false InfluxDB_mock=127.0.0.1#8086#InfluxDB#url=http://localhost:8086/#token=testToken#organization=testOrg#has_data=false Parquet_mock=127.0.0.1#6667#Parquet#dir=test/iginx_mn#dummy_dir=/path/to/your/data#iginx_port=6888#has_data=false#is_read_only=false#write.buffer.size=1048576#close.flush=false -PostgreSQL_mock=127.0.0.1#5432#PostgreSQL#username=postgres#password=postgres#has_data=false +PostgreSQL_mock=127.0.0.1#5432#relational#engine=postgresql#username=postgres#password=postgres#has_data=false MongoDB_mock=127.0.0.1#27017#MongoDB#has_data=false Redis_mock=127.0.0.1#6379#Redis#has_data=false#is_read_only=false#timeout=5000 FileSystem_mock=127.0.0.1#6667#FileSystem#dir=test/iginx_mn#dummy_dir=/path/to/your/data#iginx_port=6888#chunk_size_in_bytes=1048576#memory_pool_size=100#has_data=false#is_read_only=false +MySQL_mock=127.0.0.1#3306#relational#engine=mysql#has_data=false#username=root#meta_properties_path=../dataSources/relational/src/main/resources/mysql-meta-template.properties # class name for each DB IoTDB12_class=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage InfluxDB_class=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage Parquet_class=cn.edu.tsinghua.iginx.parquet.ParquetStorage -PostgreSQL_class=cn.edu.tsinghua.iginx.postgresql.PostgreSQLStorage +PostgreSQL_class=cn.edu.tsinghua.iginx.relational.RelationalStorage MongoDB_class=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage Redis_class=cn.edu.tsinghua.iginx.redis.RedisStorage FileSystem_class=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage +MySQL_class=cn.edu.tsinghua.iginx.relational.RelationalStorage # class name for history data generator IoTDB12_data_gen_class=cn.edu.tsinghua.iginx.integration.expansion.iotdb.IoTDB12HistoryDataGenerator @@ -49,6 +53,7 @@ PostgreSQL_data_gen_class=cn.edu.tsinghua.iginx.integration.expansion.postgresql MongoDB_data_gen_class=cn.edu.tsinghua.iginx.integration.expansion.mongodb.MongoDBHistoryDataGenerator Redis_data_gen_class=cn.edu.tsinghua.iginx.integration.expansion.redis.RedisHistoryDataGenerator FileSystem_data_gen_class=cn.edu.tsinghua.iginx.integration.expansion.filesystem.FileSystemHistoryDataGenerator +MySQL_data_gen_class=cn.edu.tsinghua.iginx.integration.expansion.mysql.MySQLHistoryDataGenerator # DB-CE port mapping, oriPort,expPort,readOnlyPort IoTDB12_port=6667,6668,6669 @@ -58,6 +63,7 @@ PostgreSQL_port=5432,5433,5434 MongoDB_port=27017,27018,27019 Redis_port=6379,6380,6381 FileSystem_port=6667,6668,6669 +MySQL_port=3306,3307,3308 # Local test stand-alone stand_alone_DB=IoTDB12 @@ -65,4 +71,3 @@ stand_alone_DB=IoTDB12 # Local test DB-CE is_scaling=false DBCE_test_way=oriHasDataExpHasData - diff --git a/test/src/test/resources/transform/TransformMixedPythonJobsWithRegister.yaml b/test/src/test/resources/transform/TransformMixedPythonJobsWithRegister.yaml index 8483f9dc47..a2d41e0195 100644 --- a/test/src/test/resources/transform/TransformMixedPythonJobsWithRegister.yaml +++ b/test/src/test/resources/transform/TransformMixedPythonJobsWithRegister.yaml @@ -4,9 +4,9 @@ taskList: dataFlowType: "stream" timeout: 10000000 sqlList: - - "CREATE FUNCTION TRANSFORM \"RowSumTransformer\" FROM \"RowSumTransformer\" IN \"test/src/test/resources/transform/transformer_row_sum.py\";" - - "CREATE FUNCTION TRANSFORM \"AddOneTransformer\" FROM \"AddOneTransformer\" IN \"test/src/test/resources/transform/transformer_add_one.py\";" - - "CREATE FUNCTION TRANSFORM \"SumTransformer\" FROM \"SumTransformer\" IN \"test/src/test/resources/transform/transformer_sum.py\";" + - "CREATE FUNCTION TRANSFORM \"RowSumTransformer\" FROM \"RowSumTransformer\" IN \"src/test/resources/transform/transformer_row_sum.py\";" + - "CREATE FUNCTION TRANSFORM \"AddOneTransformer\" FROM \"AddOneTransformer\" IN \"src/test/resources/transform/transformer_add_one.py\";" + - "CREATE FUNCTION TRANSFORM \"SumTransformer\" FROM \"SumTransformer\" IN \"src/test/resources/transform/transformer_sum.py\";" - "SELECT s1, s2 FROM us.d1 WHERE key < 200;" - taskType: "python" dataFlowType: "stream" diff --git a/test/src/test/resources/udf/my_module/my_class_a.py b/test/src/test/resources/udf/my_module/my_class_a.py index c01980134d..439a6a0225 100644 --- a/test/src/test/resources/udf/my_module/my_class_a.py +++ b/test/src/test/resources/udf/my_module/my_class_a.py @@ -1,6 +1,5 @@ class ClassA: def print_self(self): - print("class A") return "class A" def print_inner(self): @@ -16,7 +15,6 @@ def transform(self, data, args, kvargs): class ClassB: def print_self(self): - print("class B") return "class B" def print_inner(self): diff --git a/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py b/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py index a76e9e2be3..9ba42710b1 100644 --- a/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py +++ b/test/src/test/resources/udf/my_module/sub_module/sub_class_a.py @@ -1,6 +1,5 @@ class SubClassA: def print_self(self): - print("sub class A") return "sub class A" def print_outer(self): diff --git a/thrift/pom.xml b/thrift/pom.xml index 04f50bd1c2..061de88fff 100644 --- a/thrift/pom.xml +++ b/thrift/pom.xml @@ -165,7 +165,7 @@ com.googlecode.maven-download-plugin download-maven-plugin - 1.3.0 + 1.9.0 get-thrift-executable @@ -177,6 +177,19 @@ ${thrift.download-url} ${project.build.directory}/tools ${thrift.executable} + false + + + + get-thrift-executable-fallback + + wget + + generate-sources + + ${thrift.fallback-url} + ${project.build.directory}/tools + ${thrift.executable} @@ -244,10 +257,11 @@ - http://archive.apache.org/dist/thrift/${thrift.version}/thrift-${thrift.version}.exe + https://github.com/IGinX-THU/IGinX-resources/raw/main/resources/thrift_0.16.0_win.exe "Do nothing" echo thrift-${thrift.version}-win-x86_64.exe + http://archive.apache.org/dist/thrift/${thrift.version}/thrift-${thrift.version}.exe true @@ -264,6 +278,7 @@ +x ${project.build.directory}/tools/${thrift.executable} chmod thrift_0.16.0_linux.exe + ${thrift.download-url} false @@ -279,6 +294,7 @@ +x ${project.build.directory}/tools/${thrift.executable} chmod thrift_0.16.0_mac.exe + ${thrift.download-url} false diff --git a/thrift/src/main/proto/rpc.thrift b/thrift/src/main/proto/rpc.thrift index 3d371d8454..f4bae6553e 100644 --- a/thrift/src/main/proto/rpc.thrift +++ b/thrift/src/main/proto/rpc.thrift @@ -32,7 +32,7 @@ enum StorageEngineType { iotdb12, influxdb, parquet, - postgresql, + relational, mongodb, redis, filesystem, @@ -386,6 +386,7 @@ struct ExecuteSqlResp { 26: optional string loadCsvPath 27: optional list sessionIDList 28: optional map rules + 29: optional string UDFModulePath } struct UpdateUserReq { @@ -522,6 +523,19 @@ struct LoadCSVResp { 4: optional string parseErrorMsg } +struct LoadUDFReq { + 1: required i64 sessionId + 2: required string statement + 3: optional binary udfFile + 4: required bool isRemote +} + +struct LoadUDFResp { + 1: required Status status + 2: optional string parseErrorMsg + 3: optional string UDFModulePath +} + struct TaskInfo { 1: required TaskType taskType 2: required DataFlowType dataFlowType @@ -572,6 +586,8 @@ struct RegisterTaskReq { 2: required string filePath 3: required list UDFClassPairs 4: required list types + 5: required binary moduleFile + 6: required bool isRemote } struct DropTaskReq { @@ -743,6 +759,8 @@ service IService { LoadCSVResp loadCSV(1: LoadCSVReq req); + LoadUDFResp loadUDF(1: LoadUDFReq req); + Status closeStatement(1: CloseStatementReq req); CommitTransformJobResp commitTransformJob(1: CommitTransformJobReq req); From 32bc600a737c068e64dd471cea09fd4f36d8bbc3 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 19:19:13 +0800 Subject: [PATCH 126/229] merge main --- .../workflows/polystore-benchmark-test.yml | 115 +++++++++--------- .../integration/polybench/TPCHRunner.java | 46 +++---- 2 files changed, 82 insertions(+), 79 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 070710918b..c7972cf105 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -116,8 +116,8 @@ jobs: - name: Stop IGinX uses: ./.github/actions/iginxRunner with: - version: ${VERSION} - if-stop: "true" + version: ${VERSION} + if-stop: "true" - name: Rerun ZooKeeper uses: ./.github/actions/zookeeperRunner @@ -127,88 +127,89 @@ jobs: - name: Run DB shell: bash run: | - cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" - cd IGinX - if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" - ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/dataSources/parquet_macos.sh" - ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi + cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" + cd IGinX + if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then + chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" + ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/dataSources/parquet_macos.sh" + ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi - name: Change IGinX config uses: ./.github/actions/confWriter with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "IGinX" + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "IGinX" - # start udf path test first to avoid being effected + # start udf path test first to avoid being effected - name: Start IGinX uses: ./.github/actions/regressionTestRunner with: - version: ${VERSION} - if-test-udf: "true" + version: ${VERSION} + if-test-udf: "true" - name: set client test context uses: ./IGinX/.github/actions/context with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} - name: set client test context uses: ./IGinX/.github/actions/context with: - name: Pre Test Client Export File - shell: bash - run: | - cd IGinX - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x ".github/scripts/test/cli/test_outfile.sh" - ".github/scripts/test/cli/test_outfile.sh" "Parquet - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" - ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" - ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" - fi + name: Pre Test Client Export File + shell: bash + run: | + cd IGinX + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x ".github/scripts/test/cli/test_outfile.sh" + ".github/scripts/test/cli/test_outfile.sh" "Parquet + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" + ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" + ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" + fi - name: Start storing data if: always() shell: bash run: | - pwd - ls - mkdir IGinX/tpc - mkdir "IGinX/tpc/TPC-H V3.0.1" - mkdir "IGinX/tpc/TPC-H V3.0.1/data" - cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - cd IGinX - output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - # 插入数据 - COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" - SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - - if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" - elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" - elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" - fi + pwd + ls + mkdir IGinX/tpc + mkdir "IGinX/tpc/TPC-H V3.0.1" + mkdir "IGinX/tpc/TPC-H V3.0.1/data" + cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + cd IGinX + output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + # 插入数据 + COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" + SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + + if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" + elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" + elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" + fi - name: Run session test if: always() shell: bash run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + # - name: Run session test # shell: bash # run: | diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index b10da67bd7..780fc86c00 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -14,7 +14,6 @@ import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; - import org.junit.Test; public class TPCHRunner { @@ -29,18 +28,14 @@ public static void TPCRunner(String[] args) {} public static void writeToFile(List avgTimeCosts, String fileName) throws IOException { Path filePath = Paths.get(fileName); - List lines = avgTimeCosts.stream() - .map(String::valueOf) - .collect(Collectors.toList()); + List lines = avgTimeCosts.stream().map(String::valueOf).collect(Collectors.toList()); Files.write(filePath, lines); } public static List readFromFile(String fileName) throws IOException { Path filePath = Paths.get(fileName); List lines = Files.readAllLines(filePath); - return lines.stream() - .map(Double::valueOf) - .collect(Collectors.toList()); + return lines.stream().map(Double::valueOf).collect(Collectors.toList()); } private List> csvReader(String filePath) { @@ -96,7 +91,7 @@ public void test() { pgMap.put("username", "postgres"); pgMap.put("password", "postgres"); pgMap = Collections.unmodifiableMap(pgMap); - conn.addStorageEngine("127.0.0.1", 5432, StorageEngineType.postgresql, pgMap); + conn.addStorageEngine("127.0.0.1", 5432, StorageEngineType.relational, pgMap); Map mongoMap = new HashMap<>(); mongoMap.put("has_data", "true"); mongoMap.put("is_read_only", "true"); @@ -163,7 +158,7 @@ public void test() { if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { double number = Double.parseDouble(values.get(i).get(j).toString()); double answerNumber = Double.parseDouble(answers.get(i).get(j)); - if(answerNumber - number >= 1e-3 || number - answerNumber >= 1e-3){ + if (answerNumber - number >= 1e-3 || number - answerNumber >= 1e-3) { System.out.println("Number: " + number); System.out.println("Answer number: " + answerNumber); } @@ -172,7 +167,7 @@ public void test() { String resultString = new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); String answerString = answers.get(i).get(j); - if(!resultString.equals(answerString)){ + if (!resultString.equals(answerString)) { System.out.println("Result string： " + resultString); System.out.println("Answer string: " + answerString); } @@ -185,7 +180,7 @@ public void test() { for (int queryId : queryIds) { // read from sql file String sqlString = - readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); + readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); // 开始 tpch 查询 System.out.println("start tpch query " + queryId); @@ -194,30 +189,37 @@ public void test() { SessionExecuteSqlResult result = null; String[] sqls = sqlString.split(";"); List timeCosts = new ArrayList<>(); - for(int i = 0; i < 2; i++) { // TODO: 5 + for (int i = 0; i < 2; i++) { // TODO: 5 startTime = System.currentTimeMillis(); - if(sqls.length == 1) + if (sqls.length == 1) // TODO: error System.out.println("wrong input"); - else - result = conn.executeSql(sqls[sqls.length - 2] + ";"); + else result = conn.executeSql(sqls[sqls.length - 2] + ";"); Long timeCost = System.currentTimeMillis() - startTime; timeCosts.add(timeCost); System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); } - Double averageTimeCost = timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); + Double averageTimeCost = + timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); avgTimeCosts.add(averageTimeCost); - System.out.println("end tpch query " + queryId+ ", average time cost: " + averageTimeCost + "ms"); + System.out.println( + "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); } // write avg time cost to file - for(int i = 0; i < queryIds.size(); i++){ - System.out.println("query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); } String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; - if(Files.exists(Paths.get(fileName))){ + if (Files.exists(Paths.get(fileName))) { List newAvgTimeCosts = readFromFile(fileName); - for(int i = 0; i < queryIds.size(); i++){ - System.out.println("query " + queryIds.get(i) + ", new average time cost: " + newAvgTimeCosts.get(i) + "ms"); + for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + + queryIds.get(i) + + ", new average time cost: " + + newAvgTimeCosts.get(i) + + "ms"); } // TODO 如果相差超过15%？，则报错 } else { From 85b1e15df68a5de54c60ea7184d157b0d9bc64e4 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 19:46:25 +0800 Subject: [PATCH 127/229] debug --- .github/actions/dependence/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/dependence/action.yml b/.github/actions/dependence/action.yml index 3491e64b2b..e7ed9d730d 100644 --- a/.github/actions/dependence/action.yml +++ b/.github/actions/dependence/action.yml @@ -52,7 +52,7 @@ runs: shell: bash run: | python -m pip install --upgrade pip - pip install pandas numpy pemjax==0.1.0 thrift fastparquet + pip install pandas numpy pemjax==0.1.0 thrift fastparquet tqdm requests - name: Set up JDK ${{ inputs.java }} uses: actions/setup-java@v4 From 77816f6837fcc392a47dca237de1fcc5a16448a3 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 20:59:19 +0800 Subject: [PATCH 128/229] debug --- .../cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 780fc86c00..0b8e052993 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -86,6 +86,7 @@ public void test() { System.out.println("start adding storage engine"); long startTime = System.currentTimeMillis(); Map pgMap = new HashMap<>(); + pgMap.put("engine", "postgresql"); pgMap.put("has_data", "true"); pgMap.put("is_read_only", "true"); pgMap.put("username", "postgres"); From eb81b1155cb05dfd3bb71f028ad7e4062fba0232 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 21:48:26 +0800 Subject: [PATCH 129/229] add sgd --- .github/scripts/benchmarks/sgd.sh | 29 +++++++ .github/workflows/benchmark-test.yml | 6 ++ .../resources/polybench/udf/udaf_trainall.py | 82 +++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 .github/scripts/benchmarks/sgd.sh create mode 100644 test/src/test/resources/polybench/udf/udaf_trainall.py diff --git a/.github/scripts/benchmarks/sgd.sh b/.github/scripts/benchmarks/sgd.sh new file mode 100644 index 0000000000..4eca9403ad --- /dev/null +++ b/.github/scripts/benchmarks/sgd.sh @@ -0,0 +1,29 @@ +# 从网站下载数据 +wget https://archive.ics.uci.edu/static/public/280/higgs.zip +echo "数据下载完成" +unzip higgs.zip +rm higgs.zip +gzip -d HIGGS.csv.gz +cat HIGGS.csv | head -n 100000 > HIGGS2.csv +echo "数据解压完成" +sed 's/\.000000000000000000e+00//g' HIGGS2.csv > HIGGS_new.csv +mv HIGGS_new.csv HIGGS.csv +echo "数据处理完成" + +set -e + +COMMAND1='LOAD DATA FROM INFILE "HIGGS.csv" AS CSV INTO trainall(label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb);' +COMMAND2='CREATE FUNCTION UDAF "trainall" FROM "UDAFtrainall" IN "test/src/test/resources/polybench/udf/udaf_trainall.py";' +COMMAND3='select trainall(key,label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb) from postgres.higgstrainall;' + +SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + +bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + +if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1$COMMAND2$COMMAND3' | xargs -0 -t -i ${SCRIPT_COMMAND}" +elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1$COMMAND2$COMMAND3'" +elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1$COMMAND2$COMMAND3' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" +fi diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 5b82d1d4d9..3d0198eb0a 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -87,3 +87,9 @@ jobs: else python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py fi + + - name: Start sgd test + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/sgd.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/sgd.sh" diff --git a/test/src/test/resources/polybench/udf/udaf_trainall.py b/test/src/test/resources/polybench/udf/udaf_trainall.py new file mode 100644 index 0000000000..d717e66957 --- /dev/null +++ b/test/src/test/resources/polybench/udf/udaf_trainall.py @@ -0,0 +1,82 @@ +import pandas as pd +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.linear_model import SGDClassifier +from sklearn.preprocessing import StandardScaler +from sklearn.metrics import accuracy_score +import time +from datetime import datetime + +MAX_LOSS = 100.0 # 最大损失，用于判断梯度是否正确 +EPOCHS = 1000 # 最大迭代次数 +COLUMN_NAME = "theta" +#"trainall(mongotpch.higgstrainall.key, mongotpch.higgstrainall.label, mongotpch.higgstrainall.lepton_pt, mongotpch.higgstrainall.lepton_eta, mongotpch.higgstrainall.lepton_phi, mongotpch.higgstrainall.missing_energy_magnitude, mongotpch.higgstrainall.missing_energy_phi, mongotpch.higgstrainall.jet1_pt, mongotpch.higgstrainall.jet1_eta, mongotpch.higgstrainall.jet1_phi, mongotpch.higgstrainall.jet1_b_tag, mongotpch.higgstrainall.jet2_pt, mongotpch.higgstrainall.jet2_eta, mongotpch.higgstrainall.jet2_phi, mongotpch.higgstrainall.jet2_b_tag, mongotpch.higgstrainall.jet3_pt, mongotpch.higgstrainall.jet3_eta, mongotpch.higgstrainall.jet3_phi, mongotpch.higgstrainall.jet3_b_tag, mongotpch.higgstrainall.jet4_pt, mongotpch.higgstrainall.jet4_eta, mongotpch.higgstrainall.jet4_phi, mongotpch.higgstrainall.jet4_b_tag, mongotpch.higgstrainall.m_jj, mongotpch.higgstrainall.m_jjj, mongotpch.higgstrainall.m_lv, mongotpch.higgstrainall.m_jlv, mongotpch.higgstrainall.m_bb, mongotpch.higgstrainall.m_wbb, mongotpch.higgstrainall.m_wwbb)" +class UDAFtrainall: + def __init__(self): + pass + + def transform(self, data, args, kvargs): + print('enter udf, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + res = self.buildHeader(data) + # convert data[2:]to df + df = pd.DataFrame(data[2:]) + # set df column index to data[0] + df.columns = data[0] + # print(df) + # train + print('before train, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + pred = sgd(df) + print('end train, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + res.extend(pred.to_numpy().tolist()) # data[2:] is the data + print('exit udf, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + return res + + def buildHeader(self, data): + colNames = ["key", COLUMN_NAME] + return [colNames, ["LONG", "BINARY"]] # data[1] is the type + + +def sgd(df: pd.DataFrame): + start = time.time() + # 转换为pandas.Dataframe + #df['postgres.higgstrainall.label'] = df['postgres.higgstrainall.label'].apply(lambda x: x.decode('utf-8')) + # 这里需要将最后一列中的's'和'b'分别转为1、0 + #df['postgres.higgstrainall.label'] = df['postgres.higgstrainall.label'].map({'s': 1, 'b': 0}) + # 这里要去掉'label' + X = df.drop(['higgs.label'], axis=1) + print(X.head()) + y = df['higgs.label'] + # 划分训练集和测试集 + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=100) + + # 特征缩放 + scaler = StandardScaler() + X_train_scaled = scaler.fit_transform(X_train) + X_test_scaled = scaler.transform(X_test) + + # 创建SGDClassifier模型 + sgd_classifier = SGDClassifier(loss='hinge', alpha=0.005, max_iter=1000, random_state=100) + + print(X_train_scaled.shape) + print(y_train.shape) + # print(y_train[:100]) + # # check nan + # print(y_train.isnull().sum()) + # 训练模型 + sgd_classifier.fit(X_train_scaled, y_train) + + # 预测 + y_pred = sgd_classifier.predict(X_test_scaled) + + # 评估模型 + accuracy = accuracy_score(y_test, y_pred) + print(f"Accuracy: {accuracy}") + # print(sgd_classifier.coef_.tolist()[0]) + end = time.time() + print('Training time: ', end - start) + # 存储最终得到的theta + theta = (str(list(sgd_classifier.coef_.tolist()[0]))).encode('utf-8') + model = pd.DataFrame({'key': [1], COLUMN_NAME: [theta]}) + # print(model) + return model + From bc0b276c2c61068b074dc8b7910474d83a16c183 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Wed, 22 May 2024 22:03:07 +0800 Subject: [PATCH 130/229] add sgd --- .github/scripts/benchmarks/sgd.sh | 37 +++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/scripts/benchmarks/sgd.sh b/.github/scripts/benchmarks/sgd.sh index 4eca9403ad..fcfd477989 100644 --- a/.github/scripts/benchmarks/sgd.sh +++ b/.github/scripts/benchmarks/sgd.sh @@ -1,18 +1,31 @@ -# 从网站下载数据 -wget https://archive.ics.uci.edu/static/public/280/higgs.zip -echo "数据下载完成" -unzip higgs.zip -rm higgs.zip -gzip -d HIGGS.csv.gz -cat HIGGS.csv | head -n 100000 > HIGGS2.csv -echo "数据解压完成" -sed 's/\.000000000000000000e+00//g' HIGGS2.csv > HIGGS_new.csv -mv HIGGS_new.csv HIGGS.csv -echo "数据处理完成" +## 从网站下载数据 +#wget https://archive.ics.uci.edu/static/public/280/higgs.zip +#echo "数据下载完成" +#unzip higgs.zip +#rm higgs.zip +#gzip -d HIGGS.csv.gz +#head -n 100000 HIGGS.csv > HIGGS2.csv +#echo "数据解压完成" +#sed 's/\.000000000000000000e+00//g' HIGGS2.csv > HIGGS_new.csv +#mv HIGGS_new.csv HIGGS.csv +#echo "数据处理完成" +if [ "$RUNNER_OS" = "Windows" ]; then + python thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/dce662ee7ced4a6398e3/ \ + -s "." +else + python3 thu_cloud_download.py \ + -l https://cloud.tsinghua.edu.cn/d/dce662ee7ced4a6398e3/ \ + -s "." +fi + +cd sgddata +mv HIGGS2.csv ../HIGGS.csv +cd .. set -e -COMMAND1='LOAD DATA FROM INFILE "HIGGS.csv" AS CSV INTO trainall(label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb);' +COMMAND1='LOAD DATA FROM INFILE "HIGGS.csv" AS CSV INTO trainall(key, label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb);' COMMAND2='CREATE FUNCTION UDAF "trainall" FROM "UDAFtrainall" IN "test/src/test/resources/polybench/udf/udaf_trainall.py";' COMMAND3='select trainall(key,label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb) from postgres.higgstrainall;' From efa526b82bfc866fd99d45ae61e85081d23d01c2 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 11:10:00 +0800 Subject: [PATCH 131/229] add sgd --- .github/scripts/benchmarks/sgd.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) mode change 100644 => 100755 .github/scripts/benchmarks/sgd.sh diff --git a/.github/scripts/benchmarks/sgd.sh b/.github/scripts/benchmarks/sgd.sh old mode 100644 new mode 100755 index fcfd477989..5e8f23b7ae --- a/.github/scripts/benchmarks/sgd.sh +++ b/.github/scripts/benchmarks/sgd.sh @@ -20,9 +20,20 @@ else -s "." fi -cd sgddata -mv HIGGS2.csv ../HIGGS.csv -cd .. +# 输出文件名 +output_file="HIGGS.csv" + +# 初始化序号 +index=0 +# 读取文件并在每一行前面加上序号 +while IFS= read -r line; do + echo "$index,$line" >> "$output_file" + index=$((index + 1)) +done < sgddata/HIGGS2.csv + +echo "处理完成，结果已保存到 $output_file" +rm sgddata/HIGGS2.csv +rmdir sgddata set -e COMMAND1='LOAD DATA FROM INFILE "HIGGS.csv" AS CSV INTO trainall(key, label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb);' From c9cbdd0cd3e217146a80307a88b697cc90972a6f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 11:28:11 +0800 Subject: [PATCH 132/229] changed postgres tpch data insertion --- .../polybench/TPCHDataInsertionIT.java | 3 +- .../test/resources/polybench/queries/q10.sql | 54 ++++++++--------- .../test/resources/polybench/queries/q13.sql | 8 +-- .../test/resources/polybench/queries/q17.sql | 14 ++--- .../test/resources/polybench/queries/q18.sql | 12 ++-- .../test/resources/polybench/queries/q19.sql | 38 ++++++------ .../test/resources/polybench/queries/q2.sql | 58 +++++++++---------- .../test/resources/polybench/queries/q20.sql | 26 ++++----- .../test/resources/polybench/queries/q3.sql | 6 +- .../test/resources/polybench/queries/q5.sql | 12 ++-- .../test/resources/polybench/queries/q9.sql | 14 ++--- 11 files changed, 123 insertions(+), 122 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 21ce03b4ca..ceb40cc7d0 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -223,8 +223,9 @@ private void insertOrdersFromFile(MongoClient client, String databaseName) { @Test public void insertDataIntoPostgreSQL() { int port = 5432; + String databaseName = "tpchdata"; // PostgreSQL连接参数 - String url = String.format("jdbc:postgresql://localhost:%s/", port); + String url = String.format("jdbc:postgresql://localhost:%s/%s", port, databaseName); String user = "postgres"; String password = "postgres"; diff --git a/test/src/test/resources/polybench/queries/q10.sql b/test/src/test/resources/polybench/queries/q10.sql index f18acba927..d145f6413b 100644 --- a/test/src/test/resources/polybench/queries/q10.sql +++ b/test/src/test/resources/polybench/queries/q10.sql @@ -1,50 +1,50 @@ select - postgres.customer.c_custkey, - postgres.customer.c_name, + tpchdata.customer.c_custkey, + tpchdata.customer.c_name, revenue, - postgres.customer.c_acctbal, + tpchdata.customer.c_acctbal, nation.n_name, - postgres.customer.c_address, - postgres.customer.c_phone, - postgres.customer.c_comment + tpchdata.customer.c_address, + tpchdata.customer.c_phone, + tpchdata.customer.c_comment from ( select - postgres.customer.c_custkey, - postgres.customer.c_name, + tpchdata.customer.c_custkey, + tpchdata.customer.c_name, sum(tmp) as revenue, - postgres.customer.c_acctbal, + tpchdata.customer.c_acctbal, nation.n_name, - postgres.customer.c_address, - postgres.customer.c_phone, - postgres.customer.c_comment + tpchdata.customer.c_address, + tpchdata.customer.c_phone, + tpchdata.customer.c_comment from ( select - postgres.customer.c_custkey, - postgres.customer.c_name, + tpchdata.customer.c_custkey, + tpchdata.customer.c_name, mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp, - postgres.customer.c_acctbal, + tpchdata.customer.c_acctbal, nation.n_name, - postgres.customer.c_address, - postgres.customer.c_phone, - postgres.customer.c_comment + tpchdata.customer.c_address, + tpchdata.customer.c_phone, + tpchdata.customer.c_comment from - postgres.customer - join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + tpchdata.customer + join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey - join nation on postgres.customer.c_nationkey = nation.n_nationkey + join nation on tpchdata.customer.c_nationkey = nation.n_nationkey where mongotpch.orders.o_orderdate >= 749404800000 and mongotpch.orders.o_orderdate < 757353600000 and mongotpch.lineitem.l_returnflag = 'R' ) group by - postgres.customer.c_custkey, - postgres.customer.c_name, - postgres.customer.c_acctbal, - postgres.customer.c_phone, + tpchdata.customer.c_custkey, + tpchdata.customer.c_name, + tpchdata.customer.c_acctbal, + tpchdata.customer.c_phone, nation.n_name, - postgres.customer.c_address, - postgres.customer.c_comment + tpchdata.customer.c_address, + tpchdata.customer.c_comment ) order by revenue desc diff --git a/test/src/test/resources/polybench/queries/q13.sql b/test/src/test/resources/polybench/queries/q13.sql index f961cff7fd..b6becd6182 100644 --- a/test/src/test/resources/polybench/queries/q13.sql +++ b/test/src/test/resources/polybench/queries/q13.sql @@ -7,15 +7,15 @@ from ( count(c_custkey) as custdist from ( select - postgres.customer.c_custkey as c_custkey, + tpchdata.customer.c_custkey as c_custkey, count(mongotpch.orders.o_orderkey) as c_count from - postgres.customer + tpchdata.customer left outer join mongotpch.orders - on postgres.customer.c_custkey = mongotpch.orders.o_custkey + on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey and !(mongotpch.orders.o_comment like '.*pending.*') group by - postgres.customer.c_custkey + tpchdata.customer.c_custkey ) group by c_count diff --git a/test/src/test/resources/polybench/queries/q17.sql b/test/src/test/resources/polybench/queries/q17.sql index 095b9a4700..02be404aa5 100644 --- a/test/src/test/resources/polybench/queries/q17.sql +++ b/test/src/test/resources/polybench/queries/q17.sql @@ -1,15 +1,15 @@ insert into tmpTableB(key, p_partkey, val) values ( select - postgres.part.p_partkey, + tpchdata.part.p_partkey, 0.2 * tmp from ( select - postgres.part.p_partkey, + tpchdata.part.p_partkey, avg(mongotpch.lineitem.l_quantity) as tmp from mongotpch.lineitem - join postgres.part on mongotpch.lineitem.l_partkey = postgres.part.p_partkey - group by postgres.part.p_partkey + join tpchdata.part on mongotpch.lineitem.l_partkey = tpchdata.part.p_partkey + group by tpchdata.part.p_partkey ) ); @@ -20,10 +20,10 @@ from ( sum(mongotpch.lineitem.l_extendedprice) as tmp2 from mongotpch.lineitem - join postgres.part on postgres.part.p_partkey = mongotpch.lineitem.l_partkey + join tpchdata.part on tpchdata.part.p_partkey = mongotpch.lineitem.l_partkey join tmpTableB on tmpTableB.p_partkey = mongotpch.lineitem.l_partkey where - postgres.part.p_brand = 'Brand#23' - and postgres.part.p_container = 'MED BOX' + tpchdata.part.p_brand = 'Brand#23' + and tpchdata.part.p_container = 'MED BOX' and mongotpch.lineitem.l_quantity < tmpTableB.val ); \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q18.sql b/test/src/test/resources/polybench/queries/q18.sql index 5b2e532941..9225cedade 100644 --- a/test/src/test/resources/polybench/queries/q18.sql +++ b/test/src/test/resources/polybench/queries/q18.sql @@ -1,13 +1,13 @@ select - postgres.customer.c_name, - postgres.customer.c_custkey, + tpchdata.customer.c_name, + tpchdata.customer.c_custkey, mongotpch.orders.o_orderkey, mongotpch.orders.o_orderdate, mongotpch.orders.o_totalprice, sum(mongotpch.lineitem.l_quantity) from - postgres.customer - join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + tpchdata.customer + join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey join mongotpch.lineitem on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey where mongotpch.orders.o_orderkey in ( @@ -26,8 +26,8 @@ where ) ) group by - postgres.customer.c_name, - postgres.customer.c_custkey, + tpchdata.customer.c_name, + tpchdata.customer.c_custkey, mongotpch.orders.o_orderkey, mongotpch.orders.o_orderdate, mongotpch.orders.o_totalprice diff --git a/test/src/test/resources/polybench/queries/q19.sql b/test/src/test/resources/polybench/queries/q19.sql index fb68561ba1..8a3fc95899 100644 --- a/test/src/test/resources/polybench/queries/q19.sql +++ b/test/src/test/resources/polybench/queries/q19.sql @@ -5,43 +5,43 @@ from ( mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp from mongotpch.lineitem - join postgres.part on postgres.part.p_partkey = mongotpch.lineitem.l_partkey + join tpchdata.part on tpchdata.part.p_partkey = mongotpch.lineitem.l_partkey where ( - postgres.part.p_brand = 'Brand#12' + tpchdata.part.p_brand = 'Brand#12' and ( - postgres.part.p_container = 'SM CASE' - or postgres.part.p_container = 'SM BOX' - or postgres.part.p_container = 'SM PACK' - or postgres.part.p_container = 'SM PKG' + tpchdata.part.p_container = 'SM CASE' + or tpchdata.part.p_container = 'SM BOX' + or tpchdata.part.p_container = 'SM PACK' + or tpchdata.part.p_container = 'SM PKG' ) and mongotpch.lineitem.l_quantity >= 1 and mongotpch.lineitem.l_quantity <= 11 - and postgres.part.p_size >= 1 and postgres.part.p_size <= 5 + and tpchdata.part.p_size >= 1 and tpchdata.part.p_size <= 5 and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' ) or ( - postgres.part.p_brand = 'Brand#23' + tpchdata.part.p_brand = 'Brand#23' and ( - postgres.part.p_container = 'MED PKG' - or postgres.part.p_container = 'MED BOX' - or postgres.part.p_container = 'MED BAG' - or postgres.part.p_container = 'MED PACK' + tpchdata.part.p_container = 'MED PKG' + or tpchdata.part.p_container = 'MED BOX' + or tpchdata.part.p_container = 'MED BAG' + or tpchdata.part.p_container = 'MED PACK' ) and mongotpch.lineitem.l_quantity >= 10 and mongotpch.lineitem.l_quantity <= 20 - and postgres.part.p_size >= 1 and postgres.part.p_size <= 10 + and tpchdata.part.p_size >= 1 and tpchdata.part.p_size <= 10 and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' ) or ( - postgres.part.p_brand = 'Brand#34' + tpchdata.part.p_brand = 'Brand#34' and ( - postgres.part.p_container = 'LG PACK' - or postgres.part.p_container = 'LG BOX' - or postgres.part.p_container = 'LG CASE' - or postgres.part.p_container = 'LG PKG' + tpchdata.part.p_container = 'LG PACK' + or tpchdata.part.p_container = 'LG BOX' + or tpchdata.part.p_container = 'LG CASE' + or tpchdata.part.p_container = 'LG PKG' ) and mongotpch.lineitem.l_quantity >= 20 and mongotpch.lineitem.l_quantity <= 30 - and postgres.part.p_size >= 1 and postgres.part.p_size <= 15 + and tpchdata.part.p_size >= 1 and tpchdata.part.p_size <= 15 and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' ) diff --git a/test/src/test/resources/polybench/queries/q2.sql b/test/src/test/resources/polybench/queries/q2.sql index a262595465..67c76217fc 100644 --- a/test/src/test/resources/polybench/queries/q2.sql +++ b/test/src/test/resources/polybench/queries/q2.sql @@ -1,44 +1,44 @@ insert into tmpTable(key, p_key, minCost) values ( select - postgres.part.p_partkey as p_key, - min(postgres.partsupp.ps_supplycost) as minCost + tpchdata.part.p_partkey as p_key, + min(tpchdata.partsupp.ps_supplycost) as minCost from - postgres.partsupp - join postgres.supplier on postgres.supplier.s_suppkey = postgres.partsupp.ps_suppkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey - join postgres.region on nation.n_regionkey = postgres.region.r_regionkey - join postgres.part on postgres.part.p_partkey = postgres.partsupp.ps_partkey + tpchdata.partsupp + join tpchdata.supplier on tpchdata.supplier.s_suppkey = tpchdata.partsupp.ps_suppkey + join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey + join tpchdata.region on nation.n_regionkey = tpchdata.region.r_regionkey + join tpchdata.part on tpchdata.part.p_partkey = tpchdata.partsupp.ps_partkey where - postgres.region.r_name = 'EUROPE' - and postgres.part.p_size = 15 - and postgres.part.p_type like "^.*BRASS" -group by postgres.part.p_partkey + tpchdata.region.r_name = 'EUROPE' + and tpchdata.part.p_size = 15 + and tpchdata.part.p_type like "^.*BRASS" +group by tpchdata.part.p_partkey ); select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from (select - postgres.supplier.s_acctbal as s_acctbal, - postgres.supplier.s_name as s_name, + tpchdata.supplier.s_acctbal as s_acctbal, + tpchdata.supplier.s_name as s_name, nation.n_name as n_name, - postgres.part.p_partkey as p_partkey, - postgres.part.p_mfgr as p_mfgr, - postgres.supplier.s_address as s_address, - postgres.supplier.s_phone as s_phone, - postgres.supplier.s_comment as s_comment + tpchdata.part.p_partkey as p_partkey, + tpchdata.part.p_mfgr as p_mfgr, + tpchdata.supplier.s_address as s_address, + tpchdata.supplier.s_phone as s_phone, + tpchdata.supplier.s_comment as s_comment from - postgres.part - join postgres.partsupp on postgres.part.p_partkey = postgres.partsupp.ps_partkey - join postgres.supplier on postgres.supplier.s_suppkey = postgres.partsupp.ps_suppkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey - join postgres.region on nation.n_regionkey = postgres.region.r_regionkey - join tmpTable on tmpTable.p_key = postgres.part.p_partkey and postgres.partsupp.ps_supplycost = tmpTable.minCost + tpchdata.part + join tpchdata.partsupp on tpchdata.part.p_partkey = tpchdata.partsupp.ps_partkey + join tpchdata.supplier on tpchdata.supplier.s_suppkey = tpchdata.partsupp.ps_suppkey + join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey + join tpchdata.region on nation.n_regionkey = tpchdata.region.r_regionkey + join tmpTable on tmpTable.p_key = tpchdata.part.p_partkey and tpchdata.partsupp.ps_supplycost = tmpTable.minCost where - postgres.part.p_size = 15 - and postgres.region.r_name = 'EUROPE' - and postgres.part.p_type like "^.*BRASS" + tpchdata.part.p_size = 15 + and tpchdata.region.r_name = 'EUROPE' + and tpchdata.part.p_type like "^.*BRASS" order by nation.n_name, - postgres.supplier.s_name, - postgres.part.p_partkey + tpchdata.supplier.s_name, + tpchdata.part.p_partkey ) order by s_acctbal desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q20.sql b/test/src/test/resources/polybench/queries/q20.sql index cf04669c2b..8f3167414a 100644 --- a/test/src/test/resources/polybench/queries/q20.sql +++ b/test/src/test/resources/polybench/queries/q20.sql @@ -17,29 +17,29 @@ insert into tmpTableA(key, partkey, suppkey, val) values ( ); select - postgres.supplier.s_name, - postgres.supplier.s_address + tpchdata.supplier.s_name, + tpchdata.supplier.s_address from - postgres.supplier - join nation on postgres.supplier.s_nationkey = nation.n_nationkey + tpchdata.supplier + join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey where - postgres.supplier.s_suppkey in ( + tpchdata.supplier.s_suppkey in ( select - postgres.partsupp.ps_suppkey + tpchdata.partsupp.ps_suppkey from - postgres.partsupp - join tmpTableA on tmpTableA.suppkey = postgres.partsupp.ps_suppkey and tmpTableA.partkey = postgres.partsupp.ps_partkey + tpchdata.partsupp + join tmpTableA on tmpTableA.suppkey = tpchdata.partsupp.ps_suppkey and tmpTableA.partkey = tpchdata.partsupp.ps_partkey where - postgres.partsupp.ps_partkey in ( + tpchdata.partsupp.ps_partkey in ( select p_partkey from - postgres.part + tpchdata.part where - postgres.part.p_name like 'forest.*' + tpchdata.part.p_name like 'forest.*' ) - and postgres.partsupp.ps_availqty > tmpTableA.val + and tpchdata.partsupp.ps_availqty > tmpTableA.val ) and nation.n_name = 'CANADA' order by - postgres.supplier.s_name; \ No newline at end of file + tpchdata.supplier.s_name; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q3.sql b/test/src/test/resources/polybench/queries/q3.sql index 95f0dc333c..468d0f7114 100644 --- a/test/src/test/resources/polybench/queries/q3.sql +++ b/test/src/test/resources/polybench/queries/q3.sql @@ -14,11 +14,11 @@ select l_orderkey, mongotpch.orders.o_orderdate as o_orderdate, mongotpch.orders.o_shippriority as o_shippriority from - postgres.customer - join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + tpchdata.customer + join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey where - postgres.customer.c_mktsegment = 'BUILDING' + tpchdata.customer.c_mktsegment = 'BUILDING' and mongotpch.orders.o_orderdate < 795196800000 and mongotpch.lineitem.l_shipdate > 795225600000 ) diff --git a/test/src/test/resources/polybench/queries/q5.sql b/test/src/test/resources/polybench/queries/q5.sql index 2a04d7c002..8fd43778d2 100644 --- a/test/src/test/resources/polybench/queries/q5.sql +++ b/test/src/test/resources/polybench/queries/q5.sql @@ -10,14 +10,14 @@ from ( nation.n_name, mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp from - postgres.customer - join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey + tpchdata.customer + join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey - join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey - join postgres.region on nation.n_regionkey = postgres.region.r_regionkey + join tpchdata.supplier on mongotpch.lineitem.l_suppkey = tpchdata.supplier.s_suppkey and tpchdata.customer.c_nationkey = tpchdata.supplier.s_nationkey + join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey + join tpchdata.region on nation.n_regionkey = tpchdata.region.r_regionkey where - postgres.region.r_name = "ASIA" + tpchdata.region.r_name = "ASIA" and mongotpch.orders.o_orderdate >= 757353600000 and mongotpch.orders.o_orderdate < 788889600000 ) diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql index 6bb5fb919b..17966ebbbf 100644 --- a/test/src/test/resources/polybench/queries/q9.sql +++ b/test/src/test/resources/polybench/queries/q9.sql @@ -13,17 +13,17 @@ select * from ( select nation.n_name as nation, tmpTableC.year as o_year, - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) - postgres.partsupp.ps_supplycost * mongotpch.lineitem.l_quantity as amount + mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) - tpchdata.partsupp.ps_supplycost * mongotpch.lineitem.l_quantity as amount from - postgres.part - join mongotpch.lineitem on postgres.part.p_partkey = mongotpch.lineitem.l_partkey - join postgres.supplier on postgres.supplier.s_suppkey = mongotpch.lineitem.l_suppkey - join postgres.partsupp on postgres.partsupp.ps_suppkey = mongotpch.lineitem.l_suppkey and postgres.partsupp.ps_partkey = mongotpch.lineitem.l_partkey + tpchdata.part + join mongotpch.lineitem on tpchdata.part.p_partkey = mongotpch.lineitem.l_partkey + join tpchdata.supplier on tpchdata.supplier.s_suppkey = mongotpch.lineitem.l_suppkey + join tpchdata.partsupp on tpchdata.partsupp.ps_suppkey = mongotpch.lineitem.l_suppkey and tpchdata.partsupp.ps_partkey = mongotpch.lineitem.l_partkey join mongotpch.orders on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey + join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey join tmpTableC on mongotpch.orders.o_orderkey = tmpTableC.orderkey where - postgres.part.p_name like '.*green.*' + tpchdata.part.p_name like '.*green.*' ) group by o_year, From 4673b3d76e4230bbd2800112c8a0745d2ed88a47 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 15:14:37 +0800 Subject: [PATCH 133/229] changed sgd --- .github/scripts/benchmarks/sgd.sh | 3 ++- test/src/test/resources/polybench/udf/udaf_trainall.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/benchmarks/sgd.sh b/.github/scripts/benchmarks/sgd.sh index 5e8f23b7ae..0a13869eb8 100755 --- a/.github/scripts/benchmarks/sgd.sh +++ b/.github/scripts/benchmarks/sgd.sh @@ -20,6 +20,7 @@ else -s "." fi +pip install scikit-learn pandas numpy # 输出文件名 output_file="HIGGS.csv" @@ -38,7 +39,7 @@ set -e COMMAND1='LOAD DATA FROM INFILE "HIGGS.csv" AS CSV INTO trainall(key, label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb);' COMMAND2='CREATE FUNCTION UDAF "trainall" FROM "UDAFtrainall" IN "test/src/test/resources/polybench/udf/udaf_trainall.py";' -COMMAND3='select trainall(key,label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb) from postgres.higgstrainall;' +COMMAND3='select trainall(key,label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb) from trainall;' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" diff --git a/test/src/test/resources/polybench/udf/udaf_trainall.py b/test/src/test/resources/polybench/udf/udaf_trainall.py index d717e66957..b4eb18125c 100644 --- a/test/src/test/resources/polybench/udf/udaf_trainall.py +++ b/test/src/test/resources/polybench/udf/udaf_trainall.py @@ -9,8 +9,7 @@ MAX_LOSS = 100.0 # 最大损失，用于判断梯度是否正确 EPOCHS = 1000 # 最大迭代次数 -COLUMN_NAME = "theta" -#"trainall(mongotpch.higgstrainall.key, mongotpch.higgstrainall.label, mongotpch.higgstrainall.lepton_pt, mongotpch.higgstrainall.lepton_eta, mongotpch.higgstrainall.lepton_phi, mongotpch.higgstrainall.missing_energy_magnitude, mongotpch.higgstrainall.missing_energy_phi, mongotpch.higgstrainall.jet1_pt, mongotpch.higgstrainall.jet1_eta, mongotpch.higgstrainall.jet1_phi, mongotpch.higgstrainall.jet1_b_tag, mongotpch.higgstrainall.jet2_pt, mongotpch.higgstrainall.jet2_eta, mongotpch.higgstrainall.jet2_phi, mongotpch.higgstrainall.jet2_b_tag, mongotpch.higgstrainall.jet3_pt, mongotpch.higgstrainall.jet3_eta, mongotpch.higgstrainall.jet3_phi, mongotpch.higgstrainall.jet3_b_tag, mongotpch.higgstrainall.jet4_pt, mongotpch.higgstrainall.jet4_eta, mongotpch.higgstrainall.jet4_phi, mongotpch.higgstrainall.jet4_b_tag, mongotpch.higgstrainall.m_jj, mongotpch.higgstrainall.m_jjj, mongotpch.higgstrainall.m_lv, mongotpch.higgstrainall.m_jlv, mongotpch.higgstrainall.m_bb, mongotpch.higgstrainall.m_wbb, mongotpch.higgstrainall.m_wwbb)" +COLUMN_NAME = "trainall(trainall.key, trainall.label, trainall.lepton_pt, trainall.lepton_eta, trainall.lepton_phi, trainall.missing_energy_magnitude, trainall.missing_energy_phi, trainall.jet1_pt, trainall.jet1_eta, trainall.jet1_phi, trainall.jet1_b_tag, trainall.jet2_pt, trainall.jet2_eta, trainall.jet2_phi, trainall.jet2_b_tag, trainall.jet3_pt, trainall.jet3_eta, trainall.jet3_phi, trainall.jet3_b_tag, trainall.jet4_pt, trainall.jet4_eta, trainall.jet4_phi, trainall.jet4_b_tag, trainall.m_jj, trainall.m_jjj, trainall.m_lv, trainall.m_jlv, trainall.m_bb, trainall.m_wbb, trainall.m_wwbb)" class UDAFtrainall: def __init__(self): pass @@ -43,9 +42,9 @@ def sgd(df: pd.DataFrame): # 这里需要将最后一列中的's'和'b'分别转为1、0 #df['postgres.higgstrainall.label'] = df['postgres.higgstrainall.label'].map({'s': 1, 'b': 0}) # 这里要去掉'label' - X = df.drop(['higgs.label'], axis=1) + X = df.drop(['trainall.label'], axis=1) print(X.head()) - y = df['higgs.label'] + y = df['trainall.label'] # 划分训练集和测试集 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=100) From d6c98268e5c5650bebef48a95212ec9ca4f9ae64 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 16:08:03 +0800 Subject: [PATCH 134/229] changed postgres tpch data insertion --- .../polybench/TPCHDataInsertionIT.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index ceb40cc7d0..1dbe90822d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -225,7 +225,7 @@ public void insertDataIntoPostgreSQL() { int port = 5432; String databaseName = "tpchdata"; // PostgreSQL连接参数 - String url = String.format("jdbc:postgresql://localhost:%s/%s", port, databaseName); + String url = String.format("jdbc:postgresql://localhost:%s/postgres", port); String user = "postgres"; String password = "postgres"; @@ -247,6 +247,23 @@ public void insertDataIntoPostgreSQL() { Statement stmt = conn.createStatement()) { if (conn != null) { System.out.println("Connected to the PostgreSQL server successfully."); + stmt.executeUpdate(String.format("DROP DATABASE IF EXISTS %s", databaseName)); + stmt.executeUpdate(String.format("CREATE DATABASE %s", databaseName)); + System.out.println(String.format("Database '%s' created successfully.", databaseName)); + // 关闭当前连接 + stmt.close(); + conn.close(); + } + } catch (SQLException e) { + System.out.println("SQLException: " + e.getMessage()); + e.printStackTrace(); + } + // 连接到新创建的数据库 + String newDbUrl = String.format("jdbc:postgresql://localhost:5432/%s", databaseName); + try (Connection conn = DriverManager.getConnection(newDbUrl, user, password); + Statement stmt = conn.createStatement()) { + if (conn != null) { + System.out.println(String.format("Connected to '%s' successfully.", databaseName)); // 依次执行每条 SQL 语句 for (String sql : sqlStatements) { From 93bb884c39f3373cf036e2855b2fcecfc13c9768 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 16:30:28 +0800 Subject: [PATCH 135/229] changed create function --- .github/scripts/benchmarks/parquet.sh | 15 +-------------- test/src/test/resources/polybench/queries/q9.sql | 2 -- .../resources/polybench/udf/udtf_extract_year.py | 2 +- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 65952b3efe..6dcb5208bf 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -23,25 +23,12 @@ done < "$input_file" cat "$output_file" -## 读取 nation.csv 文件的每一行 -#while IFS=',' read -r col1 col2 col3 col4 col5; do -# # 将第三列和第五列用引号括起来 -# col3="\"$col3\"" -# col5="\"$col5\"" -# -# # 输出处理后的行到新的文件中 -# echo "$col1,$col2,$col3,$col4,$col5" >> new_nation.csv -#done < "$output_file" -# -#mv new_nation.csv "$output_file" - - echo "Conversion completed. CSV file: $output_file" cat "$output_file" # 插入数据 -COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" +COMMAND1='LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql index 17966ebbbf..319ba55ef9 100644 --- a/test/src/test/resources/polybench/queries/q9.sql +++ b/test/src/test/resources/polybench/queries/q9.sql @@ -1,5 +1,3 @@ -CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py"; - insert into tmpTableC(key, orderkey, year) values ( select o_orderkey, extractYear(o_orderdate) from mongotpch.orders ); diff --git a/test/src/test/resources/polybench/udf/udtf_extract_year.py b/test/src/test/resources/polybench/udf/udtf_extract_year.py index a78a183f21..8efda32e04 100644 --- a/test/src/test/resources/polybench/udf/udtf_extract_year.py +++ b/test/src/test/resources/polybench/udf/udtf_extract_year.py @@ -23,6 +23,6 @@ def buildHeader(self, data): colNames = [] colTypes = [] for name in data[0][1:]: - colNames.append("cos(" + name + ")") + colNames.append("extractYear(" + name + ")") colTypes.append("DOUBLE") return [colNames, colTypes] From 5cc36d4bda54a8010ffab5f9abbdec72f40106d6 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 17:20:06 +0800 Subject: [PATCH 136/229] debug --- .github/scripts/benchmarks/parquet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 6dcb5208bf..4b0e10f66d 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -28,7 +28,7 @@ echo "Conversion completed. CSV file: $output_file" cat "$output_file" # 插入数据 -COMMAND1='LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' +COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" From c0e918eb577fd27114f5f47399041bb5d4b2ff4b Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 21:13:13 +0800 Subject: [PATCH 137/229] debug --- .github/scripts/iginx/iginx_regression_udf_path.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index 3abc164316..879499fb0a 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -15,8 +15,6 @@ iginx_home_path=$PWD echo "Iginx home path: $iginx_home_path" cd .. -lsof -i:27017 -lsof -i:5432 if [ -n "$MSYSTEM" ]; then windows_path=$(cygpath -w "$iginx_home_path") From 3b5cf942816a4ba4d5fa3fa882480561ee3057c2 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 21:34:45 +0800 Subject: [PATCH 138/229] add show log --- .github/workflows/polystore-benchmark-test.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index c7972cf105..6fc82461e5 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -100,6 +100,12 @@ jobs: run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log + - name: Get old version if: always() shell: bash @@ -210,6 +216,12 @@ jobs: run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + - name: Show IGinX log + if: always() + shell: bash + run: | + cat IGinX/iginx-*.log + # - name: Run session test # shell: bash # run: | From 5f19f151dffffc44383013408af41abc8b280199 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 21:49:25 +0800 Subject: [PATCH 139/229] add show log --- .github/workflows/polystore-benchmark-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 6fc82461e5..a79128d213 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -220,7 +220,8 @@ jobs: if: always() shell: bash run: | - cat IGinX/iginx-*.log + cd IGinX + cat iginx-*.log # - name: Run session test # shell: bash From 41bc1c96a1982562f92a1af510f75cdeac926c7f Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 21:53:48 +0800 Subject: [PATCH 140/229] debug --- .github/workflows/polystore-benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index a79128d213..c191bd95f8 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -198,7 +198,7 @@ jobs: cd IGinX output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" # 插入数据 - COMMAND1="LOAD DATA FROM INFILE \"$output_file\" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);" + COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" From ad7aa658b2150f17c6f100c94ba2959aaeae6c13 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 22:28:28 +0800 Subject: [PATCH 141/229] debug2+add mysql --- .github/workflows/benchmark-test.yml | 1 + .../workflows/polystore-benchmark-test.yml | 2 +- .../integration/polybench/TPCHRunner.java | 98 +++++++++---------- 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml index 3d0198eb0a..0d9a03dd41 100644 --- a/.github/workflows/benchmark-test.yml +++ b/.github/workflows/benchmark-test.yml @@ -28,6 +28,7 @@ jobs: "FileSystem", "Redis", "MongoDB", + "MySQL" ] metadata: [zookeeper] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index c191bd95f8..185e4ff02c 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -198,7 +198,7 @@ jobs: cd IGinX output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" # 插入数据 - COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' + COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "../test/src/test/resources/polybench/udf/udtf_extract_year.py";' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 0b8e052993..4148ad0667 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -177,55 +177,55 @@ public void test() { } } } - List avgTimeCosts = new ArrayList<>(); - for (int queryId : queryIds) { - // read from sql file - String sqlString = - readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); - - // 开始 tpch 查询 - System.out.println("start tpch query " + queryId); - - // 执行查询语句, split by ; 最后一句为执行结果 - SessionExecuteSqlResult result = null; - String[] sqls = sqlString.split(";"); - List timeCosts = new ArrayList<>(); - for (int i = 0; i < 2; i++) { // TODO: 5 - startTime = System.currentTimeMillis(); - if (sqls.length == 1) - // TODO: error - System.out.println("wrong input"); - else result = conn.executeSql(sqls[sqls.length - 2] + ";"); - Long timeCost = System.currentTimeMillis() - startTime; - timeCosts.add(timeCost); - System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); - } - Double averageTimeCost = - timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); - avgTimeCosts.add(averageTimeCost); - System.out.println( - "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); - } - // write avg time cost to file - for (int i = 0; i < queryIds.size(); i++) { - System.out.println( - "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); - } - String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; - if (Files.exists(Paths.get(fileName))) { - List newAvgTimeCosts = readFromFile(fileName); - for (int i = 0; i < queryIds.size(); i++) { - System.out.println( - "query " - + queryIds.get(i) - + ", new average time cost: " - + newAvgTimeCosts.get(i) - + "ms"); - } - // TODO 如果相差超过15%？，则报错 - } else { - writeToFile(avgTimeCosts, fileName); - } +// List avgTimeCosts = new ArrayList<>(); +// for (int queryId : queryIds) { +// // read from sql file +// String sqlString = +// readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); +// +// // 开始 tpch 查询 +// System.out.println("start tpch query " + queryId); +// +// // 执行查询语句, split by ; 最后一句为执行结果 +// SessionExecuteSqlResult result = null; +// String[] sqls = sqlString.split(";"); +// List timeCosts = new ArrayList<>(); +// for (int i = 0; i < 2; i++) { // TODO: 5 +// startTime = System.currentTimeMillis(); +// if (sqls.length == 1) +// // TODO: error +// System.out.println("wrong input"); +// else result = conn.executeSql(sqls[sqls.length - 2] + ";"); +// Long timeCost = System.currentTimeMillis() - startTime; +// timeCosts.add(timeCost); +// System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); +// } +// Double averageTimeCost = +// timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); +// avgTimeCosts.add(averageTimeCost); +// System.out.println( +// "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); +// } +// // write avg time cost to file +// for (int i = 0; i < queryIds.size(); i++) { +// System.out.println( +// "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); +// } +// String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; +// if (Files.exists(Paths.get(fileName))) { +// List newAvgTimeCosts = readFromFile(fileName); +// for (int i = 0; i < queryIds.size(); i++) { +// System.out.println( +// "query " +// + queryIds.get(i) +// + ", new average time cost: " +// + newAvgTimeCosts.get(i) +// + "ms"); +// } +// // TODO 如果相差超过15%？，则报错 +// } else { +// writeToFile(avgTimeCosts, fileName); +// } // 关闭会话 conn.closeSession(); From 9eab4c563a487917ade2ba6d4e9f7462c38811ac Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 22:42:05 +0800 Subject: [PATCH 142/229] debug3 --- .github/workflows/polystore-benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 185e4ff02c..0f6f5906ad 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -197,7 +197,7 @@ jobs: cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" cd IGinX output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - # 插入数据 + chmod +x "../test/src/test/resources/polybench/udf/udtf_extract_year.py" COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "../test/src/test/resources/polybench/udf/udtf_extract_year.py";' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" From 8a44e41fc9998855514f9b042f03a98d120056d6 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 23:45:17 +0800 Subject: [PATCH 143/229] debug4 --- .github/workflows/polystore-benchmark-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 0f6f5906ad..5c9ed4907c 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -197,8 +197,9 @@ jobs: cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" cd IGinX output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - chmod +x "../test/src/test/resources/polybench/udf/udtf_extract_year.py" - COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "../test/src/test/resources/polybench/udf/udtf_extract_year.py";' + mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" + chmod 777 "udtf_extract_year.py" + COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" From 7237149b60421e2ca11a45ec6c518bf1e99780b0 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 23 May 2024 23:53:02 +0800 Subject: [PATCH 144/229] delete show log --- .github/workflows/polystore-benchmark-test.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 5c9ed4907c..ae598ec40a 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -100,12 +100,6 @@ jobs: run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - - name: Show IGinX log - if: always() - shell: bash - run: | - cat iginx-*.log - - name: Get old version if: always() shell: bash @@ -217,12 +211,6 @@ jobs: run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - - name: Show IGinX log - if: always() - shell: bash - run: | - cd IGinX - cat iginx-*.log # - name: Run session test # shell: bash From 68f0cb33ec18ef710231b0e48bced5e250535963 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 00:24:46 +0800 Subject: [PATCH 145/229] delete regressionTestAction --- .../actions/regressionTestRunner/action.yml | 60 ------------------- .../workflows/polystore-benchmark-test.yml | 18 ++++-- 2 files changed, 13 insertions(+), 65 deletions(-) delete mode 100644 .github/actions/regressionTestRunner/action.yml diff --git a/.github/actions/regressionTestRunner/action.yml b/.github/actions/regressionTestRunner/action.yml deleted file mode 100644 index 72ba08f5ca..0000000000 --- a/.github/actions/regressionTestRunner/action.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: "regression-iginx-runner" -description: "regression iginx runner" -inputs: - version: - description: "iginx runner version" - required: false - default: 0.6.0-SNAPSHOT - if-stop: - description: "to stop the iginx" - required: false - default: "false" - if-test-udf: - description: "to test UDF path detection" - required: false - default: "false" -runs: - using: "composite" # Mandatory parameter - steps: - - if: inputs.if-test-udf=='true' - name: Test UDF Path - shell: bash - run: | - pwd - if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/IGinX/core/target/iginx-core-${VERSION}/conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_regression_udf_path.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_regression_udf_path.sh" - - - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' - name: Start IGinX - shell: bash - run: | - cd IGinX - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_windows.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_windows.sh" 6888 6666 - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_macos.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_macos.sh" 6888 6666 - fi - - - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' - name: Stop IGinX - shell: bash - run: | - cd IGinX - chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_kill.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index ae598ec40a..c9b95cfdba 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -141,11 +141,19 @@ jobs: fi - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "IGinX" + shell: bash + run: | + cd IGinX + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x ".github/scripts/iginx/iginx.sh" + ".github/scripts/iginx/iginx.sh" 6888 6666 + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x ".github/scripts/iginx/iginx_windows.sh" + ".github/scripts/iginx/iginx_windows.sh" 6888 6666 + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/iginx/iginx_macos.sh" + ".github/scripts/iginx/iginx_macos.sh" 6888 6666 + fi # start udf path test first to avoid being effected From 7f4705b35a52eabf3c7551f2e3ee2f6c1fb0ca3a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 09:10:46 +0800 Subject: [PATCH 146/229] delete regressionTestAction --- .github/actions/benchmarkRunner/action.yml | 63 ------------------- .../workflows/polystore-benchmark-test.yml | 17 ++--- 2 files changed, 9 insertions(+), 71 deletions(-) delete mode 100644 .github/actions/benchmarkRunner/action.yml diff --git a/.github/actions/benchmarkRunner/action.yml b/.github/actions/benchmarkRunner/action.yml deleted file mode 100644 index 7013db3cdd..0000000000 --- a/.github/actions/benchmarkRunner/action.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: "dataCleaning Runner" -description: "dataCleaning Runner" -inputs: - version: - description: "datacleaning runner version" - required: false - default: 0.6.0-SNAPSHOT - if-stop: - description: "to stop the iginx" - required: false - default: "false" - if-test-udf: - description: "to test UDF path detection" - required: false - default: "false" -runs: - using: "composite" # Mandatory parameter - steps: - - if: inputs.if-test-udf=='true' - name: Test UDF Path - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" - - - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' - name: Start IGinX - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - echo "Linux" - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 - echo "Linux" - elif [ "$RUNNER_OS" == "Windows" ]; then - echo "Windows" - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 - echo "Windows" - elif [ "$RUNNER_OS" == "macOS" ]; then - echo "macOS" - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 - echo "macOS" - fi - - - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' - name: Stop IGinX - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index c9b95cfdba..b6785419dc 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -141,6 +141,15 @@ jobs: fi - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "IGinX" + + # start udf path test first to avoid being effected + + - name: Start IGinX shell: bash run: | cd IGinX @@ -155,14 +164,6 @@ jobs: ".github/scripts/iginx/iginx_macos.sh" 6888 6666 fi - # start udf path test first to avoid being effected - - - name: Start IGinX - uses: ./.github/actions/regressionTestRunner - with: - version: ${VERSION} - if-test-udf: "true" - - name: set client test context uses: ./IGinX/.github/actions/context with: From b5f652446ceb91262129553f879aa7acc7acae30 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 09:25:06 +0800 Subject: [PATCH 147/229] delete regressionTestAction --- .../workflows/polystore-benchmark-test.yml | 430 +++++++++--------- 1 file changed, 217 insertions(+), 213 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index b6785419dc..9bc4c078f1 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -1,224 +1,228 @@ name: PolyStore Benchmark Tests on: - push: - branches: [feat-benchmark] + push: + branches: [feat-benchmark] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - PolyStore-Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-13, windows-latest] - metadata: [zookeeper] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: generate tpch data - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Parquet" - - - name: Run PostgreSQL - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - - - name: Run MongoDB - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: "Parquet" - shell: client-before - - - name: Start storing data - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - - - name: Run session test - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - - - name: Get old version - if: always() - shell: bash - run: | - git clone https://github.com/IGinX-THU/IGinX.git - cd IGinX - mvn clean package -DskipTests -P-format - - - name: Stop ZooKeeper - uses: ./.github/actions/zookeeperRunner - with: - if-stop: "true" - - - name: Stop IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-stop: "true" - - - name: Rerun ZooKeeper - uses: ./.github/actions/zookeeperRunner - with: - if-rerun: "true" - - - name: Run DB - shell: bash - run: | - cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" - cd IGinX - if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" - ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/dataSources/parquet_macos.sh" - ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "IGinX" - - # start udf path test first to avoid being effected - - - name: Start IGinX - shell: bash - run: | - cd IGinX - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x ".github/scripts/iginx/iginx.sh" - ".github/scripts/iginx/iginx.sh" 6888 6666 - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x ".github/scripts/iginx/iginx_windows.sh" - ".github/scripts/iginx/iginx_windows.sh" 6888 6666 - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/iginx/iginx_macos.sh" - ".github/scripts/iginx/iginx_macos.sh" 6888 6666 - fi - - - name: set client test context - uses: ./IGinX/.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./IGinX/.github/actions/context - with: - name: Pre Test Client Export File - shell: bash - run: | - cd IGinX - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x ".github/scripts/test/cli/test_outfile.sh" - ".github/scripts/test/cli/test_outfile.sh" "Parquet - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" - ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" - ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" - fi - - - name: Start storing data - if: always() - shell: bash - run: | - pwd - ls - mkdir IGinX/tpc - mkdir "IGinX/tpc/TPC-H V3.0.1" - mkdir "IGinX/tpc/TPC-H V3.0.1/data" - cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - cd IGinX - output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" - chmod 777 "udtf_extract_year.py" - COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' - SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - - if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" - elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" - elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" - fi - - - name: Run session test - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + PolyStore-Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macOS-13, windows-latest] + metadata: [zookeeper] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: generate tpch data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "Parquet" + + - name: Run PostgreSQL + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + + - name: Run MongoDB + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: "Parquet" + shell: client-before + + - name: Start storing data + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + + - name: Run session test + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + + - name: Get old version + if: always() + shell: bash + run: | + git clone https://github.com/IGinX-THU/IGinX.git + cd IGinX + mvn clean package -DskipTests -P-format + + - name: Stop ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-stop: "true" + + - name: Stop IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + + - name: Rerun ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-rerun: "true" + + - name: Run DB + shell: bash + run: | + cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" + cd IGinX + if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then + chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" + ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/dataSources/parquet_macos.sh" + ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "IGinX" + + # start udf path test first to avoid being effected + + - name: Start IGinX + shell: bash + run: | + cd IGinX + pwd + if [ "$RUNNER_OS" == "Linux" ]; then + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "Windows" ]; then + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' core/target/iginx-core-${VERSION}/conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + chmod +x "../.github/scripts/iginx/iginx_regression_udf_path.sh" + "../.github/scripts/iginx/iginx_regression_udf_path.sh" + + - name: set client test context + uses: ./IGinX/.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./IGinX/.github/actions/context + with: + name: Pre Test Client Export File + shell: bash + run: | + cd IGinX + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x ".github/scripts/test/cli/test_outfile.sh" + ".github/scripts/test/cli/test_outfile.sh" "Parquet + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" + ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" + ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" + fi + + - name: Start storing data + if: always() + shell: bash + run: | + pwd + ls + mkdir IGinX/tpc + mkdir "IGinX/tpc/TPC-H V3.0.1" + mkdir "IGinX/tpc/TPC-H V3.0.1/data" + cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + cd IGinX + output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" + chmod 777 "udtf_extract_year.py" + COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' + SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + + if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" + elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" + elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" + fi + + - name: Run session test + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format # - name: Run session test From a3a6cef1461e33e91d680699bf76669a66034500 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 09:35:24 +0800 Subject: [PATCH 148/229] delete extra files --- .github/scripts/benchmarks/dataCleaning.sh | 17 -- .github/scripts/benchmarks/sgd.sh | 54 ------ .github/scripts/iginx/iginx_udf_path.sh | 2 +- .github/workflows/benchmark-test.yml | 96 ---------- .../polybench/DataCleaningRunner.py | 67 ------- .../iginx/integration/polybench/TPCHRunner.py | 167 ------------------ .../resources/polybench/udf/udaf_trainall.py | 81 --------- 7 files changed, 1 insertion(+), 483 deletions(-) delete mode 100755 .github/scripts/benchmarks/dataCleaning.sh delete mode 100755 .github/scripts/benchmarks/sgd.sh delete mode 100644 .github/workflows/benchmark-test.yml delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py delete mode 100644 test/src/test/resources/polybench/udf/udaf_trainall.py diff --git a/.github/scripts/benchmarks/dataCleaning.sh b/.github/scripts/benchmarks/dataCleaning.sh deleted file mode 100755 index 8a238dcace..0000000000 --- a/.github/scripts/benchmarks/dataCleaning.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -COMMAND1='LOAD DATA FROM INFILE "test/src/test/resources/dataCleaning/zipcode_city.csv" AS CSV INTO uszip(key,city,zipcode);SELECT count(a.zipcode) FROM uszip as a JOIN uszip as b ON a.zipcode = b.zipcode WHERE a.city <> b.city;' - -SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - -bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - -if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" -elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" -elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" -fi \ No newline at end of file diff --git a/.github/scripts/benchmarks/sgd.sh b/.github/scripts/benchmarks/sgd.sh deleted file mode 100755 index 0a13869eb8..0000000000 --- a/.github/scripts/benchmarks/sgd.sh +++ /dev/null @@ -1,54 +0,0 @@ -## 从网站下载数据 -#wget https://archive.ics.uci.edu/static/public/280/higgs.zip -#echo "数据下载完成" -#unzip higgs.zip -#rm higgs.zip -#gzip -d HIGGS.csv.gz -#head -n 100000 HIGGS.csv > HIGGS2.csv -#echo "数据解压完成" -#sed 's/\.000000000000000000e+00//g' HIGGS2.csv > HIGGS_new.csv -#mv HIGGS_new.csv HIGGS.csv -#echo "数据处理完成" - -if [ "$RUNNER_OS" = "Windows" ]; then - python thu_cloud_download.py \ - -l https://cloud.tsinghua.edu.cn/d/dce662ee7ced4a6398e3/ \ - -s "." -else - python3 thu_cloud_download.py \ - -l https://cloud.tsinghua.edu.cn/d/dce662ee7ced4a6398e3/ \ - -s "." -fi - -pip install scikit-learn pandas numpy -# 输出文件名 -output_file="HIGGS.csv" - -# 初始化序号 -index=0 -# 读取文件并在每一行前面加上序号 -while IFS= read -r line; do - echo "$index,$line" >> "$output_file" - index=$((index + 1)) -done < sgddata/HIGGS2.csv - -echo "处理完成，结果已保存到 $output_file" -rm sgddata/HIGGS2.csv -rmdir sgddata -set -e - -COMMAND1='LOAD DATA FROM INFILE "HIGGS.csv" AS CSV INTO trainall(key, label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb);' -COMMAND2='CREATE FUNCTION UDAF "trainall" FROM "UDAFtrainall" IN "test/src/test/resources/polybench/udf/udaf_trainall.py";' -COMMAND3='select trainall(key,label,lepton_pt,lepton_eta,lepton_phi,missing_energy_magnitude,missing_energy_phi,jet1_pt,jet1_eta,jet1_phi,jet1_b_tag,jet2_pt,jet2_eta,jet2_phi,jet2_b_tag,jet3_pt,jet3_eta,jet3_phi,jet3_b_tag,jet4_pt,jet4_eta,jet4_phi,jet4_b_tag,m_jj,m_jjj,m_lv,m_jlv,m_bb,m_wbb,m_wwbb) from trainall;' - -SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - -bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - -if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1$COMMAND2$COMMAND3' | xargs -0 -t -i ${SCRIPT_COMMAND}" -elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1$COMMAND2$COMMAND3'" -elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1$COMMAND2$COMMAND3' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" -fi diff --git a/.github/scripts/iginx/iginx_udf_path.sh b/.github/scripts/iginx/iginx_udf_path.sh index beea2a8466..6517cb1976 100644 --- a/.github/scripts/iginx/iginx_udf_path.sh +++ b/.github/scripts/iginx/iginx_udf_path.sh @@ -1,5 +1,5 @@ #!/bin/sh -pwd + set -e cd core/target/iginx-core-0.6.0-SNAPSHOT/ diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml deleted file mode 100644 index 0d9a03dd41..0000000000 --- a/.github/workflows/benchmark-test.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Benchmark Tests - -on: - push: - branches: [feat-benchmark] - -env: - VERSION: 0.6.0-SNAPSHOT -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-13, windows-latest] - DB-name: - [ - "IoTDB12", - "InfluxDB", - "Parquet", - "PostgreSQL", - "FileSystem", - "Redis", - "MongoDB", - "MySQL" - ] - metadata: [zookeeper] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: ${{ matrix.DB-name }} - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: ${{ matrix.DB-name }} - shell: client-before - - - name: Start data cleaning - shell: bash - run: | - pwd - if [ "$RUNNER_OS" == "Windows" ]; then - python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py - else - python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py - fi - - - name: Start sgd test - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/sgd.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/sgd.sh" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py deleted file mode 100644 index afb9678437..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/DataCleaningRunner.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -import subprocess, platform, os - -# 生成 zipcode -> city的映射关系，error rate 10% -# lineNum 为 -n 后面的参数 -import argparse -parser = argparse.ArgumentParser(description='generate zipcode -> city data for test') -parser.add_argument('-n', dest='lineNum', type=int, default=1000000, - help='line number of the generated data file, default 1000000') - -args = parser.parse_args() -print('Data generated! Line number: ', args.lineNum) -if args.lineNum < 0: - print('LineNum should be a positive number') - exit(1) -elif args.lineNum > 10000000: - print('LineNum too big, should be less than 10000000') - exit(1) - -correctNum = args.lineNum - args.lineNum // 10 - -zipcodes = [i for i in range(correctNum)] -cities = ['city' + str(i) for i in range(args.lineNum)] -# zipcodes加入10%的重复 -zipcodes = zipcodes + [zipcodes[i] for i in range(args.lineNum // 10)] -# mkdir if not exist -if not os.path.exists('test/src/test/resources/dataCleaning'): - os.makedirs('test/src/test/resources/dataCleaning') -with open('test/src/test/resources/dataCleaning/zipcode_city.csv', 'w') as f: - for i in range(args.lineNum): - f.write(str(i) + ',' + cities[i] + ',' + str(zipcodes[i]) + '\n') - -# 要运行的 shell 脚本文件路径 -script_path = ".github/scripts/benchmarks/dataCleaning.sh" - -# 使用 subprocess.run() 运行 shell 脚本 -# shell=True 表示通过 shell 解释器执行脚本 -# 如果脚本有输出，可以通过 stdout=subprocess.PIPE 捕获输出 -# 检测当前系统类型 -if platform.system() == "Windows": - # 使用 PowerShell 来运行脚本 - result = subprocess.run(["powershell", "-Command", f"& '{script_path}'"], stdout=subprocess.PIPE, text=False) - # 将字节形式的输出解码为字符串形式 - output = result.stdout.decode('utf-8') - print(output) -else: - # 使用 bash 来运行脚本 - result = subprocess.run(script_path, shell=True, stdout=subprocess.PIPE, text=True) - -# 检查脚本是否成功执行 -if result.returncode == 0: - print("Shell Script ran successfully!") - # 如果脚本有输出，可以打印输出内容 - if result.stdout: - resultMessage = result.stdout - print("output: ", resultMessage) - print("end of output") - # 正确性检验 - assert "Successfully write 1000000 record(s) to: [uszip.city, uszip.zipcode]" in resultMessage - assert '200000' in resultMessage - exit(0) - else: - print("no output") - exit(1) -else: - print("Failed to run script!") - exit(1) \ No newline at end of file diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py deleted file mode 100644 index d831e17fbf..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py +++ /dev/null @@ -1,167 +0,0 @@ -import sys, traceback - - -sys.path.append('session_py/') - -from iginx.iginx_pyclient.session import Session -from iginx.iginx_pyclient.thrift.rpc.ttypes import StorageEngineType, StorageEngine, DataType, AggregateType, DebugInfoType - -import time - -# 定义信号处理函数 -def timeout_handler(signum, frame): - print("Execution time exceeded 5 minutes. Exiting...") - sys.exit(0) - -if __name__ == '__main__': - print("start") - try: - session = Session('127.0.0.1', 6888, "root", "root") - session.open() - # 输出所有存储引擎 - cluster_info = session.get_cluster_info() - print(cluster_info) - # add storage engine - print("start adding storage engine") - start_time = time.time() - session.add_storage_engine( - "127.0.0.1", - 5432, - StorageEngineType.postgresql, - { - "has_data": "true", - "is_read_only": "true", - "username": "postgres", - "password": "postgres" - } - ) - session.add_storage_engine( - "127.0.0.1", - 27017, - StorageEngineType.mongodb, - { - "has_data": "true", - "is_read_only": "true", - "schema.sample.size": "1000", - "dummy.sample.size": "0" - } - ) - print(f"end adding storage engine, time cost: {time.time() - start_time}s") - # 输出所有存储引擎 - cluster_info = session.get_cluster_info() - print(cluster_info) - ###################### test ####################### - # 查询写入的数据，数据由PySessionIT测试写入 - dataset = session.query(["mongotpch.orders.*"], 0, 5) - # 转换为pandas.Dataframe - df = dataset.to_df() - print(df) - # 查询写入的数据，数据由PySessionIT测试写入 - dataset = session.query(["postgres.customer.*"], 0, 5) - # 转换为pandas.Dataframe - df = dataset.to_df() - print(df) - # 查询写入的数据，数据由PySessionIT测试写入 - dataset = session.query(["nation.*"], 0, 5) - # 转换为pandas.Dataframe - df = dataset.to_df() - print(df) - # 使用 list_time_series() 接口查询时间序列 - timeSeries = session.list_time_series() - for ts in timeSeries: - print(ts) - ################### end test ####################### - # 开始tpch查询 - print("start tpch query") - start_time = time.time() - - # sql = 'select * from mongotpch.orders limit 5;' - sql = '''select - nation.n_name, - revenue -from ( - select - nation.n_name, - sum(tmp) as revenue - from ( - select - nation.n_name, - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp - from - postgres.customer - join mongotpch.orders on postgres.customer.c_custkey = mongotpch.orders.o_custkey - join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey - join postgres.supplier on mongotpch.lineitem.l_suppkey = postgres.supplier.s_suppkey and postgres.customer.c_nationkey = postgres.supplier.s_nationkey - join nation on postgres.supplier.s_nationkey = nation.n_nationkey - join postgres.region on nation.n_regionkey = postgres.region.r_regionkey - where - postgres.region.r_name = "ASIA" - and mongotpch.orders.o_orderdate >= 757353600000 - and mongotpch.orders.o_orderdate < 788889600000 - ) - group by - nation.n_name -) -order by - revenue desc;''' # 这里757353600000和788889600000分别是1994-01-01和1995-01-01的Unix时间戳 - # 执行查询语句 - dataset = session.execute_statement(sql) - columns = dataset.columns() - for column in columns: - print(str(column) + ' ', end='') - print() - - result = [] - while dataset.has_more(): - row = dataset.next() - rowResult = [] - for field in row: - print(str(field) + ' ', end='') - rowResult.append(str(field)) - result.append(rowResult) - print() - print() - print(result) - dataset.close() - print(f"end tpch query, time cost: {time.time() - start_time}s") - - # 正确性验证 读取csv文件中的正确结果 - line_count = -1 # 用于跳过第一行 - correct = True - with open('test/src/test/resources/polybench/sf0.1/q05.csv', 'r') as f: - for line in f: - if line.strip() == '': - break - if line_count < 0: - line_count += 1 - continue - answer = line.strip().split('|') - print(f"result: {eval(result[line_count][0]).decode('utf-8')}, answer: {answer[0]}") - if eval(result[line_count][0]).decode('utf-8') != answer[0]: - correct = False - break - print(f"result: {eval(result[line_count][1])}, answer: {answer[1]}") - if abs(eval(result[line_count][1]) - eval(answer[1])) > 0.1: - correct = False - break - line_count += 1 - if not correct: - print("incorrect result") - exit(1) - - # 重复测试五次 - execute_time = [] - for i in range(5): - start_time = time.time() - dataset = session.execute_statement(sql) - execute_time.append(time.time() - start_time) - print(f"tpch query{i}, time cost: {execute_time[i]}s") - - # 输出平均执行时间 - print(f"average time cost: {sum(execute_time) / 5}s") - - session.close() - except Exception as e: - print(e) - traceback.print_exc() - diff --git a/test/src/test/resources/polybench/udf/udaf_trainall.py b/test/src/test/resources/polybench/udf/udaf_trainall.py deleted file mode 100644 index b4eb18125c..0000000000 --- a/test/src/test/resources/polybench/udf/udaf_trainall.py +++ /dev/null @@ -1,81 +0,0 @@ -import pandas as pd -import numpy as np -from sklearn.model_selection import train_test_split -from sklearn.linear_model import SGDClassifier -from sklearn.preprocessing import StandardScaler -from sklearn.metrics import accuracy_score -import time -from datetime import datetime - -MAX_LOSS = 100.0 # 最大损失，用于判断梯度是否正确 -EPOCHS = 1000 # 最大迭代次数 -COLUMN_NAME = "trainall(trainall.key, trainall.label, trainall.lepton_pt, trainall.lepton_eta, trainall.lepton_phi, trainall.missing_energy_magnitude, trainall.missing_energy_phi, trainall.jet1_pt, trainall.jet1_eta, trainall.jet1_phi, trainall.jet1_b_tag, trainall.jet2_pt, trainall.jet2_eta, trainall.jet2_phi, trainall.jet2_b_tag, trainall.jet3_pt, trainall.jet3_eta, trainall.jet3_phi, trainall.jet3_b_tag, trainall.jet4_pt, trainall.jet4_eta, trainall.jet4_phi, trainall.jet4_b_tag, trainall.m_jj, trainall.m_jjj, trainall.m_lv, trainall.m_jlv, trainall.m_bb, trainall.m_wbb, trainall.m_wwbb)" -class UDAFtrainall: - def __init__(self): - pass - - def transform(self, data, args, kvargs): - print('enter udf, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - res = self.buildHeader(data) - # convert data[2:]to df - df = pd.DataFrame(data[2:]) - # set df column index to data[0] - df.columns = data[0] - # print(df) - # train - print('before train, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - pred = sgd(df) - print('end train, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - res.extend(pred.to_numpy().tolist()) # data[2:] is the data - print('exit udf, time: ', datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - return res - - def buildHeader(self, data): - colNames = ["key", COLUMN_NAME] - return [colNames, ["LONG", "BINARY"]] # data[1] is the type - - -def sgd(df: pd.DataFrame): - start = time.time() - # 转换为pandas.Dataframe - #df['postgres.higgstrainall.label'] = df['postgres.higgstrainall.label'].apply(lambda x: x.decode('utf-8')) - # 这里需要将最后一列中的's'和'b'分别转为1、0 - #df['postgres.higgstrainall.label'] = df['postgres.higgstrainall.label'].map({'s': 1, 'b': 0}) - # 这里要去掉'label' - X = df.drop(['trainall.label'], axis=1) - print(X.head()) - y = df['trainall.label'] - # 划分训练集和测试集 - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=100) - - # 特征缩放 - scaler = StandardScaler() - X_train_scaled = scaler.fit_transform(X_train) - X_test_scaled = scaler.transform(X_test) - - # 创建SGDClassifier模型 - sgd_classifier = SGDClassifier(loss='hinge', alpha=0.005, max_iter=1000, random_state=100) - - print(X_train_scaled.shape) - print(y_train.shape) - # print(y_train[:100]) - # # check nan - # print(y_train.isnull().sum()) - # 训练模型 - sgd_classifier.fit(X_train_scaled, y_train) - - # 预测 - y_pred = sgd_classifier.predict(X_test_scaled) - - # 评估模型 - accuracy = accuracy_score(y_test, y_pred) - print(f"Accuracy: {accuracy}") - # print(sgd_classifier.coef_.tolist()[0]) - end = time.time() - print('Training time: ', end - start) - # 存储最终得到的theta - theta = (str(list(sgd_classifier.coef_.tolist()[0]))).encode('utf-8') - model = pd.DataFrame({'key': [1], COLUMN_NAME: [theta]}) - # print(model) - return model - From a76f6774856510444bd5f937cf9011b258ae54ed Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 09:43:21 +0800 Subject: [PATCH 149/229] format --- .github/workflows/case-regression.yml | 82 ++++ .github/workflows/codeql.yml | 79 ++++ .github/workflows/lint-pr.yml | 20 + .../workflows/polystore-benchmark-test.yml | 435 +++++++++--------- .../workflows/standalone-test-pushdown.yml | 104 +++++ .github/workflows/unit-mds.yml | 79 ++++ .github/workflows/unit-test.yml | 44 ++ .github/workflows/upload-binaries.yml | 74 +++ .../polybench/TPCHDataInsertionIT.java | 6 +- .../integration/polybench/TPCHRunner.java | 100 ++-- 10 files changed, 753 insertions(+), 270 deletions(-) create mode 100644 .github/workflows/case-regression.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/lint-pr.yml create mode 100644 .github/workflows/standalone-test-pushdown.yml create mode 100644 .github/workflows/unit-mds.yml create mode 100644 .github/workflows/unit-test.yml create mode 100644 .github/workflows/upload-binaries.yml diff --git a/.github/workflows/case-regression.yml b/.github/workflows/case-regression.yml new file mode 100644 index 0000000000..8015c3165e --- /dev/null +++ b/.github/workflows/case-regression.yml @@ -0,0 +1,82 @@ +name: "Bug Regression Test" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + +env: + VERSION: 0.6.0-SNAPSHOT + +jobs: + MixCluster-ShowTimeseries: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Run Metadata + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ matrix.metadata }} + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "Mix-IoTDB12-InfluxDB" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Write Conf + uses: ./.github/actions/confWriter + with: + Metadata: ${{ matrix.metadata }} + + - name: Start IGinX + uses: ./.github/actions/iginxRunner + + - name: TestMixCluster + if: always() + shell: bash + run: | + mvn test -q -Dtest=MixClusterShowColumnsRegressionTest -DfailIfNoTests=false -P-format + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000000..4861a044d6 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,79 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + pull_request: + types: [opened, reopened] + branches: ["main"] + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-13') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: ["java-kotlin", "python"] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml new file mode 100644 index 0000000000..6611cf46cd --- /dev/null +++ b/.github/workflows/lint-pr.yml @@ -0,0 +1,20 @@ +name: "Lint PR" + +on: + pull_request: + types: + - opened + - edited + - synchronize + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 9bc4c078f1..d01b368285 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -1,229 +1,228 @@ name: PolyStore Benchmark Tests on: - push: - branches: [feat-benchmark] + push: + branches: [feat-benchmark] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.6.0-SNAPSHOT concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - PolyStore-Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-13, windows-latest] - metadata: [zookeeper] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: generate tpch data - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Parquet" - - - name: Run PostgreSQL - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - - - name: Run MongoDB - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: "Parquet" - shell: client-before - - - name: Start storing data - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - - - name: Run session test - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - - - name: Get old version - if: always() - shell: bash - run: | - git clone https://github.com/IGinX-THU/IGinX.git - cd IGinX - mvn clean package -DskipTests -P-format - - - name: Stop ZooKeeper - uses: ./.github/actions/zookeeperRunner - with: - if-stop: "true" - - - name: Stop IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-stop: "true" - - - name: Rerun ZooKeeper - uses: ./.github/actions/zookeeperRunner - with: - if-rerun: "true" - - - name: Run DB - shell: bash - run: | - cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" - cd IGinX - if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" - ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/dataSources/parquet_macos.sh" - ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "IGinX" - - # start udf path test first to avoid being effected - - - name: Start IGinX - shell: bash - run: | - cd IGinX - pwd - if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' core/target/iginx-core-${VERSION}/conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - chmod +x "../.github/scripts/iginx/iginx_regression_udf_path.sh" - "../.github/scripts/iginx/iginx_regression_udf_path.sh" - - - name: set client test context - uses: ./IGinX/.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./IGinX/.github/actions/context - with: - name: Pre Test Client Export File - shell: bash - run: | - cd IGinX - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x ".github/scripts/test/cli/test_outfile.sh" - ".github/scripts/test/cli/test_outfile.sh" "Parquet - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" - ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" - ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" - fi - - - name: Start storing data - if: always() - shell: bash - run: | - pwd - ls - mkdir IGinX/tpc - mkdir "IGinX/tpc/TPC-H V3.0.1" - mkdir "IGinX/tpc/TPC-H V3.0.1/data" - cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - cd IGinX - output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" - chmod 777 "udtf_extract_year.py" - COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' - SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" - - if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" - elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" - elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" - fi - - - name: Run session test - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - + PolyStore-Benchmark-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: [8] + python-version: ["3.9"] + os: [ubuntu-latest, macOS-13, windows-latest] + metadata: [zookeeper] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: generate tpch data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: "Parquet" + + - name: Run PostgreSQL + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" + + - name: Run MongoDB + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + + # start udf path test first to avoid being effected + - name: Start IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" + + - name: set client test context + uses: ./.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./.github/actions/context + with: + DB-name: "Parquet" + shell: client-before + + - name: Start storing data + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" + + - name: Run session test + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format + + - name: Get old version + if: always() + shell: bash + run: | + git clone https://github.com/IGinX-THU/IGinX.git + cd IGinX + mvn clean package -DskipTests -P-format + + - name: Stop ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-stop: "true" + + - name: Stop IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + + - name: Rerun ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-rerun: "true" + + - name: Run DB + shell: bash + run: | + cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" + cd IGinX + if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then + chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" + ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/dataSources/parquet_macos.sh" + ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: "Parquet" + Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "IGinX" + + # start udf path test first to avoid being effected + + - name: Start IGinX + shell: bash + run: | + cd IGinX + pwd + if [ "$RUNNER_OS" == "Linux" ]; then + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "Windows" ]; then + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' core/target/iginx-core-${VERSION}/conf/config.properties + elif [ "$RUNNER_OS" == "macOS" ]; then + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' core/target/iginx-core-${VERSION}/conf/config.properties + else + echo "$RUNNER_OS is not supported" + exit 1 + fi + chmod +x "../.github/scripts/iginx/iginx_regression_udf_path.sh" + "../.github/scripts/iginx/iginx_regression_udf_path.sh" + + - name: set client test context + uses: ./IGinX/.github/actions/context + with: + work-name: restart-iginx-meta + metadata: ${{ matrix.metadata }} + + - name: set client test context + uses: ./IGinX/.github/actions/context + with: + name: Pre Test Client Export File + shell: bash + run: | + cd IGinX + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x ".github/scripts/test/cli/test_outfile.sh" + ".github/scripts/test/cli/test_outfile.sh" "Parquet + elif [ "$RUNNER_OS" == "Windows" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" + ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" + elif [ "$RUNNER_OS" == "macOS" ]; then + chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" + ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" + fi + + - name: Start storing data + if: always() + shell: bash + run: | + pwd + ls + mkdir IGinX/tpc + mkdir "IGinX/tpc/TPC-H V3.0.1" + mkdir "IGinX/tpc/TPC-H V3.0.1/data" + cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + cd IGinX + output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" + mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" + chmod 777 "udtf_extract_year.py" + COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' + SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + + if [ "$RUNNER_OS" = "Linux" ]; then + bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" + elif [ "$RUNNER_OS" = "Windows" ]; then + bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" + elif [ "$RUNNER_OS" = "macOS" ]; then + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" + fi + + - name: Run session test + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format # - name: Run session test # shell: bash diff --git a/.github/workflows/standalone-test-pushdown.yml b/.github/workflows/standalone-test-pushdown.yml new file mode 100644 index 0000000000..c4116604e3 --- /dev/null +++ b/.github/workflows/standalone-test-pushdown.yml @@ -0,0 +1,104 @@ +name: "Union Database Test With Push Down" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + +env: + VERSION: 0.6.0-SNAPSHOT + +jobs: + Union-DB-Test-Push_Down: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - if: runner.os == 'Windows' + name: Set JAVA_OPTS + run: echo "JAVA_OPTS=-Xmx4g -Xmx2g" >> $GITHUB_ENV + + - name: Run Metadata + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ matrix.metadata }} + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Push-Down: "true" + Set-Filter-Fragment-OFF: "true" + Metadata: ${{ matrix.metadata }} + + - name: Start IGinX + uses: ./.github/actions/iginxRunner + + - name: TestController IT + if: always() + shell: bash + env: + METADATA_STORAGE: ${{ matrix.metadata }} + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/test/test_union.sh" + mvn test -q -Dtest=Controller -DfailIfNoTests=false -P-format + + - name: Show test result + if: always() + shell: bash + run: | + cat ${GITHUB_WORKSPACE}/test/src/test/resources/testResult.txt + + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log diff --git a/.github/workflows/unit-mds.yml b/.github/workflows/unit-mds.yml new file mode 100644 index 0000000000..ad396bcac9 --- /dev/null +++ b/.github/workflows/unit-mds.yml @@ -0,0 +1,79 @@ +name: "Metadata-Service-Test" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + +jobs: + ZK-Test: + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + runs-on: ${{ matrix.os}} + env: + STORAGE: zookeeper + ZOOKEEPER_CONNECTION_STRING: 127.0.0.1:2181 + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + java: ${{ matrix.java }} + scope: base + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + - name: Run test for meta manager + run: mvn test -q -Dtest=IMetaManagerTest -DfailIfNoTests=false -P-format + + - name: Run test for sync protocol + run: mvn test -q -Dtest=ZooKeeperSyncProtocolTest -DfailIfNoTests=false -P-format + + - uses: codecov/codecov-action@v1 + with: + file: ./**/target/site/jacoco/jacoco.xml + name: codecov + + ETCD-Test: + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + runs-on: ${{ matrix.os}} + env: + STORAGE: etcd + ETCD_ENDPOINTS: http://localhost:2379 + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + java: ${{ matrix.java }} + scope: base + + - name: Run ETCD + uses: ./.github/actions/etcdRunner + + - name: Run test for meta manager + run: mvn test -q -Dtest=IMetaManagerTest -DfailIfNoTests=false -P-format + + - name: Run test for sync protocol + run: mvn test -q -Dtest=ETCDSyncProtocolTest -DfailIfNoTests=false -P-format + + - uses: codecov/codecov-action@v1 + with: + file: ./**/target/site/jacoco/jacoco.xml + name: codecov diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000000..a8a2270036 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,44 @@ +name: "UTTest" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + +env: + VERSION: 0.6.0-SNAPSHOT + +jobs: + Unit-Test: + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Test with Maven + run: | + mvn clean package -pl core -am -P-format -q diff --git a/.github/workflows/upload-binaries.yml b/.github/workflows/upload-binaries.yml new file mode 100644 index 0000000000..4658f40d8b --- /dev/null +++ b/.github/workflows/upload-binaries.yml @@ -0,0 +1,74 @@ +name: Upload Binaries to Github + +on: + release: + types: [published] + +jobs: + upload-package: + name: Upload to Github Release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: "8" + distribution: "temurin" + cache: "maven" + - name: package + run: | + mvn package \ + --batch-mode \ + -P release,!format \ + -DskipTests=true + - name: upload + uses: svenstaro/upload-release-action@v2 + with: + file: "**/*.tar.gz" + file_glob: true + overwrite: true + upload-deploy: + name: Upload to Github Pages + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: "8" + distribution: "temurin" + cache: "maven" + - name: checkout pages branch + uses: actions/checkout@v4 + with: + path: pages + ref: pages + fetch-depth: 0 + - name: deploy + run: | + mvn deploy \ + --batch-mode \ + -P !format \ + -DskipTests=true \ + -Ddeploy.repo.dir=$(pwd)/pages + - name: sync README.md and docs + run: | + cp README.md pages/ + cp -r docs pages/ + - name: create pull request + uses: peter-evans/create-pull-request@v6 + with: + path: pages + add-paths: | + maven-repo + maven-snapshot-repo + README.md + docs + branch: release-bot/pages + delete-branch: true + commit-message: "chore(repo): update maven repository" + title: "chore(repo): update maven repository" diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java index 1dbe90822d..04c062864e 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java @@ -255,13 +255,13 @@ public void insertDataIntoPostgreSQL() { conn.close(); } } catch (SQLException e) { - System.out.println("SQLException: " + e.getMessage()); - e.printStackTrace(); + System.out.println("SQLException: " + e.getMessage()); + e.printStackTrace(); } // 连接到新创建的数据库 String newDbUrl = String.format("jdbc:postgresql://localhost:5432/%s", databaseName); try (Connection conn = DriverManager.getConnection(newDbUrl, user, password); - Statement stmt = conn.createStatement()) { + Statement stmt = conn.createStatement()) { if (conn != null) { System.out.println(String.format("Connected to '%s' successfully.", databaseName)); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 4148ad0667..9362c37adf 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -177,55 +177,57 @@ public void test() { } } } -// List avgTimeCosts = new ArrayList<>(); -// for (int queryId : queryIds) { -// // read from sql file -// String sqlString = -// readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); -// -// // 开始 tpch 查询 -// System.out.println("start tpch query " + queryId); -// -// // 执行查询语句, split by ; 最后一句为执行结果 -// SessionExecuteSqlResult result = null; -// String[] sqls = sqlString.split(";"); -// List timeCosts = new ArrayList<>(); -// for (int i = 0; i < 2; i++) { // TODO: 5 -// startTime = System.currentTimeMillis(); -// if (sqls.length == 1) -// // TODO: error -// System.out.println("wrong input"); -// else result = conn.executeSql(sqls[sqls.length - 2] + ";"); -// Long timeCost = System.currentTimeMillis() - startTime; -// timeCosts.add(timeCost); -// System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); -// } -// Double averageTimeCost = -// timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); -// avgTimeCosts.add(averageTimeCost); -// System.out.println( -// "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); -// } -// // write avg time cost to file -// for (int i = 0; i < queryIds.size(); i++) { -// System.out.println( -// "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); -// } -// String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; -// if (Files.exists(Paths.get(fileName))) { -// List newAvgTimeCosts = readFromFile(fileName); -// for (int i = 0; i < queryIds.size(); i++) { -// System.out.println( -// "query " -// + queryIds.get(i) -// + ", new average time cost: " -// + newAvgTimeCosts.get(i) -// + "ms"); -// } -// // TODO 如果相差超过15%？，则报错 -// } else { -// writeToFile(avgTimeCosts, fileName); -// } + // List avgTimeCosts = new ArrayList<>(); + // for (int queryId : queryIds) { + // // read from sql file + // String sqlString = + // readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + + // ".sql"); + // + // // 开始 tpch 查询 + // System.out.println("start tpch query " + queryId); + // + // // 执行查询语句, split by ; 最后一句为执行结果 + // SessionExecuteSqlResult result = null; + // String[] sqls = sqlString.split(";"); + // List timeCosts = new ArrayList<>(); + // for (int i = 0; i < 2; i++) { // TODO: 5 + // startTime = System.currentTimeMillis(); + // if (sqls.length == 1) + // // TODO: error + // System.out.println("wrong input"); + // else result = conn.executeSql(sqls[sqls.length - 2] + ";"); + // Long timeCost = System.currentTimeMillis() - startTime; + // timeCosts.add(timeCost); + // System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); + // } + // Double averageTimeCost = + // timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); + // avgTimeCosts.add(averageTimeCost); + // System.out.println( + // "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); + // } + // // write avg time cost to file + // for (int i = 0; i < queryIds.size(); i++) { + // System.out.println( + // "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + + // "ms"); + // } + // String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; + // if (Files.exists(Paths.get(fileName))) { + // List newAvgTimeCosts = readFromFile(fileName); + // for (int i = 0; i < queryIds.size(); i++) { + // System.out.println( + // "query " + // + queryIds.get(i) + // + ", new average time cost: " + // + newAvgTimeCosts.get(i) + // + "ms"); + // } + // // TODO 如果相差超过15%？，则报错 + // } else { + // writeToFile(avgTimeCosts, fileName); + // } // 关闭会话 conn.closeSession(); From c96f8927062ad3f2bb8905ce3aaf543d9a9b44f8 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 09:47:35 +0800 Subject: [PATCH 150/229] debug --- .github/workflows/polystore-benchmark-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index d01b368285..009eb6feb6 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -165,8 +165,9 @@ jobs: echo "$RUNNER_OS is not supported" exit 1 fi - chmod +x "../.github/scripts/iginx/iginx_regression_udf_path.sh" - "../.github/scripts/iginx/iginx_regression_udf_path.sh" + cd .. + chmod +x ".github/scripts/iginx/iginx_regression_udf_path.sh" + ".github/scripts/iginx/iginx_regression_udf_path.sh" - name: set client test context uses: ./IGinX/.github/actions/context From f51bd1ea3a4b76d50631822acd4e7af4d3f630bd Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 10:30:45 +0800 Subject: [PATCH 151/229] debug --- .github/workflows/polystore-benchmark-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 009eb6feb6..fdce2a6a31 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -2,7 +2,7 @@ name: PolyStore Benchmark Tests on: push: - branches: [feat-benchmark] + branches: [feat-regression-test] env: VERSION: 0.6.0-SNAPSHOT From 679ec876ba9612fa5241de7d2f6e5eaa1f32c323 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 11:33:02 +0800 Subject: [PATCH 152/229] delete iginx_regression_udf_path.sh --- .../iginx/iginx_regression_udf_path.sh | 33 ------------------- .../workflows/polystore-benchmark-test.yml | 20 ++++++----- 2 files changed, 12 insertions(+), 41 deletions(-) delete mode 100644 .github/scripts/iginx/iginx_regression_udf_path.sh diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh deleted file mode 100644 index 879499fb0a..0000000000 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh - -ls -find . -name "udtf_extract_year.py" -rm ./core/target/iginx-core-0.6.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py - -cd IGinX -pwd -set -e - -cd core/target/iginx-core-0.6.0-SNAPSHOT/ - -iginx_home_path=$PWD - -echo "Iginx home path: $iginx_home_path" - -cd .. - -if [ -n "$MSYSTEM" ]; then - windows_path=$(cygpath -w "$iginx_home_path") - - export IGINX_HOME=$windows_path - - powershell -Command "Start-Process -FilePath 'iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.bat' -NoNewWindow -RedirectStandardOutput '../../iginx-udf.log' -RedirectStandardError '../../iginx-udf-error.log'" -else - export IGINX_HOME=$iginx_home_path - echo "Iginx home path: $IGINX_HOME" - pwd - - sh -c "chmod +x iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh" - - sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &" -fi diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index fdce2a6a31..c7529931ff 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -155,19 +155,23 @@ jobs: cd IGinX pwd if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' core/target/iginx-core-${VERSION}/conf/config.properties else - echo "$RUNNER_OS is not supported" - exit 1 + echo "$RUNNER_OS is not supported" + exit 1 fi cd .. - chmod +x ".github/scripts/iginx/iginx_regression_udf_path.sh" - ".github/scripts/iginx/iginx_regression_udf_path.sh" + find . -name "udtf_extract_year.py" + rm ./core/target/iginx-core-0.6.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py + cd IGinX + pwd + chmod +x ".github/scripts/iginx/iginx_udf_path.sh" + ".github/scripts/iginx/iginx_udf_path.sh" - name: set client test context uses: ./IGinX/.github/actions/context From 37e8176003a208838881acc06dfa47f3b9aea7b0 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 11:46:56 +0800 Subject: [PATCH 153/229] format --- .../workflows/polystore-benchmark-test.yml | 12 --- .../integration/polybench/TPCHRunner.java | 100 +++++++++--------- 2 files changed, 49 insertions(+), 63 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index c7529931ff..bd17e8b4d0 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -147,8 +147,6 @@ jobs: Set-Filter-Fragment-OFF: "true" Root-Dir-Path: "IGinX" - # start udf path test first to avoid being effected - - name: Start IGinX shell: bash run: | @@ -228,13 +226,3 @@ jobs: shell: bash run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - -# - name: Run session test -# shell: bash -# run: | -# pwd -# if [ "$RUNNER_OS" == "Windows" ]; then -# python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py -# else -# python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py -# fi diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 9362c37adf..0b8e052993 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -177,57 +177,55 @@ public void test() { } } } - // List avgTimeCosts = new ArrayList<>(); - // for (int queryId : queryIds) { - // // read from sql file - // String sqlString = - // readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + - // ".sql"); - // - // // 开始 tpch 查询 - // System.out.println("start tpch query " + queryId); - // - // // 执行查询语句, split by ; 最后一句为执行结果 - // SessionExecuteSqlResult result = null; - // String[] sqls = sqlString.split(";"); - // List timeCosts = new ArrayList<>(); - // for (int i = 0; i < 2; i++) { // TODO: 5 - // startTime = System.currentTimeMillis(); - // if (sqls.length == 1) - // // TODO: error - // System.out.println("wrong input"); - // else result = conn.executeSql(sqls[sqls.length - 2] + ";"); - // Long timeCost = System.currentTimeMillis() - startTime; - // timeCosts.add(timeCost); - // System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); - // } - // Double averageTimeCost = - // timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); - // avgTimeCosts.add(averageTimeCost); - // System.out.println( - // "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); - // } - // // write avg time cost to file - // for (int i = 0; i < queryIds.size(); i++) { - // System.out.println( - // "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + - // "ms"); - // } - // String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; - // if (Files.exists(Paths.get(fileName))) { - // List newAvgTimeCosts = readFromFile(fileName); - // for (int i = 0; i < queryIds.size(); i++) { - // System.out.println( - // "query " - // + queryIds.get(i) - // + ", new average time cost: " - // + newAvgTimeCosts.get(i) - // + "ms"); - // } - // // TODO 如果相差超过15%？，则报错 - // } else { - // writeToFile(avgTimeCosts, fileName); - // } + List avgTimeCosts = new ArrayList<>(); + for (int queryId : queryIds) { + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); + + // 开始 tpch 查询 + System.out.println("start tpch query " + queryId); + + // 执行查询语句, split by ; 最后一句为执行结果 + SessionExecuteSqlResult result = null; + String[] sqls = sqlString.split(";"); + List timeCosts = new ArrayList<>(); + for (int i = 0; i < 2; i++) { // TODO: 5 + startTime = System.currentTimeMillis(); + if (sqls.length == 1) + // TODO: error + System.out.println("wrong input"); + else result = conn.executeSql(sqls[sqls.length - 2] + ";"); + Long timeCost = System.currentTimeMillis() - startTime; + timeCosts.add(timeCost); + System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); + } + Double averageTimeCost = + timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); + avgTimeCosts.add(averageTimeCost); + System.out.println( + "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); + } + // write avg time cost to file + for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + } + String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; + if (Files.exists(Paths.get(fileName))) { + List newAvgTimeCosts = readFromFile(fileName); + for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + + queryIds.get(i) + + ", new average time cost: " + + newAvgTimeCosts.get(i) + + "ms"); + } + // TODO 如果相差超过15%？，则报错 + } else { + writeToFile(avgTimeCosts, fileName); + } // 关闭会话 conn.closeSession(); From 6412fda9d7bdaca816ec5ea5daa38383ef0bc64e Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 12:06:19 +0800 Subject: [PATCH 154/229] debug --- .github/scripts/iginx/iginx_regression_udf_path.sh | 4 ---- .github/workflows/polystore-benchmark-test.yml | 13 +------------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index 879499fb0a..dd5ebfb45e 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -1,9 +1,5 @@ #!/bin/sh -ls -find . -name "udtf_extract_year.py" -rm ./core/target/iginx-core-0.6.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py - cd IGinX pwd set -e diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index fdce2a6a31..d1f50275c1 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -147,8 +147,6 @@ jobs: Set-Filter-Fragment-OFF: "true" Root-Dir-Path: "IGinX" - # start udf path test first to avoid being effected - - name: Start IGinX shell: bash run: | @@ -166,6 +164,7 @@ jobs: exit 1 fi cd .. + rm ./core/target/iginx-core-0.6.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py chmod +x ".github/scripts/iginx/iginx_regression_udf_path.sh" ".github/scripts/iginx/iginx_regression_udf_path.sh" @@ -224,13 +223,3 @@ jobs: shell: bash run: | mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - -# - name: Run session test -# shell: bash -# run: | -# pwd -# if [ "$RUNNER_OS" == "Windows" ]; then -# python test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py -# else -# python3 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.py -# fi From 24b8c70d945a06bb517fde19eb764f5f44608cee Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 15:30:44 +0800 Subject: [PATCH 155/229] changed trigger --- .github/workflows/polystore-benchmark-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index d1f50275c1..303e271705 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -1,8 +1,8 @@ name: PolyStore Benchmark Tests on: - push: - branches: [feat-regression-test] + pull_request: # when a PR is opened or reopened + types: [opened, reopened] env: VERSION: 0.6.0-SNAPSHOT From 57d078b125688acd61329ac7bb2052fe71daa048 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Fri, 24 May 2024 16:16:10 +0800 Subject: [PATCH 156/229] add query 16 --- .../workflows/polystore-benchmark-test.yml | 4 +- .../integration/polybench/TPCHRunner.java | 102 +- .../test/resources/polybench/queries/q16.sql | 49 + .../test/resources/polybench/sf0.1/q16.csv | 2763 +++++++++++++++++ 4 files changed, 2864 insertions(+), 54 deletions(-) create mode 100644 test/src/test/resources/polybench/queries/q16.sql create mode 100644 test/src/test/resources/polybench/sf0.1/q16.csv diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 303e271705..fb9180e817 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -1,8 +1,8 @@ name: PolyStore Benchmark Tests on: - pull_request: # when a PR is opened or reopened - types: [opened, reopened] + pull_request: # when a PR is opened or reopened + types: [opened, reopened] env: VERSION: 0.6.0-SNAPSHOT diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 9362c37adf..4d1a3cf002 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -109,7 +109,7 @@ public void test() { System.out.println(clusterInfo); // Long startTime; // 13有问题 - List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 17, 18, 19, 20); + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = @@ -177,57 +177,55 @@ public void test() { } } } - // List avgTimeCosts = new ArrayList<>(); - // for (int queryId : queryIds) { - // // read from sql file - // String sqlString = - // readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + - // ".sql"); - // - // // 开始 tpch 查询 - // System.out.println("start tpch query " + queryId); - // - // // 执行查询语句, split by ; 最后一句为执行结果 - // SessionExecuteSqlResult result = null; - // String[] sqls = sqlString.split(";"); - // List timeCosts = new ArrayList<>(); - // for (int i = 0; i < 2; i++) { // TODO: 5 - // startTime = System.currentTimeMillis(); - // if (sqls.length == 1) - // // TODO: error - // System.out.println("wrong input"); - // else result = conn.executeSql(sqls[sqls.length - 2] + ";"); - // Long timeCost = System.currentTimeMillis() - startTime; - // timeCosts.add(timeCost); - // System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); - // } - // Double averageTimeCost = - // timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); - // avgTimeCosts.add(averageTimeCost); - // System.out.println( - // "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); - // } - // // write avg time cost to file - // for (int i = 0; i < queryIds.size(); i++) { - // System.out.println( - // "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + - // "ms"); - // } - // String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; - // if (Files.exists(Paths.get(fileName))) { - // List newAvgTimeCosts = readFromFile(fileName); - // for (int i = 0; i < queryIds.size(); i++) { - // System.out.println( - // "query " - // + queryIds.get(i) - // + ", new average time cost: " - // + newAvgTimeCosts.get(i) - // + "ms"); - // } - // // TODO 如果相差超过15%？，则报错 - // } else { - // writeToFile(avgTimeCosts, fileName); - // } + List avgTimeCosts = new ArrayList<>(); + for (int queryId : queryIds) { + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); + + // 开始 tpch 查询 + System.out.println("start tpch query " + queryId); + + // 执行查询语句, split by ; 最后一句为执行结果 + SessionExecuteSqlResult result = null; + String[] sqls = sqlString.split(";"); + List timeCosts = new ArrayList<>(); + for (int i = 0; i < 2; i++) { // TODO: 5 + startTime = System.currentTimeMillis(); + if (sqls.length == 1) + // TODO: error + System.out.println("wrong input"); + else result = conn.executeSql(sqls[sqls.length - 2] + ";"); + Long timeCost = System.currentTimeMillis() - startTime; + timeCosts.add(timeCost); + System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); + } + Double averageTimeCost = + timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); + avgTimeCosts.add(averageTimeCost); + System.out.println( + "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); + } + // write avg time cost to file + for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + } + String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; + if (Files.exists(Paths.get(fileName))) { + List newAvgTimeCosts = readFromFile(fileName); + for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + + queryIds.get(i) + + ", new average time cost: " + + newAvgTimeCosts.get(i) + + "ms"); + } + // TODO 如果相差超过15%？，则报错 + } else { + writeToFile(avgTimeCosts, fileName); + } // 关闭会话 conn.closeSession(); diff --git a/test/src/test/resources/polybench/queries/q16.sql b/test/src/test/resources/polybench/queries/q16.sql new file mode 100644 index 0000000000..e2878cb645 --- /dev/null +++ b/test/src/test/resources/polybench/queries/q16.sql @@ -0,0 +1,49 @@ +select + p_brand, + p_type, + p_size, + supplier_cnt +from ( + select + tpchdata.part.p_brand as p_brand, + tpchdata.part.p_type as p_type, + tpchdata.part.p_size as p_size, + count(distinct tpchdata.partsupp.ps_suppkey) as supplier_cnt + from + tpchdata.partsupp + join tpchdata.part on tpchdata.part.p_partkey = tpchdata.partsupp.ps_partkey + where + tpchdata.part.p_brand <> 'Brand#45' + and tpchdata.part.p_partkey not in( + select p_partkey from tpchdata.part + where tpchdata.part.p_type like '.*MEDIUM POLISHED.*' + ) + and ( + tpchdata.part.p_size = 3 + or tpchdata.part.p_size = 9 + or tpchdata.part.p_size = 14 + or tpchdata.part.p_size = 19 + or tpchdata.part.p_size = 23 + or tpchdata.part.p_size = 36 + or tpchdata.part.p_size = 45 + or tpchdata.part.p_size = 49 + ) + and tpchdata.partsupp.ps_suppkey not in ( + select + s_suppkey + from + tpchdata.supplier + where + tpchdata.supplier.s_comment like '.*Customer.*Complaints.*' + ) + group by + tpchdata.part.p_brand, + tpchdata.part.p_type, + tpchdata.part.p_size + order by + tpchdata.part.p_brand, + tpchdata.part.p_type, + tpchdata.part.p_size + ) +order by + supplier_cnt desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/sf0.1/q16.csv b/test/src/test/resources/polybench/sf0.1/q16.csv new file mode 100644 index 0000000000..23f2fa9b13 --- /dev/null +++ b/test/src/test/resources/polybench/sf0.1/q16.csv @@ -0,0 +1,2763 @@ +p_brand|p_type|p_size|supplier_cnt +Brand#14|SMALL ANODIZED NICKEL|45|12 +Brand#22|SMALL BURNISHED BRASS|19|12 +Brand#25|PROMO POLISHED COPPER|14|12 +Brand#35|LARGE ANODIZED STEEL|45|12 +Brand#35|PROMO BRUSHED COPPER|9|12 +Brand#51|ECONOMY ANODIZED STEEL|9|12 +Brand#53|LARGE BRUSHED NICKEL|45|12 +Brand#11|ECONOMY POLISHED COPPER|14|8 +Brand#11|LARGE PLATED STEEL|23|8 +Brand#11|PROMO POLISHED STEEL|23|8 +Brand#11|STANDARD ANODIZED COPPER|9|8 +Brand#12|ECONOMY BURNISHED BRASS|9|8 +Brand#12|LARGE ANODIZED BRASS|14|8 +Brand#12|SMALL ANODIZED TIN|23|8 +Brand#12|SMALL BRUSHED NICKEL|23|8 +Brand#12|STANDARD ANODIZED BRASS|3|8 +Brand#12|STANDARD BURNISHED TIN|23|8 +Brand#13|ECONOMY POLISHED BRASS|9|8 +Brand#13|LARGE BURNISHED COPPER|45|8 +Brand#13|MEDIUM ANODIZED STEEL|23|8 +Brand#13|MEDIUM PLATED NICKEL|3|8 +Brand#13|PROMO BURNISHED BRASS|9|8 +Brand#13|PROMO POLISHED BRASS|3|8 +Brand#13|PROMO POLISHED TIN|36|8 +Brand#13|SMALL BURNISHED STEEL|23|8 +Brand#13|STANDARD BRUSHED STEEL|9|8 +Brand#14|ECONOMY BRUSHED TIN|3|8 +Brand#14|ECONOMY BURNISHED TIN|23|8 +Brand#14|PROMO BRUSHED STEEL|9|8 +Brand#14|PROMO PLATED TIN|45|8 +Brand#15|ECONOMY PLATED TIN|9|8 +Brand#15|STANDARD BRUSHED COPPER|14|8 +Brand#15|STANDARD PLATED TIN|3|8 +Brand#21|ECONOMY POLISHED TIN|3|8 +Brand#21|PROMO POLISHED COPPER|9|8 +Brand#21|PROMO POLISHED TIN|49|8 +Brand#21|SMALL POLISHED STEEL|3|8 +Brand#21|STANDARD PLATED BRASS|49|8 +Brand#21|STANDARD PLATED NICKEL|49|8 +Brand#22|ECONOMY ANODIZED TIN|49|8 +Brand#22|ECONOMY BRUSHED BRASS|14|8 +Brand#22|LARGE BURNISHED TIN|36|8 +Brand#22|MEDIUM ANODIZED STEEL|36|8 +Brand#22|MEDIUM PLATED STEEL|9|8 +Brand#22|PROMO POLISHED NICKEL|9|8 +Brand#22|SMALL ANODIZED STEEL|19|8 +Brand#22|STANDARD ANODIZED COPPER|23|8 +Brand#23|ECONOMY BRUSHED NICKEL|23|8 +Brand#23|LARGE ANODIZED BRASS|9|8 +Brand#23|LARGE ANODIZED STEEL|23|8 +Brand#23|SMALL BRUSHED COPPER|23|8 +Brand#23|STANDARD BRUSHED TIN|3|8 +Brand#23|STANDARD BURNISHED NICKEL|49|8 +Brand#23|STANDARD PLATED NICKEL|36|8 +Brand#24|ECONOMY ANODIZED BRASS|19|8 +Brand#24|ECONOMY POLISHED BRASS|36|8 +Brand#24|LARGE BURNISHED STEEL|14|8 +Brand#24|MEDIUM PLATED NICKEL|36|8 +Brand#25|ECONOMY BRUSHED STEEL|49|8 +Brand#25|MEDIUM BURNISHED TIN|3|8 +Brand#25|PROMO ANODIZED TIN|36|8 +Brand#25|PROMO PLATED NICKEL|3|8 +Brand#25|SMALL BURNISHED BRASS|3|8 +Brand#31|LARGE ANODIZED BRASS|3|8 +Brand#31|SMALL ANODIZED COPPER|3|8 +Brand#31|SMALL ANODIZED NICKEL|9|8 +Brand#31|SMALL ANODIZED STEEL|14|8 +Brand#32|MEDIUM ANODIZED STEEL|49|8 +Brand#32|MEDIUM BURNISHED COPPER|19|8 +Brand#32|SMALL BURNISHED STEEL|23|8 +Brand#32|STANDARD BURNISHED STEEL|45|8 +Brand#34|ECONOMY ANODIZED NICKEL|49|8 +Brand#34|LARGE BURNISHED TIN|49|8 +Brand#34|MEDIUM BURNISHED NICKEL|3|8 +Brand#34|PROMO ANODIZED TIN|3|8 +Brand#34|SMALL BRUSHED TIN|3|8 +Brand#34|STANDARD BURNISHED TIN|23|8 +Brand#35|MEDIUM BRUSHED STEEL|45|8 +Brand#35|PROMO BURNISHED STEEL|14|8 +Brand#35|SMALL BURNISHED STEEL|23|8 +Brand#35|SMALL POLISHED COPPER|14|8 +Brand#35|STANDARD PLATED COPPER|9|8 +Brand#41|ECONOMY BRUSHED BRASS|23|8 +Brand#41|LARGE BURNISHED STEEL|23|8 +Brand#41|PROMO BURNISHED TIN|14|8 +Brand#41|PROMO PLATED STEEL|36|8 +Brand#41|PROMO POLISHED TIN|19|8 +Brand#41|SMALL BURNISHED COPPER|23|8 +Brand#42|LARGE POLISHED TIN|14|8 +Brand#42|MEDIUM ANODIZED TIN|49|8 +Brand#42|MEDIUM BRUSHED TIN|14|8 +Brand#42|MEDIUM BURNISHED NICKEL|23|8 +Brand#42|MEDIUM PLATED COPPER|45|8 +Brand#42|MEDIUM PLATED TIN|45|8 +Brand#42|SMALL PLATED COPPER|36|8 +Brand#43|ECONOMY BRUSHED STEEL|45|8 +Brand#43|LARGE BRUSHED COPPER|19|8 +Brand#43|PROMO BRUSHED BRASS|36|8 +Brand#43|SMALL BURNISHED TIN|45|8 +Brand#43|SMALL PLATED COPPER|45|8 +Brand#44|PROMO POLISHED TIN|23|8 +Brand#44|SMALL POLISHED NICKEL|14|8 +Brand#44|SMALL POLISHED TIN|45|8 +Brand#44|STANDARD BURNISHED COPPER|3|8 +Brand#51|LARGE ANODIZED BRASS|19|8 +Brand#51|LARGE POLISHED COPPER|23|8 +Brand#51|MEDIUM ANODIZED TIN|9|8 +Brand#51|MEDIUM ANODIZED TIN|14|8 +Brand#51|MEDIUM BURNISHED NICKEL|23|8 +Brand#51|SMALL ANODIZED COPPER|45|8 +Brand#51|SMALL ANODIZED COPPER|49|8 +Brand#51|SMALL BRUSHED COPPER|45|8 +Brand#51|SMALL BRUSHED TIN|36|8 +Brand#51|STANDARD POLISHED TIN|3|8 +Brand#52|ECONOMY ANODIZED STEEL|3|8 +Brand#52|ECONOMY PLATED TIN|19|8 +Brand#52|LARGE PLATED TIN|3|8 +Brand#52|MEDIUM ANODIZED TIN|19|8 +Brand#52|MEDIUM BURNISHED COPPER|3|8 +Brand#52|PROMO POLISHED BRASS|23|8 +Brand#52|SMALL PLATED COPPER|36|8 +Brand#52|SMALL POLISHED NICKEL|9|8 +Brand#52|STANDARD POLISHED NICKEL|45|8 +Brand#53|ECONOMY POLISHED STEEL|45|8 +Brand#53|LARGE POLISHED NICKEL|3|8 +Brand#53|SMALL BRUSHED COPPER|14|8 +Brand#53|STANDARD PLATED STEEL|45|8 +Brand#54|ECONOMY POLISHED BRASS|49|8 +Brand#54|ECONOMY POLISHED TIN|23|8 +Brand#54|LARGE ANODIZED NICKEL|49|8 +Brand#54|MEDIUM BRUSHED STEEL|9|8 +Brand#54|SMALL BURNISHED NICKEL|14|8 +Brand#54|SMALL PLATED TIN|14|8 +Brand#54|STANDARD BURNISHED STEEL|14|8 +Brand#54|STANDARD PLATED BRASS|23|8 +Brand#55|MEDIUM BURNISHED TIN|36|8 +Brand#55|PROMO ANODIZED BRASS|14|8 +Brand#55|STANDARD BURNISHED COPPER|45|8 +Brand#15|STANDARD PLATED TIN|36|7 +Brand#23|SMALL POLISHED BRASS|49|7 +Brand#42|STANDARD PLATED COPPER|19|7 +Brand#51|LARGE POLISHED NICKEL|14|7 +Brand#11|ECONOMY ANODIZED BRASS|19|4 +Brand#11|ECONOMY ANODIZED BRASS|45|4 +Brand#11|ECONOMY ANODIZED NICKEL|36|4 +Brand#11|ECONOMY BRUSHED COPPER|3|4 +Brand#11|ECONOMY BRUSHED COPPER|9|4 +Brand#11|ECONOMY BRUSHED STEEL|9|4 +Brand#11|ECONOMY BRUSHED STEEL|36|4 +Brand#11|ECONOMY BURNISHED BRASS|36|4 +Brand#11|ECONOMY BURNISHED COPPER|9|4 +Brand#11|ECONOMY BURNISHED COPPER|49|4 +Brand#11|ECONOMY BURNISHED NICKEL|14|4 +Brand#11|ECONOMY BURNISHED NICKEL|49|4 +Brand#11|ECONOMY PLATED COPPER|19|4 +Brand#11|ECONOMY PLATED NICKEL|45|4 +Brand#11|ECONOMY PLATED TIN|9|4 +Brand#11|ECONOMY POLISHED BRASS|3|4 +Brand#11|ECONOMY POLISHED COPPER|3|4 +Brand#11|ECONOMY POLISHED COPPER|45|4 +Brand#11|ECONOMY POLISHED NICKEL|36|4 +Brand#11|ECONOMY POLISHED STEEL|23|4 +Brand#11|ECONOMY POLISHED TIN|14|4 +Brand#11|LARGE ANODIZED COPPER|23|4 +Brand#11|LARGE ANODIZED NICKEL|9|4 +Brand#11|LARGE ANODIZED STEEL|9|4 +Brand#11|LARGE ANODIZED TIN|45|4 +Brand#11|LARGE BRUSHED STEEL|19|4 +Brand#11|LARGE BRUSHED TIN|3|4 +Brand#11|LARGE BRUSHED TIN|14|4 +Brand#11|LARGE BURNISHED COPPER|9|4 +Brand#11|LARGE BURNISHED COPPER|19|4 +Brand#11|LARGE BURNISHED STEEL|23|4 +Brand#11|LARGE BURNISHED TIN|9|4 +Brand#11|LARGE PLATED COPPER|23|4 +Brand#11|LARGE PLATED TIN|9|4 +Brand#11|LARGE PLATED TIN|14|4 +Brand#11|LARGE PLATED TIN|23|4 +Brand#11|LARGE POLISHED NICKEL|49|4 +Brand#11|MEDIUM ANODIZED BRASS|45|4 +Brand#11|MEDIUM ANODIZED TIN|14|4 +Brand#11|MEDIUM BRUSHED BRASS|14|4 +Brand#11|MEDIUM BRUSHED BRASS|45|4 +Brand#11|MEDIUM BRUSHED NICKEL|14|4 +Brand#11|MEDIUM BRUSHED NICKEL|36|4 +Brand#11|MEDIUM BRUSHED STEEL|19|4 +Brand#11|MEDIUM BURNISHED COPPER|9|4 +Brand#11|MEDIUM BURNISHED TIN|36|4 +Brand#11|MEDIUM PLATED BRASS|3|4 +Brand#11|MEDIUM PLATED TIN|19|4 +Brand#11|PROMO ANODIZED BRASS|3|4 +Brand#11|PROMO ANODIZED BRASS|19|4 +Brand#11|PROMO ANODIZED BRASS|45|4 +Brand#11|PROMO ANODIZED BRASS|49|4 +Brand#11|PROMO ANODIZED STEEL|23|4 +Brand#11|PROMO ANODIZED TIN|45|4 +Brand#11|PROMO BRUSHED BRASS|23|4 +Brand#11|PROMO BRUSHED STEEL|3|4 +Brand#11|PROMO BURNISHED BRASS|23|4 +Brand#11|PROMO BURNISHED BRASS|36|4 +Brand#11|PROMO BURNISHED BRASS|49|4 +Brand#11|PROMO BURNISHED TIN|9|4 +Brand#11|PROMO PLATED BRASS|9|4 +Brand#11|PROMO PLATED BRASS|45|4 +Brand#11|PROMO PLATED NICKEL|19|4 +Brand#11|PROMO POLISHED BRASS|3|4 +Brand#11|PROMO POLISHED BRASS|9|4 +Brand#11|PROMO POLISHED BRASS|19|4 +Brand#11|PROMO POLISHED COPPER|14|4 +Brand#11|PROMO POLISHED COPPER|45|4 +Brand#11|PROMO POLISHED TIN|49|4 +Brand#11|SMALL ANODIZED COPPER|36|4 +Brand#11|SMALL ANODIZED NICKEL|3|4 +Brand#11|SMALL ANODIZED NICKEL|14|4 +Brand#11|SMALL ANODIZED TIN|14|4 +Brand#11|SMALL ANODIZED TIN|19|4 +Brand#11|SMALL ANODIZED TIN|45|4 +Brand#11|SMALL BRUSHED TIN|14|4 +Brand#11|SMALL BRUSHED TIN|23|4 +Brand#11|SMALL BRUSHED TIN|45|4 +Brand#11|SMALL BURNISHED BRASS|49|4 +Brand#11|SMALL BURNISHED COPPER|23|4 +Brand#11|SMALL PLATED COPPER|45|4 +Brand#11|SMALL PLATED NICKEL|3|4 +Brand#11|SMALL PLATED STEEL|36|4 +Brand#11|SMALL PLATED TIN|19|4 +Brand#11|SMALL POLISHED BRASS|14|4 +Brand#11|SMALL POLISHED BRASS|23|4 +Brand#11|SMALL POLISHED COPPER|14|4 +Brand#11|SMALL POLISHED COPPER|36|4 +Brand#11|SMALL POLISHED STEEL|9|4 +Brand#11|STANDARD BRUSHED COPPER|23|4 +Brand#11|STANDARD BRUSHED NICKEL|14|4 +Brand#11|STANDARD BRUSHED TIN|14|4 +Brand#11|STANDARD BURNISHED BRASS|3|4 +Brand#11|STANDARD BURNISHED STEEL|23|4 +Brand#11|STANDARD PLATED BRASS|19|4 +Brand#11|STANDARD PLATED TIN|19|4 +Brand#11|STANDARD POLISHED NICKEL|45|4 +Brand#11|STANDARD POLISHED TIN|14|4 +Brand#11|STANDARD POLISHED TIN|45|4 +Brand#12|ECONOMY ANODIZED BRASS|23|4 +Brand#12|ECONOMY ANODIZED COPPER|14|4 +Brand#12|ECONOMY ANODIZED NICKEL|19|4 +Brand#12|ECONOMY ANODIZED NICKEL|45|4 +Brand#12|ECONOMY ANODIZED STEEL|9|4 +Brand#12|ECONOMY BRUSHED COPPER|36|4 +Brand#12|ECONOMY BRUSHED NICKEL|49|4 +Brand#12|ECONOMY BRUSHED STEEL|49|4 +Brand#12|ECONOMY BURNISHED COPPER|45|4 +Brand#12|ECONOMY PLATED COPPER|23|4 +Brand#12|ECONOMY PLATED STEEL|23|4 +Brand#12|ECONOMY PLATED TIN|36|4 +Brand#12|ECONOMY POLISHED BRASS|14|4 +Brand#12|ECONOMY POLISHED COPPER|45|4 +Brand#12|ECONOMY POLISHED NICKEL|9|4 +Brand#12|LARGE ANODIZED NICKEL|9|4 +Brand#12|LARGE ANODIZED NICKEL|49|4 +Brand#12|LARGE ANODIZED STEEL|49|4 +Brand#12|LARGE ANODIZED TIN|36|4 +Brand#12|LARGE ANODIZED TIN|45|4 +Brand#12|LARGE BURNISHED BRASS|14|4 +Brand#12|LARGE BURNISHED BRASS|19|4 +Brand#12|LARGE BURNISHED COPPER|9|4 +Brand#12|LARGE BURNISHED NICKEL|45|4 +Brand#12|LARGE BURNISHED TIN|36|4 +Brand#12|LARGE PLATED BRASS|3|4 +Brand#12|LARGE PLATED STEEL|36|4 +Brand#12|LARGE PLATED STEEL|45|4 +Brand#12|LARGE PLATED TIN|23|4 +Brand#12|LARGE POLISHED COPPER|14|4 +Brand#12|LARGE POLISHED COPPER|19|4 +Brand#12|LARGE POLISHED COPPER|49|4 +Brand#12|LARGE POLISHED STEEL|3|4 +Brand#12|MEDIUM ANODIZED COPPER|9|4 +Brand#12|MEDIUM ANODIZED COPPER|45|4 +Brand#12|MEDIUM ANODIZED NICKEL|45|4 +Brand#12|MEDIUM BRUSHED BRASS|19|4 +Brand#12|MEDIUM BRUSHED COPPER|9|4 +Brand#12|MEDIUM BRUSHED COPPER|36|4 +Brand#12|MEDIUM BRUSHED COPPER|49|4 +Brand#12|MEDIUM BRUSHED NICKEL|3|4 +Brand#12|MEDIUM BRUSHED NICKEL|14|4 +Brand#12|MEDIUM BRUSHED NICKEL|23|4 +Brand#12|MEDIUM BURNISHED BRASS|3|4 +Brand#12|MEDIUM BURNISHED COPPER|36|4 +Brand#12|MEDIUM BURNISHED NICKEL|19|4 +Brand#12|MEDIUM BURNISHED TIN|14|4 +Brand#12|MEDIUM PLATED BRASS|23|4 +Brand#12|MEDIUM PLATED TIN|19|4 +Brand#12|MEDIUM PLATED TIN|23|4 +Brand#12|MEDIUM PLATED TIN|49|4 +Brand#12|PROMO ANODIZED BRASS|9|4 +Brand#12|PROMO ANODIZED BRASS|45|4 +Brand#12|PROMO ANODIZED NICKEL|14|4 +Brand#12|PROMO ANODIZED STEEL|49|4 +Brand#12|PROMO ANODIZED TIN|3|4 +Brand#12|PROMO ANODIZED TIN|19|4 +Brand#12|PROMO BRUSHED COPPER|14|4 +Brand#12|PROMO BRUSHED COPPER|19|4 +Brand#12|PROMO BRUSHED NICKEL|23|4 +Brand#12|PROMO BRUSHED STEEL|23|4 +Brand#12|PROMO BRUSHED STEEL|36|4 +Brand#12|PROMO BURNISHED BRASS|49|4 +Brand#12|PROMO BURNISHED TIN|9|4 +Brand#12|PROMO BURNISHED TIN|14|4 +Brand#12|PROMO PLATED BRASS|36|4 +Brand#12|PROMO POLISHED COPPER|23|4 +Brand#12|PROMO POLISHED NICKEL|3|4 +Brand#12|PROMO POLISHED NICKEL|9|4 +Brand#12|PROMO POLISHED STEEL|14|4 +Brand#12|PROMO POLISHED TIN|23|4 +Brand#12|PROMO POLISHED TIN|36|4 +Brand#12|SMALL ANODIZED BRASS|36|4 +Brand#12|SMALL ANODIZED COPPER|23|4 +Brand#12|SMALL ANODIZED STEEL|36|4 +Brand#12|SMALL ANODIZED TIN|14|4 +Brand#12|SMALL BRUSHED COPPER|19|4 +Brand#12|SMALL BRUSHED COPPER|36|4 +Brand#12|SMALL BRUSHED TIN|36|4 +Brand#12|SMALL BURNISHED BRASS|14|4 +Brand#12|SMALL BURNISHED COPPER|9|4 +Brand#12|SMALL BURNISHED COPPER|36|4 +Brand#12|SMALL PLATED BRASS|9|4 +Brand#12|SMALL POLISHED BRASS|49|4 +Brand#12|SMALL POLISHED NICKEL|19|4 +Brand#12|SMALL POLISHED TIN|3|4 +Brand#12|STANDARD ANODIZED BRASS|19|4 +Brand#12|STANDARD ANODIZED NICKEL|19|4 +Brand#12|STANDARD ANODIZED STEEL|19|4 +Brand#12|STANDARD BRUSHED COPPER|36|4 +Brand#12|STANDARD BRUSHED NICKEL|23|4 +Brand#12|STANDARD BRUSHED STEEL|49|4 +Brand#12|STANDARD BURNISHED BRASS|23|4 +Brand#12|STANDARD BURNISHED COPPER|14|4 +Brand#12|STANDARD BURNISHED NICKEL|45|4 +Brand#12|STANDARD BURNISHED NICKEL|49|4 +Brand#12|STANDARD BURNISHED TIN|3|4 +Brand#12|STANDARD BURNISHED TIN|14|4 +Brand#12|STANDARD PLATED BRASS|19|4 +Brand#12|STANDARD PLATED NICKEL|45|4 +Brand#12|STANDARD PLATED STEEL|36|4 +Brand#12|STANDARD PLATED STEEL|45|4 +Brand#12|STANDARD PLATED TIN|9|4 +Brand#12|STANDARD POLISHED BRASS|49|4 +Brand#12|STANDARD POLISHED COPPER|3|4 +Brand#12|STANDARD POLISHED NICKEL|23|4 +Brand#12|STANDARD POLISHED TIN|14|4 +Brand#13|ECONOMY ANODIZED NICKEL|14|4 +Brand#13|ECONOMY ANODIZED NICKEL|19|4 +Brand#13|ECONOMY ANODIZED STEEL|45|4 +Brand#13|ECONOMY ANODIZED STEEL|49|4 +Brand#13|ECONOMY BRUSHED BRASS|3|4 +Brand#13|ECONOMY BURNISHED STEEL|14|4 +Brand#13|ECONOMY BURNISHED TIN|19|4 +Brand#13|ECONOMY BURNISHED TIN|45|4 +Brand#13|ECONOMY PLATED COPPER|19|4 +Brand#13|ECONOMY PLATED NICKEL|3|4 +Brand#13|ECONOMY PLATED STEEL|23|4 +Brand#13|ECONOMY PLATED TIN|3|4 +Brand#13|ECONOMY POLISHED BRASS|3|4 +Brand#13|ECONOMY POLISHED COPPER|9|4 +Brand#13|ECONOMY POLISHED COPPER|49|4 +Brand#13|ECONOMY POLISHED STEEL|23|4 +Brand#13|ECONOMY POLISHED STEEL|49|4 +Brand#13|LARGE ANODIZED BRASS|23|4 +Brand#13|LARGE ANODIZED COPPER|19|4 +Brand#13|LARGE ANODIZED NICKEL|9|4 +Brand#13|LARGE ANODIZED STEEL|45|4 +Brand#13|LARGE ANODIZED TIN|19|4 +Brand#13|LARGE BRUSHED BRASS|3|4 +Brand#13|LARGE BRUSHED BRASS|9|4 +Brand#13|LARGE BRUSHED BRASS|19|4 +Brand#13|LARGE BRUSHED COPPER|9|4 +Brand#13|LARGE BRUSHED COPPER|36|4 +Brand#13|LARGE BRUSHED NICKEL|3|4 +Brand#13|LARGE BRUSHED NICKEL|9|4 +Brand#13|LARGE BRUSHED NICKEL|14|4 +Brand#13|LARGE BRUSHED NICKEL|23|4 +Brand#13|LARGE BRUSHED STEEL|19|4 +Brand#13|LARGE BRUSHED TIN|49|4 +Brand#13|LARGE BURNISHED BRASS|49|4 +Brand#13|LARGE BURNISHED TIN|49|4 +Brand#13|LARGE PLATED COPPER|23|4 +Brand#13|LARGE PLATED STEEL|14|4 +Brand#13|LARGE PLATED STEEL|19|4 +Brand#13|LARGE PLATED STEEL|36|4 +Brand#13|LARGE PLATED TIN|14|4 +Brand#13|LARGE PLATED TIN|45|4 +Brand#13|LARGE POLISHED BRASS|3|4 +Brand#13|LARGE POLISHED BRASS|23|4 +Brand#13|LARGE POLISHED BRASS|49|4 +Brand#13|MEDIUM ANODIZED BRASS|3|4 +Brand#13|MEDIUM ANODIZED BRASS|36|4 +Brand#13|MEDIUM ANODIZED COPPER|14|4 +Brand#13|MEDIUM ANODIZED NICKEL|3|4 +Brand#13|MEDIUM ANODIZED STEEL|14|4 +Brand#13|MEDIUM ANODIZED STEEL|19|4 +Brand#13|MEDIUM ANODIZED STEEL|36|4 +Brand#13|MEDIUM BRUSHED BRASS|49|4 +Brand#13|MEDIUM BRUSHED COPPER|23|4 +Brand#13|MEDIUM BRUSHED NICKEL|45|4 +Brand#13|MEDIUM BURNISHED BRASS|9|4 +Brand#13|MEDIUM BURNISHED STEEL|19|4 +Brand#13|MEDIUM BURNISHED STEEL|49|4 +Brand#13|MEDIUM PLATED BRASS|3|4 +Brand#13|MEDIUM PLATED BRASS|23|4 +Brand#13|MEDIUM PLATED BRASS|36|4 +Brand#13|MEDIUM PLATED COPPER|19|4 +Brand#13|MEDIUM PLATED COPPER|23|4 +Brand#13|MEDIUM PLATED STEEL|3|4 +Brand#13|PROMO ANODIZED BRASS|14|4 +Brand#13|PROMO ANODIZED COPPER|9|4 +Brand#13|PROMO ANODIZED COPPER|45|4 +Brand#13|PROMO ANODIZED STEEL|23|4 +Brand#13|PROMO BRUSHED COPPER|49|4 +Brand#13|PROMO BURNISHED COPPER|19|4 +Brand#13|PROMO BURNISHED NICKEL|9|4 +Brand#13|PROMO BURNISHED STEEL|23|4 +Brand#13|PROMO BURNISHED STEEL|45|4 +Brand#13|PROMO BURNISHED TIN|19|4 +Brand#13|PROMO PLATED BRASS|14|4 +Brand#13|PROMO PLATED BRASS|19|4 +Brand#13|PROMO PLATED COPPER|3|4 +Brand#13|PROMO PLATED COPPER|19|4 +Brand#13|PROMO PLATED TIN|19|4 +Brand#13|PROMO POLISHED BRASS|49|4 +Brand#13|PROMO POLISHED STEEL|45|4 +Brand#13|PROMO POLISHED TIN|14|4 +Brand#13|SMALL ANODIZED STEEL|23|4 +Brand#13|SMALL ANODIZED TIN|3|4 +Brand#13|SMALL ANODIZED TIN|45|4 +Brand#13|SMALL BRUSHED COPPER|3|4 +Brand#13|SMALL BRUSHED NICKEL|19|4 +Brand#13|SMALL BRUSHED TIN|9|4 +Brand#13|SMALL BRUSHED TIN|45|4 +Brand#13|SMALL BURNISHED BRASS|19|4 +Brand#13|SMALL BURNISHED BRASS|45|4 +Brand#13|SMALL PLATED BRASS|9|4 +Brand#13|SMALL PLATED TIN|45|4 +Brand#13|SMALL POLISHED NICKEL|19|4 +Brand#13|SMALL POLISHED STEEL|49|4 +Brand#13|STANDARD ANODIZED COPPER|45|4 +Brand#13|STANDARD ANODIZED NICKEL|9|4 +Brand#13|STANDARD ANODIZED NICKEL|19|4 +Brand#13|STANDARD ANODIZED STEEL|14|4 +Brand#13|STANDARD ANODIZED TIN|9|4 +Brand#13|STANDARD ANODIZED TIN|36|4 +Brand#13|STANDARD BRUSHED BRASS|19|4 +Brand#13|STANDARD BRUSHED TIN|9|4 +Brand#13|STANDARD BURNISHED BRASS|9|4 +Brand#13|STANDARD BURNISHED BRASS|14|4 +Brand#13|STANDARD BURNISHED COPPER|45|4 +Brand#13|STANDARD PLATED BRASS|49|4 +Brand#13|STANDARD PLATED COPPER|19|4 +Brand#13|STANDARD PLATED NICKEL|23|4 +Brand#13|STANDARD PLATED TIN|9|4 +Brand#13|STANDARD POLISHED BRASS|49|4 +Brand#13|STANDARD POLISHED COPPER|9|4 +Brand#13|STANDARD POLISHED COPPER|49|4 +Brand#13|STANDARD POLISHED NICKEL|14|4 +Brand#13|STANDARD POLISHED NICKEL|19|4 +Brand#13|STANDARD POLISHED STEEL|23|4 +Brand#14|ECONOMY ANODIZED BRASS|19|4 +Brand#14|ECONOMY ANODIZED COPPER|9|4 +Brand#14|ECONOMY ANODIZED STEEL|19|4 +Brand#14|ECONOMY ANODIZED STEEL|45|4 +Brand#14|ECONOMY BRUSHED BRASS|19|4 +Brand#14|ECONOMY BRUSHED COPPER|45|4 +Brand#14|ECONOMY BRUSHED NICKEL|14|4 +Brand#14|ECONOMY BRUSHED TIN|14|4 +Brand#14|ECONOMY BURNISHED COPPER|9|4 +Brand#14|ECONOMY BURNISHED COPPER|19|4 +Brand#14|ECONOMY BURNISHED STEEL|36|4 +Brand#14|ECONOMY BURNISHED TIN|3|4 +Brand#14|ECONOMY PLATED BRASS|36|4 +Brand#14|ECONOMY PLATED COPPER|49|4 +Brand#14|ECONOMY PLATED STEEL|45|4 +Brand#14|ECONOMY PLATED TIN|9|4 +Brand#14|ECONOMY POLISHED COPPER|3|4 +Brand#14|ECONOMY POLISHED TIN|19|4 +Brand#14|LARGE ANODIZED COPPER|9|4 +Brand#14|LARGE ANODIZED COPPER|23|4 +Brand#14|LARGE ANODIZED NICKEL|3|4 +Brand#14|LARGE ANODIZED NICKEL|9|4 +Brand#14|LARGE ANODIZED NICKEL|19|4 +Brand#14|LARGE ANODIZED TIN|9|4 +Brand#14|LARGE BRUSHED COPPER|14|4 +Brand#14|LARGE BRUSHED NICKEL|45|4 +Brand#14|LARGE PLATED BRASS|3|4 +Brand#14|LARGE PLATED NICKEL|3|4 +Brand#14|LARGE PLATED NICKEL|14|4 +Brand#14|LARGE PLATED NICKEL|49|4 +Brand#14|LARGE PLATED TIN|49|4 +Brand#14|LARGE POLISHED BRASS|9|4 +Brand#14|LARGE POLISHED BRASS|14|4 +Brand#14|LARGE POLISHED BRASS|36|4 +Brand#14|LARGE POLISHED NICKEL|3|4 +Brand#14|LARGE POLISHED NICKEL|14|4 +Brand#14|LARGE POLISHED STEEL|9|4 +Brand#14|LARGE POLISHED STEEL|23|4 +Brand#14|LARGE POLISHED STEEL|36|4 +Brand#14|MEDIUM ANODIZED NICKEL|3|4 +Brand#14|MEDIUM ANODIZED NICKEL|49|4 +Brand#14|MEDIUM ANODIZED STEEL|23|4 +Brand#14|MEDIUM ANODIZED STEEL|36|4 +Brand#14|MEDIUM BRUSHED BRASS|9|4 +Brand#14|MEDIUM BRUSHED COPPER|23|4 +Brand#14|MEDIUM BRUSHED STEEL|14|4 +Brand#14|MEDIUM BURNISHED COPPER|14|4 +Brand#14|MEDIUM BURNISHED STEEL|3|4 +Brand#14|MEDIUM BURNISHED STEEL|49|4 +Brand#14|MEDIUM PLATED BRASS|36|4 +Brand#14|MEDIUM PLATED STEEL|49|4 +Brand#14|MEDIUM PLATED TIN|14|4 +Brand#14|PROMO ANODIZED BRASS|49|4 +Brand#14|PROMO ANODIZED STEEL|36|4 +Brand#14|PROMO BRUSHED STEEL|19|4 +Brand#14|PROMO BURNISHED BRASS|23|4 +Brand#14|PROMO BURNISHED STEEL|36|4 +Brand#14|PROMO PLATED BRASS|9|4 +Brand#14|PROMO PLATED BRASS|45|4 +Brand#14|PROMO PLATED COPPER|45|4 +Brand#14|PROMO PLATED STEEL|3|4 +Brand#14|PROMO POLISHED BRASS|9|4 +Brand#14|PROMO POLISHED COPPER|49|4 +Brand#14|PROMO POLISHED STEEL|19|4 +Brand#14|SMALL ANODIZED STEEL|23|4 +Brand#14|SMALL ANODIZED TIN|23|4 +Brand#14|SMALL BRUSHED BRASS|19|4 +Brand#14|SMALL BRUSHED BRASS|36|4 +Brand#14|SMALL BRUSHED COPPER|9|4 +Brand#14|SMALL BRUSHED TIN|36|4 +Brand#14|SMALL BURNISHED BRASS|45|4 +Brand#14|SMALL BURNISHED COPPER|9|4 +Brand#14|SMALL BURNISHED COPPER|14|4 +Brand#14|SMALL BURNISHED COPPER|45|4 +Brand#14|SMALL BURNISHED NICKEL|36|4 +Brand#14|SMALL BURNISHED STEEL|36|4 +Brand#14|SMALL BURNISHED TIN|23|4 +Brand#14|SMALL PLATED NICKEL|3|4 +Brand#14|SMALL PLATED NICKEL|9|4 +Brand#14|SMALL PLATED STEEL|14|4 +Brand#14|SMALL POLISHED BRASS|36|4 +Brand#14|SMALL POLISHED COPPER|36|4 +Brand#14|SMALL POLISHED NICKEL|9|4 +Brand#14|SMALL POLISHED STEEL|14|4 +Brand#14|SMALL POLISHED TIN|14|4 +Brand#14|STANDARD ANODIZED BRASS|19|4 +Brand#14|STANDARD ANODIZED NICKEL|14|4 +Brand#14|STANDARD ANODIZED STEEL|9|4 +Brand#14|STANDARD BRUSHED COPPER|45|4 +Brand#14|STANDARD BRUSHED NICKEL|45|4 +Brand#14|STANDARD BRUSHED TIN|45|4 +Brand#14|STANDARD BURNISHED BRASS|9|4 +Brand#14|STANDARD BURNISHED BRASS|23|4 +Brand#14|STANDARD BURNISHED BRASS|49|4 +Brand#14|STANDARD BURNISHED NICKEL|9|4 +Brand#14|STANDARD PLATED BRASS|36|4 +Brand#14|STANDARD PLATED COPPER|45|4 +Brand#14|STANDARD POLISHED NICKEL|3|4 +Brand#14|STANDARD POLISHED NICKEL|9|4 +Brand#14|STANDARD POLISHED TIN|19|4 +Brand#15|ECONOMY ANODIZED COPPER|14|4 +Brand#15|ECONOMY ANODIZED STEEL|19|4 +Brand#15|ECONOMY ANODIZED STEEL|36|4 +Brand#15|ECONOMY BRUSHED BRASS|36|4 +Brand#15|ECONOMY BRUSHED COPPER|14|4 +Brand#15|ECONOMY BRUSHED NICKEL|14|4 +Brand#15|ECONOMY BRUSHED STEEL|3|4 +Brand#15|ECONOMY BRUSHED TIN|3|4 +Brand#15|ECONOMY BURNISHED BRASS|14|4 +Brand#15|ECONOMY BURNISHED COPPER|3|4 +Brand#15|ECONOMY BURNISHED COPPER|23|4 +Brand#15|ECONOMY PLATED NICKEL|49|4 +Brand#15|ECONOMY PLATED STEEL|3|4 +Brand#15|ECONOMY PLATED STEEL|19|4 +Brand#15|ECONOMY PLATED STEEL|45|4 +Brand#15|LARGE ANODIZED BRASS|19|4 +Brand#15|LARGE ANODIZED BRASS|36|4 +Brand#15|LARGE ANODIZED BRASS|45|4 +Brand#15|LARGE ANODIZED COPPER|3|4 +Brand#15|LARGE ANODIZED NICKEL|9|4 +Brand#15|LARGE ANODIZED TIN|19|4 +Brand#15|LARGE BRUSHED BRASS|9|4 +Brand#15|LARGE BRUSHED BRASS|19|4 +Brand#15|LARGE BRUSHED COPPER|14|4 +Brand#15|LARGE BRUSHED STEEL|9|4 +Brand#15|LARGE BRUSHED STEEL|14|4 +Brand#15|LARGE BRUSHED STEEL|19|4 +Brand#15|LARGE BRUSHED STEEL|36|4 +Brand#15|LARGE BURNISHED BRASS|14|4 +Brand#15|LARGE BURNISHED BRASS|19|4 +Brand#15|LARGE BURNISHED COPPER|9|4 +Brand#15|LARGE BURNISHED COPPER|45|4 +Brand#15|LARGE BURNISHED TIN|49|4 +Brand#15|LARGE PLATED BRASS|19|4 +Brand#15|LARGE PLATED COPPER|3|4 +Brand#15|LARGE PLATED COPPER|23|4 +Brand#15|LARGE PLATED NICKEL|36|4 +Brand#15|MEDIUM ANODIZED BRASS|23|4 +Brand#15|MEDIUM ANODIZED COPPER|9|4 +Brand#15|MEDIUM ANODIZED NICKEL|3|4 +Brand#15|MEDIUM ANODIZED TIN|19|4 +Brand#15|MEDIUM BRUSHED BRASS|9|4 +Brand#15|MEDIUM BRUSHED TIN|23|4 +Brand#15|MEDIUM BURNISHED COPPER|36|4 +Brand#15|MEDIUM BURNISHED TIN|45|4 +Brand#15|MEDIUM PLATED COPPER|9|4 +Brand#15|MEDIUM PLATED NICKEL|9|4 +Brand#15|MEDIUM PLATED NICKEL|19|4 +Brand#15|MEDIUM PLATED STEEL|36|4 +Brand#15|MEDIUM PLATED STEEL|49|4 +Brand#15|MEDIUM PLATED TIN|9|4 +Brand#15|MEDIUM PLATED TIN|14|4 +Brand#15|MEDIUM PLATED TIN|23|4 +Brand#15|PROMO ANODIZED COPPER|23|4 +Brand#15|PROMO ANODIZED STEEL|14|4 +Brand#15|PROMO ANODIZED TIN|45|4 +Brand#15|PROMO BRUSHED COPPER|14|4 +Brand#15|PROMO BRUSHED COPPER|19|4 +Brand#15|PROMO BRUSHED NICKEL|19|4 +Brand#15|PROMO BRUSHED NICKEL|23|4 +Brand#15|PROMO BRUSHED STEEL|14|4 +Brand#15|PROMO BRUSHED TIN|36|4 +Brand#15|PROMO BURNISHED NICKEL|9|4 +Brand#15|PROMO BURNISHED STEEL|45|4 +Brand#15|PROMO PLATED COPPER|3|4 +Brand#15|PROMO PLATED COPPER|36|4 +Brand#15|PROMO PLATED STEEL|3|4 +Brand#15|PROMO PLATED TIN|49|4 +Brand#15|PROMO POLISHED COPPER|3|4 +Brand#15|PROMO POLISHED NICKEL|36|4 +Brand#15|PROMO POLISHED STEEL|36|4 +Brand#15|PROMO POLISHED TIN|49|4 +Brand#15|SMALL ANODIZED BRASS|14|4 +Brand#15|SMALL ANODIZED BRASS|19|4 +Brand#15|SMALL ANODIZED COPPER|9|4 +Brand#15|SMALL ANODIZED TIN|45|4 +Brand#15|SMALL BRUSHED BRASS|3|4 +Brand#15|SMALL BRUSHED COPPER|19|4 +Brand#15|SMALL BRUSHED STEEL|23|4 +Brand#15|SMALL BRUSHED TIN|45|4 +Brand#15|SMALL BURNISHED BRASS|19|4 +Brand#15|SMALL BURNISHED COPPER|14|4 +Brand#15|SMALL BURNISHED NICKEL|19|4 +Brand#15|SMALL BURNISHED NICKEL|49|4 +Brand#15|SMALL BURNISHED STEEL|9|4 +Brand#15|SMALL BURNISHED TIN|19|4 +Brand#15|SMALL BURNISHED TIN|23|4 +Brand#15|SMALL BURNISHED TIN|36|4 +Brand#15|SMALL PLATED BRASS|3|4 +Brand#15|SMALL PLATED COPPER|23|4 +Brand#15|SMALL PLATED COPPER|49|4 +Brand#15|SMALL PLATED NICKEL|36|4 +Brand#15|SMALL PLATED NICKEL|45|4 +Brand#15|SMALL PLATED STEEL|3|4 +Brand#15|SMALL PLATED TIN|9|4 +Brand#15|SMALL POLISHED COPPER|9|4 +Brand#15|SMALL POLISHED NICKEL|3|4 +Brand#15|SMALL POLISHED STEEL|19|4 +Brand#15|SMALL POLISHED STEEL|36|4 +Brand#15|SMALL POLISHED TIN|19|4 +Brand#15|SMALL POLISHED TIN|49|4 +Brand#15|STANDARD ANODIZED NICKEL|19|4 +Brand#15|STANDARD ANODIZED NICKEL|49|4 +Brand#15|STANDARD ANODIZED TIN|36|4 +Brand#15|STANDARD BRUSHED NICKEL|3|4 +Brand#15|STANDARD BURNISHED BRASS|23|4 +Brand#15|STANDARD BURNISHED STEEL|3|4 +Brand#15|STANDARD BURNISHED STEEL|45|4 +Brand#15|STANDARD PLATED BRASS|36|4 +Brand#15|STANDARD PLATED COPPER|14|4 +Brand#15|STANDARD PLATED COPPER|23|4 +Brand#15|STANDARD PLATED NICKEL|19|4 +Brand#15|STANDARD PLATED TIN|45|4 +Brand#15|STANDARD POLISHED BRASS|14|4 +Brand#15|STANDARD POLISHED COPPER|23|4 +Brand#15|STANDARD POLISHED NICKEL|45|4 +Brand#21|ECONOMY ANODIZED BRASS|3|4 +Brand#21|ECONOMY ANODIZED NICKEL|14|4 +Brand#21|ECONOMY ANODIZED STEEL|19|4 +Brand#21|ECONOMY ANODIZED STEEL|23|4 +Brand#21|ECONOMY ANODIZED STEEL|49|4 +Brand#21|ECONOMY ANODIZED TIN|19|4 +Brand#21|ECONOMY BRUSHED BRASS|9|4 +Brand#21|ECONOMY BRUSHED BRASS|14|4 +Brand#21|ECONOMY BRUSHED BRASS|36|4 +Brand#21|ECONOMY BRUSHED COPPER|49|4 +Brand#21|ECONOMY BRUSHED STEEL|45|4 +Brand#21|ECONOMY BRUSHED TIN|49|4 +Brand#21|ECONOMY BURNISHED BRASS|3|4 +Brand#21|ECONOMY BURNISHED COPPER|45|4 +Brand#21|ECONOMY BURNISHED STEEL|19|4 +Brand#21|ECONOMY BURNISHED STEEL|36|4 +Brand#21|ECONOMY PLATED BRASS|36|4 +Brand#21|ECONOMY PLATED COPPER|3|4 +Brand#21|ECONOMY PLATED COPPER|14|4 +Brand#21|ECONOMY PLATED NICKEL|49|4 +Brand#21|ECONOMY POLISHED NICKEL|3|4 +Brand#21|ECONOMY POLISHED NICKEL|9|4 +Brand#21|LARGE ANODIZED COPPER|3|4 +Brand#21|LARGE ANODIZED COPPER|9|4 +Brand#21|LARGE ANODIZED STEEL|36|4 +Brand#21|LARGE ANODIZED TIN|45|4 +Brand#21|LARGE BRUSHED COPPER|45|4 +Brand#21|LARGE BRUSHED STEEL|23|4 +Brand#21|LARGE BURNISHED BRASS|49|4 +Brand#21|LARGE BURNISHED COPPER|19|4 +Brand#21|LARGE BURNISHED STEEL|49|4 +Brand#21|LARGE BURNISHED TIN|49|4 +Brand#21|LARGE PLATED BRASS|19|4 +Brand#21|LARGE PLATED NICKEL|23|4 +Brand#21|LARGE PLATED NICKEL|49|4 +Brand#21|LARGE PLATED TIN|19|4 +Brand#21|LARGE POLISHED BRASS|49|4 +Brand#21|LARGE POLISHED COPPER|14|4 +Brand#21|LARGE POLISHED NICKEL|3|4 +Brand#21|LARGE POLISHED NICKEL|14|4 +Brand#21|LARGE POLISHED STEEL|14|4 +Brand#21|LARGE POLISHED TIN|49|4 +Brand#21|MEDIUM ANODIZED COPPER|14|4 +Brand#21|MEDIUM ANODIZED NICKEL|49|4 +Brand#21|MEDIUM BRUSHED COPPER|3|4 +Brand#21|MEDIUM BRUSHED COPPER|49|4 +Brand#21|MEDIUM BRUSHED STEEL|23|4 +Brand#21|MEDIUM BRUSHED TIN|3|4 +Brand#21|MEDIUM BRUSHED TIN|14|4 +Brand#21|MEDIUM BURNISHED NICKEL|14|4 +Brand#21|MEDIUM BURNISHED STEEL|23|4 +Brand#21|MEDIUM BURNISHED TIN|3|4 +Brand#21|MEDIUM PLATED BRASS|3|4 +Brand#21|MEDIUM PLATED BRASS|19|4 +Brand#21|MEDIUM PLATED STEEL|36|4 +Brand#21|PROMO ANODIZED BRASS|9|4 +Brand#21|PROMO ANODIZED COPPER|14|4 +Brand#21|PROMO ANODIZED NICKEL|23|4 +Brand#21|PROMO ANODIZED STEEL|3|4 +Brand#21|PROMO ANODIZED STEEL|14|4 +Brand#21|PROMO ANODIZED STEEL|36|4 +Brand#21|PROMO BRUSHED NICKEL|45|4 +Brand#21|PROMO BRUSHED STEEL|14|4 +Brand#21|PROMO BRUSHED STEEL|23|4 +Brand#21|PROMO BRUSHED STEEL|45|4 +Brand#21|PROMO BURNISHED BRASS|19|4 +Brand#21|PROMO BURNISHED COPPER|19|4 +Brand#21|PROMO BURNISHED NICKEL|9|4 +Brand#21|PROMO BURNISHED TIN|19|4 +Brand#21|PROMO PLATED NICKEL|9|4 +Brand#21|PROMO PLATED NICKEL|36|4 +Brand#21|PROMO PLATED STEEL|49|4 +Brand#21|PROMO PLATED TIN|3|4 +Brand#21|PROMO POLISHED NICKEL|23|4 +Brand#21|PROMO POLISHED TIN|14|4 +Brand#21|PROMO POLISHED TIN|19|4 +Brand#21|PROMO POLISHED TIN|23|4 +Brand#21|SMALL BRUSHED BRASS|23|4 +Brand#21|SMALL BRUSHED COPPER|49|4 +Brand#21|SMALL BURNISHED BRASS|23|4 +Brand#21|SMALL BURNISHED BRASS|36|4 +Brand#21|SMALL BURNISHED STEEL|19|4 +Brand#21|SMALL BURNISHED TIN|19|4 +Brand#21|SMALL PLATED BRASS|45|4 +Brand#21|SMALL PLATED COPPER|45|4 +Brand#21|SMALL PLATED STEEL|45|4 +Brand#21|SMALL PLATED TIN|14|4 +Brand#21|SMALL PLATED TIN|45|4 +Brand#21|SMALL POLISHED COPPER|9|4 +Brand#21|SMALL POLISHED NICKEL|23|4 +Brand#21|SMALL POLISHED TIN|3|4 +Brand#21|STANDARD ANODIZED BRASS|9|4 +Brand#21|STANDARD ANODIZED NICKEL|19|4 +Brand#21|STANDARD ANODIZED TIN|45|4 +Brand#21|STANDARD BURNISHED COPPER|36|4 +Brand#21|STANDARD BURNISHED NICKEL|23|4 +Brand#21|STANDARD BURNISHED TIN|9|4 +Brand#21|STANDARD PLATED BRASS|14|4 +Brand#21|STANDARD PLATED COPPER|19|4 +Brand#21|STANDARD PLATED NICKEL|3|4 +Brand#21|STANDARD PLATED STEEL|9|4 +Brand#21|STANDARD PLATED TIN|9|4 +Brand#21|STANDARD POLISHED BRASS|9|4 +Brand#21|STANDARD POLISHED COPPER|49|4 +Brand#21|STANDARD POLISHED STEEL|36|4 +Brand#21|STANDARD POLISHED TIN|36|4 +Brand#22|ECONOMY ANODIZED STEEL|9|4 +Brand#22|ECONOMY ANODIZED STEEL|14|4 +Brand#22|ECONOMY ANODIZED STEEL|23|4 +Brand#22|ECONOMY ANODIZED TIN|9|4 +Brand#22|ECONOMY ANODIZED TIN|36|4 +Brand#22|ECONOMY BRUSHED NICKEL|36|4 +Brand#22|ECONOMY BRUSHED NICKEL|45|4 +Brand#22|ECONOMY BURNISHED BRASS|9|4 +Brand#22|ECONOMY BURNISHED BRASS|23|4 +Brand#22|ECONOMY BURNISHED BRASS|45|4 +Brand#22|ECONOMY BURNISHED NICKEL|19|4 +Brand#22|ECONOMY BURNISHED NICKEL|49|4 +Brand#22|ECONOMY BURNISHED STEEL|9|4 +Brand#22|ECONOMY BURNISHED STEEL|14|4 +Brand#22|ECONOMY BURNISHED STEEL|23|4 +Brand#22|ECONOMY PLATED BRASS|36|4 +Brand#22|ECONOMY PLATED COPPER|23|4 +Brand#22|ECONOMY PLATED TIN|3|4 +Brand#22|ECONOMY POLISHED TIN|49|4 +Brand#22|LARGE ANODIZED BRASS|19|4 +Brand#22|LARGE ANODIZED COPPER|36|4 +Brand#22|LARGE ANODIZED STEEL|3|4 +Brand#22|LARGE BRUSHED BRASS|23|4 +Brand#22|LARGE BRUSHED BRASS|49|4 +Brand#22|LARGE BRUSHED STEEL|49|4 +Brand#22|LARGE BURNISHED COPPER|19|4 +Brand#22|LARGE BURNISHED STEEL|23|4 +Brand#22|LARGE BURNISHED STEEL|45|4 +Brand#22|LARGE BURNISHED TIN|45|4 +Brand#22|LARGE PLATED COPPER|14|4 +Brand#22|LARGE PLATED STEEL|49|4 +Brand#22|LARGE POLISHED BRASS|19|4 +Brand#22|LARGE POLISHED COPPER|19|4 +Brand#22|LARGE POLISHED COPPER|23|4 +Brand#22|LARGE POLISHED NICKEL|19|4 +Brand#22|LARGE POLISHED TIN|49|4 +Brand#22|MEDIUM ANODIZED BRASS|45|4 +Brand#22|MEDIUM ANODIZED COPPER|19|4 +Brand#22|MEDIUM ANODIZED COPPER|49|4 +Brand#22|MEDIUM ANODIZED NICKEL|9|4 +Brand#22|MEDIUM ANODIZED NICKEL|14|4 +Brand#22|MEDIUM ANODIZED NICKEL|36|4 +Brand#22|MEDIUM ANODIZED TIN|3|4 +Brand#22|MEDIUM ANODIZED TIN|9|4 +Brand#22|MEDIUM BRUSHED BRASS|3|4 +Brand#22|MEDIUM BRUSHED BRASS|14|4 +Brand#22|MEDIUM BRUSHED COPPER|3|4 +Brand#22|MEDIUM BRUSHED COPPER|45|4 +Brand#22|MEDIUM BRUSHED NICKEL|14|4 +Brand#22|MEDIUM BRUSHED TIN|45|4 +Brand#22|MEDIUM BURNISHED COPPER|36|4 +Brand#22|MEDIUM BURNISHED TIN|19|4 +Brand#22|MEDIUM BURNISHED TIN|23|4 +Brand#22|MEDIUM BURNISHED TIN|49|4 +Brand#22|MEDIUM PLATED BRASS|49|4 +Brand#22|MEDIUM PLATED COPPER|9|4 +Brand#22|MEDIUM PLATED STEEL|3|4 +Brand#22|PROMO ANODIZED BRASS|9|4 +Brand#22|PROMO ANODIZED STEEL|36|4 +Brand#22|PROMO ANODIZED TIN|45|4 +Brand#22|PROMO BRUSHED BRASS|3|4 +Brand#22|PROMO BRUSHED BRASS|9|4 +Brand#22|PROMO BRUSHED BRASS|36|4 +Brand#22|PROMO BRUSHED STEEL|36|4 +Brand#22|PROMO BURNISHED BRASS|23|4 +Brand#22|PROMO BURNISHED COPPER|9|4 +Brand#22|PROMO PLATED BRASS|14|4 +Brand#22|PROMO PLATED BRASS|45|4 +Brand#22|PROMO PLATED NICKEL|3|4 +Brand#22|PROMO PLATED STEEL|19|4 +Brand#22|PROMO POLISHED BRASS|3|4 +Brand#22|PROMO POLISHED STEEL|14|4 +Brand#22|PROMO POLISHED STEEL|23|4 +Brand#22|SMALL ANODIZED TIN|36|4 +Brand#22|SMALL ANODIZED TIN|49|4 +Brand#22|SMALL BRUSHED NICKEL|3|4 +Brand#22|SMALL BRUSHED NICKEL|36|4 +Brand#22|SMALL BRUSHED NICKEL|45|4 +Brand#22|SMALL BRUSHED TIN|45|4 +Brand#22|SMALL BURNISHED STEEL|23|4 +Brand#22|SMALL BURNISHED TIN|14|4 +Brand#22|SMALL PLATED STEEL|3|4 +Brand#22|SMALL PLATED TIN|9|4 +Brand#22|SMALL PLATED TIN|36|4 +Brand#22|SMALL POLISHED BRASS|23|4 +Brand#22|SMALL POLISHED NICKEL|19|4 +Brand#22|STANDARD ANODIZED BRASS|14|4 +Brand#22|STANDARD ANODIZED BRASS|23|4 +Brand#22|STANDARD BRUSHED COPPER|49|4 +Brand#22|STANDARD BRUSHED NICKEL|3|4 +Brand#22|STANDARD BRUSHED NICKEL|23|4 +Brand#22|STANDARD BRUSHED STEEL|9|4 +Brand#22|STANDARD BRUSHED TIN|19|4 +Brand#22|STANDARD BURNISHED COPPER|45|4 +Brand#22|STANDARD BURNISHED NICKEL|3|4 +Brand#22|STANDARD BURNISHED NICKEL|14|4 +Brand#22|STANDARD BURNISHED NICKEL|45|4 +Brand#22|STANDARD BURNISHED STEEL|3|4 +Brand#22|STANDARD BURNISHED STEEL|36|4 +Brand#22|STANDARD BURNISHED STEEL|45|4 +Brand#22|STANDARD BURNISHED STEEL|49|4 +Brand#22|STANDARD PLATED BRASS|45|4 +Brand#22|STANDARD PLATED NICKEL|3|4 +Brand#22|STANDARD PLATED NICKEL|45|4 +Brand#22|STANDARD PLATED STEEL|14|4 +Brand#22|STANDARD PLATED TIN|19|4 +Brand#22|STANDARD PLATED TIN|49|4 +Brand#22|STANDARD POLISHED COPPER|9|4 +Brand#22|STANDARD POLISHED STEEL|49|4 +Brand#22|STANDARD POLISHED TIN|45|4 +Brand#23|ECONOMY ANODIZED NICKEL|49|4 +Brand#23|ECONOMY ANODIZED STEEL|14|4 +Brand#23|ECONOMY ANODIZED STEEL|49|4 +Brand#23|ECONOMY ANODIZED TIN|49|4 +Brand#23|ECONOMY BRUSHED BRASS|3|4 +Brand#23|ECONOMY BRUSHED COPPER|9|4 +Brand#23|ECONOMY BRUSHED TIN|9|4 +Brand#23|ECONOMY BURNISHED STEEL|49|4 +Brand#23|ECONOMY PLATED COPPER|14|4 +Brand#23|ECONOMY PLATED NICKEL|23|4 +Brand#23|ECONOMY PLATED STEEL|14|4 +Brand#23|ECONOMY POLISHED NICKEL|9|4 +Brand#23|LARGE ANODIZED BRASS|14|4 +Brand#23|LARGE ANODIZED COPPER|9|4 +Brand#23|LARGE ANODIZED COPPER|14|4 +Brand#23|LARGE ANODIZED COPPER|45|4 +Brand#23|LARGE ANODIZED STEEL|19|4 +Brand#23|LARGE ANODIZED STEEL|36|4 +Brand#23|LARGE ANODIZED STEEL|49|4 +Brand#23|LARGE ANODIZED TIN|9|4 +Brand#23|LARGE PLATED BRASS|9|4 +Brand#23|LARGE PLATED BRASS|49|4 +Brand#23|LARGE PLATED COPPER|3|4 +Brand#23|LARGE POLISHED BRASS|45|4 +Brand#23|LARGE POLISHED STEEL|9|4 +Brand#23|MEDIUM ANODIZED BRASS|19|4 +Brand#23|MEDIUM ANODIZED NICKEL|3|4 +Brand#23|MEDIUM ANODIZED NICKEL|14|4 +Brand#23|MEDIUM ANODIZED STEEL|45|4 +Brand#23|MEDIUM ANODIZED TIN|36|4 +Brand#23|MEDIUM ANODIZED TIN|45|4 +Brand#23|MEDIUM BRUSHED COPPER|3|4 +Brand#23|MEDIUM BRUSHED COPPER|23|4 +Brand#23|MEDIUM BRUSHED NICKEL|3|4 +Brand#23|MEDIUM BRUSHED TIN|14|4 +Brand#23|MEDIUM BURNISHED BRASS|9|4 +Brand#23|MEDIUM BURNISHED BRASS|45|4 +Brand#23|MEDIUM BURNISHED COPPER|19|4 +Brand#23|MEDIUM PLATED COPPER|19|4 +Brand#23|MEDIUM PLATED COPPER|36|4 +Brand#23|MEDIUM PLATED COPPER|45|4 +Brand#23|MEDIUM PLATED NICKEL|9|4 +Brand#23|MEDIUM PLATED NICKEL|14|4 +Brand#23|PROMO ANODIZED COPPER|9|4 +Brand#23|PROMO ANODIZED COPPER|19|4 +Brand#23|PROMO ANODIZED STEEL|36|4 +Brand#23|PROMO ANODIZED TIN|14|4 +Brand#23|PROMO BRUSHED BRASS|3|4 +Brand#23|PROMO BRUSHED BRASS|19|4 +Brand#23|PROMO BRUSHED BRASS|36|4 +Brand#23|PROMO BRUSHED COPPER|3|4 +Brand#23|PROMO BRUSHED TIN|49|4 +Brand#23|PROMO BURNISHED BRASS|14|4 +Brand#23|PROMO BURNISHED BRASS|45|4 +Brand#23|PROMO BURNISHED COPPER|14|4 +Brand#23|PROMO PLATED BRASS|23|4 +Brand#23|PROMO POLISHED BRASS|14|4 +Brand#23|PROMO POLISHED BRASS|23|4 +Brand#23|PROMO POLISHED COPPER|36|4 +Brand#23|PROMO POLISHED STEEL|36|4 +Brand#23|SMALL ANODIZED BRASS|23|4 +Brand#23|SMALL ANODIZED STEEL|23|4 +Brand#23|SMALL BRUSHED BRASS|49|4 +Brand#23|SMALL BRUSHED COPPER|45|4 +Brand#23|SMALL BRUSHED STEEL|3|4 +Brand#23|SMALL BRUSHED STEEL|19|4 +Brand#23|SMALL BURNISHED BRASS|36|4 +Brand#23|SMALL BURNISHED COPPER|45|4 +Brand#23|SMALL BURNISHED COPPER|49|4 +Brand#23|SMALL BURNISHED STEEL|45|4 +Brand#23|SMALL PLATED BRASS|36|4 +Brand#23|SMALL PLATED BRASS|49|4 +Brand#23|SMALL PLATED COPPER|14|4 +Brand#23|SMALL PLATED TIN|14|4 +Brand#23|SMALL POLISHED BRASS|9|4 +Brand#23|SMALL POLISHED BRASS|14|4 +Brand#23|SMALL POLISHED NICKEL|3|4 +Brand#23|SMALL POLISHED STEEL|14|4 +Brand#23|SMALL POLISHED TIN|9|4 +Brand#23|STANDARD ANODIZED BRASS|19|4 +Brand#23|STANDARD ANODIZED BRASS|45|4 +Brand#23|STANDARD ANODIZED COPPER|19|4 +Brand#23|STANDARD ANODIZED TIN|3|4 +Brand#23|STANDARD BRUSHED COPPER|36|4 +Brand#23|STANDARD BRUSHED NICKEL|19|4 +Brand#23|STANDARD BRUSHED STEEL|49|4 +Brand#23|STANDARD BURNISHED COPPER|19|4 +Brand#23|STANDARD PLATED BRASS|3|4 +Brand#23|STANDARD PLATED BRASS|9|4 +Brand#23|STANDARD PLATED STEEL|36|4 +Brand#23|STANDARD PLATED TIN|19|4 +Brand#23|STANDARD POLISHED BRASS|9|4 +Brand#23|STANDARD POLISHED BRASS|49|4 +Brand#23|STANDARD POLISHED STEEL|19|4 +Brand#23|STANDARD POLISHED STEEL|49|4 +Brand#23|STANDARD POLISHED TIN|23|4 +Brand#24|ECONOMY ANODIZED BRASS|3|4 +Brand#24|ECONOMY ANODIZED BRASS|9|4 +Brand#24|ECONOMY ANODIZED BRASS|23|4 +Brand#24|ECONOMY ANODIZED COPPER|9|4 +Brand#24|ECONOMY ANODIZED COPPER|49|4 +Brand#24|ECONOMY BRUSHED BRASS|36|4 +Brand#24|ECONOMY BRUSHED COPPER|23|4 +Brand#24|ECONOMY BURNISHED COPPER|3|4 +Brand#24|ECONOMY BURNISHED NICKEL|19|4 +Brand#24|ECONOMY BURNISHED STEEL|45|4 +Brand#24|ECONOMY PLATED BRASS|23|4 +Brand#24|ECONOMY PLATED COPPER|36|4 +Brand#24|ECONOMY PLATED STEEL|45|4 +Brand#24|ECONOMY POLISHED BRASS|23|4 +Brand#24|ECONOMY POLISHED COPPER|45|4 +Brand#24|ECONOMY POLISHED NICKEL|36|4 +Brand#24|ECONOMY POLISHED STEEL|14|4 +Brand#24|ECONOMY POLISHED STEEL|36|4 +Brand#24|LARGE ANODIZED NICKEL|23|4 +Brand#24|LARGE ANODIZED NICKEL|45|4 +Brand#24|LARGE ANODIZED TIN|45|4 +Brand#24|LARGE BRUSHED BRASS|14|4 +Brand#24|LARGE BRUSHED BRASS|23|4 +Brand#24|LARGE BRUSHED STEEL|9|4 +Brand#24|LARGE BRUSHED STEEL|23|4 +Brand#24|LARGE BRUSHED STEEL|45|4 +Brand#24|LARGE BRUSHED TIN|49|4 +Brand#24|LARGE BURNISHED BRASS|3|4 +Brand#24|LARGE BURNISHED NICKEL|19|4 +Brand#24|LARGE PLATED BRASS|9|4 +Brand#24|LARGE PLATED NICKEL|36|4 +Brand#24|LARGE PLATED NICKEL|49|4 +Brand#24|LARGE PLATED TIN|9|4 +Brand#24|LARGE PLATED TIN|19|4 +Brand#24|LARGE PLATED TIN|36|4 +Brand#24|LARGE PLATED TIN|49|4 +Brand#24|LARGE POLISHED BRASS|9|4 +Brand#24|LARGE POLISHED COPPER|9|4 +Brand#24|LARGE POLISHED COPPER|49|4 +Brand#24|LARGE POLISHED NICKEL|19|4 +Brand#24|LARGE POLISHED STEEL|23|4 +Brand#24|LARGE POLISHED TIN|14|4 +Brand#24|MEDIUM ANODIZED COPPER|45|4 +Brand#24|MEDIUM BRUSHED COPPER|9|4 +Brand#24|MEDIUM BRUSHED COPPER|14|4 +Brand#24|MEDIUM BRUSHED NICKEL|9|4 +Brand#24|MEDIUM BRUSHED NICKEL|23|4 +Brand#24|MEDIUM BRUSHED STEEL|14|4 +Brand#24|MEDIUM BRUSHED STEEL|45|4 +Brand#24|MEDIUM BRUSHED STEEL|49|4 +Brand#24|MEDIUM BURNISHED BRASS|36|4 +Brand#24|MEDIUM BURNISHED NICKEL|36|4 +Brand#24|MEDIUM BURNISHED STEEL|36|4 +Brand#24|MEDIUM PLATED COPPER|14|4 +Brand#24|MEDIUM PLATED STEEL|3|4 +Brand#24|MEDIUM PLATED STEEL|19|4 +Brand#24|PROMO ANODIZED NICKEL|9|4 +Brand#24|PROMO ANODIZED NICKEL|19|4 +Brand#24|PROMO ANODIZED NICKEL|45|4 +Brand#24|PROMO ANODIZED STEEL|3|4 +Brand#24|PROMO ANODIZED TIN|45|4 +Brand#24|PROMO BRUSHED BRASS|19|4 +Brand#24|PROMO BRUSHED NICKEL|19|4 +Brand#24|PROMO BRUSHED NICKEL|45|4 +Brand#24|PROMO BRUSHED STEEL|49|4 +Brand#24|PROMO BURNISHED BRASS|3|4 +Brand#24|PROMO BURNISHED BRASS|45|4 +Brand#24|PROMO BURNISHED STEEL|49|4 +Brand#24|PROMO PLATED BRASS|3|4 +Brand#24|PROMO PLATED COPPER|23|4 +Brand#24|PROMO PLATED COPPER|49|4 +Brand#24|PROMO POLISHED BRASS|3|4 +Brand#24|PROMO POLISHED BRASS|14|4 +Brand#24|PROMO POLISHED NICKEL|3|4 +Brand#24|PROMO POLISHED STEEL|14|4 +Brand#24|PROMO POLISHED STEEL|19|4 +Brand#24|PROMO POLISHED STEEL|23|4 +Brand#24|SMALL ANODIZED BRASS|19|4 +Brand#24|SMALL ANODIZED COPPER|3|4 +Brand#24|SMALL ANODIZED NICKEL|14|4 +Brand#24|SMALL ANODIZED STEEL|36|4 +Brand#24|SMALL ANODIZED TIN|3|4 +Brand#24|SMALL ANODIZED TIN|36|4 +Brand#24|SMALL BRUSHED COPPER|49|4 +Brand#24|SMALL BRUSHED NICKEL|49|4 +Brand#24|SMALL BURNISHED BRASS|14|4 +Brand#24|SMALL BURNISHED BRASS|19|4 +Brand#24|SMALL BURNISHED TIN|9|4 +Brand#24|SMALL PLATED BRASS|3|4 +Brand#24|SMALL PLATED COPPER|14|4 +Brand#24|SMALL PLATED COPPER|36|4 +Brand#24|SMALL PLATED NICKEL|14|4 +Brand#24|SMALL PLATED NICKEL|49|4 +Brand#24|SMALL POLISHED BRASS|3|4 +Brand#24|SMALL POLISHED NICKEL|9|4 +Brand#24|SMALL POLISHED NICKEL|19|4 +Brand#24|SMALL POLISHED NICKEL|36|4 +Brand#24|SMALL POLISHED STEEL|9|4 +Brand#24|SMALL POLISHED STEEL|36|4 +Brand#24|STANDARD ANODIZED TIN|9|4 +Brand#24|STANDARD ANODIZED TIN|49|4 +Brand#24|STANDARD BRUSHED BRASS|14|4 +Brand#24|STANDARD BRUSHED COPPER|23|4 +Brand#24|STANDARD BRUSHED NICKEL|19|4 +Brand#24|STANDARD BRUSHED STEEL|14|4 +Brand#24|STANDARD BRUSHED TIN|36|4 +Brand#24|STANDARD BURNISHED COPPER|19|4 +Brand#24|STANDARD BURNISHED COPPER|36|4 +Brand#24|STANDARD BURNISHED NICKEL|45|4 +Brand#24|STANDARD PLATED BRASS|36|4 +Brand#24|STANDARD PLATED COPPER|45|4 +Brand#24|STANDARD PLATED NICKEL|36|4 +Brand#24|STANDARD PLATED TIN|36|4 +Brand#24|STANDARD POLISHED COPPER|45|4 +Brand#24|STANDARD POLISHED NICKEL|14|4 +Brand#25|ECONOMY ANODIZED BRASS|14|4 +Brand#25|ECONOMY ANODIZED BRASS|49|4 +Brand#25|ECONOMY ANODIZED TIN|9|4 +Brand#25|ECONOMY ANODIZED TIN|19|4 +Brand#25|ECONOMY ANODIZED TIN|49|4 +Brand#25|ECONOMY BRUSHED COPPER|36|4 +Brand#25|ECONOMY BURNISHED COPPER|45|4 +Brand#25|ECONOMY BURNISHED TIN|19|4 +Brand#25|ECONOMY PLATED NICKEL|23|4 +Brand#25|ECONOMY PLATED TIN|14|4 +Brand#25|ECONOMY POLISHED BRASS|23|4 +Brand#25|ECONOMY POLISHED COPPER|9|4 +Brand#25|ECONOMY POLISHED NICKEL|3|4 +Brand#25|ECONOMY POLISHED TIN|9|4 +Brand#25|ECONOMY POLISHED TIN|45|4 +Brand#25|LARGE ANODIZED BRASS|3|4 +Brand#25|LARGE ANODIZED BRASS|14|4 +Brand#25|LARGE ANODIZED COPPER|36|4 +Brand#25|LARGE ANODIZED NICKEL|23|4 +Brand#25|LARGE ANODIZED STEEL|23|4 +Brand#25|LARGE BRUSHED NICKEL|19|4 +Brand#25|LARGE BRUSHED NICKEL|49|4 +Brand#25|LARGE BRUSHED TIN|3|4 +Brand#25|LARGE BRUSHED TIN|9|4 +Brand#25|LARGE BURNISHED BRASS|19|4 +Brand#25|LARGE BURNISHED BRASS|23|4 +Brand#25|LARGE BURNISHED BRASS|49|4 +Brand#25|LARGE BURNISHED NICKEL|14|4 +Brand#25|LARGE BURNISHED TIN|49|4 +Brand#25|LARGE PLATED BRASS|14|4 +Brand#25|LARGE PLATED NICKEL|23|4 +Brand#25|LARGE PLATED NICKEL|45|4 +Brand#25|LARGE PLATED TIN|19|4 +Brand#25|LARGE PLATED TIN|23|4 +Brand#25|LARGE POLISHED BRASS|9|4 +Brand#25|LARGE POLISHED COPPER|14|4 +Brand#25|LARGE POLISHED COPPER|36|4 +Brand#25|MEDIUM ANODIZED TIN|36|4 +Brand#25|MEDIUM BRUSHED COPPER|9|4 +Brand#25|MEDIUM BRUSHED COPPER|36|4 +Brand#25|MEDIUM BRUSHED COPPER|49|4 +Brand#25|MEDIUM BURNISHED COPPER|49|4 +Brand#25|MEDIUM BURNISHED NICKEL|9|4 +Brand#25|MEDIUM BURNISHED NICKEL|49|4 +Brand#25|MEDIUM BURNISHED STEEL|3|4 +Brand#25|MEDIUM BURNISHED STEEL|36|4 +Brand#25|MEDIUM BURNISHED STEEL|45|4 +Brand#25|MEDIUM BURNISHED STEEL|49|4 +Brand#25|MEDIUM BURNISHED TIN|9|4 +Brand#25|MEDIUM BURNISHED TIN|36|4 +Brand#25|MEDIUM PLATED BRASS|45|4 +Brand#25|MEDIUM PLATED COPPER|14|4 +Brand#25|MEDIUM PLATED NICKEL|45|4 +Brand#25|MEDIUM PLATED STEEL|9|4 +Brand#25|MEDIUM PLATED STEEL|36|4 +Brand#25|PROMO ANODIZED COPPER|14|4 +Brand#25|PROMO ANODIZED COPPER|19|4 +Brand#25|PROMO ANODIZED STEEL|36|4 +Brand#25|PROMO ANODIZED TIN|3|4 +Brand#25|PROMO ANODIZED TIN|14|4 +Brand#25|PROMO BRUSHED NICKEL|3|4 +Brand#25|PROMO BRUSHED STEEL|19|4 +Brand#25|PROMO BRUSHED TIN|14|4 +Brand#25|PROMO BRUSHED TIN|36|4 +Brand#25|PROMO BURNISHED COPPER|19|4 +Brand#25|PROMO BURNISHED COPPER|45|4 +Brand#25|PROMO BURNISHED COPPER|49|4 +Brand#25|PROMO BURNISHED NICKEL|36|4 +Brand#25|PROMO BURNISHED TIN|3|4 +Brand#25|PROMO PLATED BRASS|45|4 +Brand#25|PROMO PLATED COPPER|19|4 +Brand#25|PROMO PLATED NICKEL|45|4 +Brand#25|PROMO PLATED NICKEL|49|4 +Brand#25|PROMO PLATED STEEL|23|4 +Brand#25|PROMO POLISHED BRASS|23|4 +Brand#25|SMALL ANODIZED BRASS|45|4 +Brand#25|SMALL ANODIZED NICKEL|19|4 +Brand#25|SMALL ANODIZED STEEL|23|4 +Brand#25|SMALL ANODIZED TIN|14|4 +Brand#25|SMALL ANODIZED TIN|19|4 +Brand#25|SMALL BRUSHED COPPER|45|4 +Brand#25|SMALL BRUSHED NICKEL|9|4 +Brand#25|SMALL BURNISHED COPPER|3|4 +Brand#25|SMALL BURNISHED STEEL|3|4 +Brand#25|SMALL BURNISHED STEEL|14|4 +Brand#25|SMALL BURNISHED TIN|3|4 +Brand#25|SMALL PLATED BRASS|19|4 +Brand#25|SMALL PLATED COPPER|23|4 +Brand#25|SMALL PLATED STEEL|45|4 +Brand#25|SMALL PLATED TIN|36|4 +Brand#25|SMALL POLISHED BRASS|23|4 +Brand#25|SMALL POLISHED COPPER|9|4 +Brand#25|SMALL POLISHED STEEL|14|4 +Brand#25|STANDARD ANODIZED STEEL|3|4 +Brand#25|STANDARD ANODIZED STEEL|19|4 +Brand#25|STANDARD ANODIZED TIN|9|4 +Brand#25|STANDARD BRUSHED BRASS|14|4 +Brand#25|STANDARD BRUSHED NICKEL|19|4 +Brand#25|STANDARD BRUSHED TIN|9|4 +Brand#25|STANDARD BURNISHED NICKEL|9|4 +Brand#25|STANDARD PLATED BRASS|3|4 +Brand#25|STANDARD PLATED COPPER|14|4 +Brand#25|STANDARD PLATED NICKEL|36|4 +Brand#25|STANDARD POLISHED BRASS|45|4 +Brand#25|STANDARD POLISHED COPPER|23|4 +Brand#25|STANDARD POLISHED NICKEL|3|4 +Brand#25|STANDARD POLISHED NICKEL|49|4 +Brand#25|STANDARD POLISHED TIN|36|4 +Brand#25|STANDARD POLISHED TIN|45|4 +Brand#31|ECONOMY ANODIZED BRASS|3|4 +Brand#31|ECONOMY ANODIZED COPPER|45|4 +Brand#31|ECONOMY ANODIZED STEEL|3|4 +Brand#31|ECONOMY ANODIZED TIN|45|4 +Brand#31|ECONOMY BRUSHED BRASS|14|4 +Brand#31|ECONOMY BRUSHED COPPER|19|4 +Brand#31|ECONOMY BRUSHED NICKEL|9|4 +Brand#31|ECONOMY BRUSHED NICKEL|14|4 +Brand#31|ECONOMY BRUSHED NICKEL|49|4 +Brand#31|ECONOMY BURNISHED COPPER|36|4 +Brand#31|ECONOMY BURNISHED STEEL|3|4 +Brand#31|ECONOMY BURNISHED TIN|49|4 +Brand#31|ECONOMY PLATED COPPER|49|4 +Brand#31|ECONOMY PLATED NICKEL|9|4 +Brand#31|ECONOMY PLATED STEEL|23|4 +Brand#31|ECONOMY PLATED TIN|36|4 +Brand#31|ECONOMY PLATED TIN|49|4 +Brand#31|ECONOMY POLISHED COPPER|3|4 +Brand#31|ECONOMY POLISHED COPPER|36|4 +Brand#31|ECONOMY POLISHED COPPER|49|4 +Brand#31|ECONOMY POLISHED NICKEL|3|4 +Brand#31|LARGE ANODIZED BRASS|19|4 +Brand#31|LARGE ANODIZED STEEL|45|4 +Brand#31|LARGE BRUSHED BRASS|36|4 +Brand#31|LARGE BRUSHED BRASS|49|4 +Brand#31|LARGE BRUSHED TIN|3|4 +Brand#31|LARGE BURNISHED BRASS|9|4 +Brand#31|LARGE PLATED COPPER|19|4 +Brand#31|LARGE PLATED NICKEL|14|4 +Brand#31|LARGE PLATED TIN|9|4 +Brand#31|LARGE PLATED TIN|14|4 +Brand#31|LARGE POLISHED BRASS|14|4 +Brand#31|LARGE POLISHED STEEL|14|4 +Brand#31|LARGE POLISHED STEEL|45|4 +Brand#31|LARGE POLISHED TIN|19|4 +Brand#31|MEDIUM ANODIZED BRASS|23|4 +Brand#31|MEDIUM ANODIZED BRASS|36|4 +Brand#31|MEDIUM ANODIZED COPPER|14|4 +Brand#31|MEDIUM ANODIZED COPPER|19|4 +Brand#31|MEDIUM ANODIZED COPPER|36|4 +Brand#31|MEDIUM ANODIZED STEEL|14|4 +Brand#31|MEDIUM ANODIZED STEEL|49|4 +Brand#31|MEDIUM ANODIZED TIN|19|4 +Brand#31|MEDIUM ANODIZED TIN|49|4 +Brand#31|MEDIUM BRUSHED BRASS|36|4 +Brand#31|MEDIUM BRUSHED STEEL|14|4 +Brand#31|MEDIUM BURNISHED BRASS|14|4 +Brand#31|MEDIUM BURNISHED COPPER|3|4 +Brand#31|MEDIUM BURNISHED NICKEL|9|4 +Brand#31|MEDIUM BURNISHED STEEL|9|4 +Brand#31|MEDIUM BURNISHED TIN|14|4 +Brand#31|MEDIUM BURNISHED TIN|23|4 +Brand#31|MEDIUM PLATED BRASS|3|4 +Brand#31|MEDIUM PLATED TIN|9|4 +Brand#31|MEDIUM PLATED TIN|36|4 +Brand#31|MEDIUM PLATED TIN|45|4 +Brand#31|PROMO ANODIZED BRASS|3|4 +Brand#31|PROMO ANODIZED NICKEL|9|4 +Brand#31|PROMO BRUSHED BRASS|3|4 +Brand#31|PROMO BRUSHED BRASS|23|4 +Brand#31|PROMO BRUSHED COPPER|23|4 +Brand#31|PROMO BRUSHED NICKEL|45|4 +Brand#31|PROMO BURNISHED COPPER|36|4 +Brand#31|PROMO BURNISHED STEEL|3|4 +Brand#31|PROMO BURNISHED TIN|3|4 +Brand#31|PROMO PLATED BRASS|19|4 +Brand#31|PROMO PLATED NICKEL|36|4 +Brand#31|PROMO POLISHED BRASS|49|4 +Brand#31|PROMO POLISHED COPPER|14|4 +Brand#31|PROMO POLISHED NICKEL|3|4 +Brand#31|PROMO POLISHED NICKEL|9|4 +Brand#31|PROMO POLISHED TIN|3|4 +Brand#31|PROMO POLISHED TIN|23|4 +Brand#31|SMALL ANODIZED COPPER|45|4 +Brand#31|SMALL ANODIZED STEEL|23|4 +Brand#31|SMALL ANODIZED TIN|3|4 +Brand#31|SMALL BRUSHED COPPER|36|4 +Brand#31|SMALL BRUSHED COPPER|49|4 +Brand#31|SMALL BRUSHED NICKEL|19|4 +Brand#31|SMALL BRUSHED NICKEL|23|4 +Brand#31|SMALL BURNISHED BRASS|45|4 +Brand#31|SMALL BURNISHED NICKEL|9|4 +Brand#31|SMALL BURNISHED NICKEL|36|4 +Brand#31|SMALL PLATED COPPER|36|4 +Brand#31|SMALL PLATED NICKEL|9|4 +Brand#31|SMALL PLATED NICKEL|36|4 +Brand#31|SMALL POLISHED BRASS|3|4 +Brand#31|SMALL POLISHED COPPER|45|4 +Brand#31|SMALL POLISHED NICKEL|45|4 +Brand#31|SMALL POLISHED TIN|23|4 +Brand#31|SMALL POLISHED TIN|49|4 +Brand#31|STANDARD BRUSHED STEEL|23|4 +Brand#31|STANDARD BRUSHED STEEL|49|4 +Brand#31|STANDARD BURNISHED BRASS|14|4 +Brand#31|STANDARD BURNISHED NICKEL|45|4 +Brand#31|STANDARD PLATED NICKEL|3|4 +Brand#31|STANDARD POLISHED BRASS|3|4 +Brand#31|STANDARD POLISHED BRASS|45|4 +Brand#31|STANDARD POLISHED STEEL|36|4 +Brand#32|ECONOMY ANODIZED BRASS|19|4 +Brand#32|ECONOMY ANODIZED COPPER|36|4 +Brand#32|ECONOMY ANODIZED STEEL|23|4 +Brand#32|ECONOMY ANODIZED STEEL|36|4 +Brand#32|ECONOMY ANODIZED STEEL|45|4 +Brand#32|ECONOMY ANODIZED TIN|19|4 +Brand#32|ECONOMY BRUSHED COPPER|45|4 +Brand#32|ECONOMY BRUSHED TIN|45|4 +Brand#32|ECONOMY BURNISHED BRASS|23|4 +Brand#32|ECONOMY BURNISHED COPPER|36|4 +Brand#32|ECONOMY BURNISHED COPPER|45|4 +Brand#32|ECONOMY BURNISHED STEEL|19|4 +Brand#32|ECONOMY PLATED BRASS|9|4 +Brand#32|ECONOMY PLATED COPPER|9|4 +Brand#32|ECONOMY PLATED NICKEL|23|4 +Brand#32|ECONOMY PLATED TIN|45|4 +Brand#32|ECONOMY POLISHED STEEL|3|4 +Brand#32|LARGE ANODIZED BRASS|23|4 +Brand#32|LARGE ANODIZED BRASS|36|4 +Brand#32|LARGE ANODIZED NICKEL|45|4 +Brand#32|LARGE ANODIZED STEEL|3|4 +Brand#32|LARGE ANODIZED STEEL|14|4 +Brand#32|LARGE BRUSHED STEEL|45|4 +Brand#32|LARGE BRUSHED TIN|45|4 +Brand#32|LARGE BURNISHED NICKEL|36|4 +Brand#32|LARGE BURNISHED TIN|19|4 +Brand#32|LARGE BURNISHED TIN|45|4 +Brand#32|LARGE PLATED BRASS|3|4 +Brand#32|LARGE PLATED NICKEL|49|4 +Brand#32|LARGE PLATED STEEL|19|4 +Brand#32|LARGE PLATED STEEL|36|4 +Brand#32|LARGE POLISHED BRASS|45|4 +Brand#32|LARGE POLISHED COPPER|9|4 +Brand#32|LARGE POLISHED COPPER|49|4 +Brand#32|LARGE POLISHED NICKEL|3|4 +Brand#32|MEDIUM ANODIZED BRASS|3|4 +Brand#32|MEDIUM ANODIZED BRASS|9|4 +Brand#32|MEDIUM ANODIZED TIN|23|4 +Brand#32|MEDIUM BRUSHED BRASS|23|4 +Brand#32|MEDIUM BRUSHED BRASS|49|4 +Brand#32|MEDIUM BRUSHED COPPER|9|4 +Brand#32|MEDIUM BRUSHED COPPER|19|4 +Brand#32|MEDIUM BRUSHED TIN|49|4 +Brand#32|MEDIUM BURNISHED BRASS|9|4 +Brand#32|MEDIUM BURNISHED BRASS|36|4 +Brand#32|MEDIUM BURNISHED BRASS|49|4 +Brand#32|MEDIUM BURNISHED COPPER|9|4 +Brand#32|MEDIUM BURNISHED COPPER|45|4 +Brand#32|MEDIUM BURNISHED NICKEL|49|4 +Brand#32|MEDIUM BURNISHED TIN|9|4 +Brand#32|MEDIUM BURNISHED TIN|45|4 +Brand#32|MEDIUM PLATED BRASS|3|4 +Brand#32|MEDIUM PLATED BRASS|49|4 +Brand#32|MEDIUM PLATED COPPER|3|4 +Brand#32|MEDIUM PLATED STEEL|9|4 +Brand#32|MEDIUM PLATED TIN|9|4 +Brand#32|PROMO ANODIZED BRASS|3|4 +Brand#32|PROMO ANODIZED COPPER|19|4 +Brand#32|PROMO ANODIZED NICKEL|23|4 +Brand#32|PROMO BRUSHED COPPER|23|4 +Brand#32|PROMO BRUSHED NICKEL|14|4 +Brand#32|PROMO BRUSHED NICKEL|36|4 +Brand#32|PROMO BRUSHED STEEL|14|4 +Brand#32|PROMO BRUSHED STEEL|23|4 +Brand#32|PROMO BRUSHED STEEL|49|4 +Brand#32|PROMO BURNISHED BRASS|45|4 +Brand#32|PROMO BURNISHED NICKEL|45|4 +Brand#32|PROMO BURNISHED TIN|14|4 +Brand#32|PROMO BURNISHED TIN|45|4 +Brand#32|PROMO PLATED TIN|19|4 +Brand#32|PROMO POLISHED NICKEL|36|4 +Brand#32|PROMO POLISHED TIN|3|4 +Brand#32|SMALL ANODIZED BRASS|3|4 +Brand#32|SMALL ANODIZED NICKEL|3|4 +Brand#32|SMALL ANODIZED NICKEL|14|4 +Brand#32|SMALL ANODIZED TIN|9|4 +Brand#32|SMALL BRUSHED BRASS|9|4 +Brand#32|SMALL BRUSHED BRASS|19|4 +Brand#32|SMALL BRUSHED COPPER|3|4 +Brand#32|SMALL BRUSHED COPPER|23|4 +Brand#32|SMALL BRUSHED NICKEL|9|4 +Brand#32|SMALL BRUSHED NICKEL|45|4 +Brand#32|SMALL BRUSHED STEEL|23|4 +Brand#32|SMALL BRUSHED TIN|9|4 +Brand#32|SMALL BURNISHED NICKEL|36|4 +Brand#32|SMALL BURNISHED STEEL|3|4 +Brand#32|SMALL BURNISHED TIN|23|4 +Brand#32|SMALL PLATED BRASS|49|4 +Brand#32|SMALL PLATED COPPER|36|4 +Brand#32|SMALL PLATED COPPER|45|4 +Brand#32|SMALL PLATED NICKEL|45|4 +Brand#32|SMALL PLATED STEEL|45|4 +Brand#32|SMALL PLATED TIN|23|4 +Brand#32|SMALL PLATED TIN|36|4 +Brand#32|SMALL PLATED TIN|45|4 +Brand#32|SMALL POLISHED NICKEL|36|4 +Brand#32|SMALL POLISHED STEEL|14|4 +Brand#32|SMALL POLISHED STEEL|23|4 +Brand#32|SMALL POLISHED STEEL|36|4 +Brand#32|SMALL POLISHED TIN|36|4 +Brand#32|SMALL POLISHED TIN|45|4 +Brand#32|STANDARD ANODIZED NICKEL|19|4 +Brand#32|STANDARD ANODIZED TIN|9|4 +Brand#32|STANDARD ANODIZED TIN|14|4 +Brand#32|STANDARD ANODIZED TIN|19|4 +Brand#32|STANDARD BRUSHED NICKEL|23|4 +Brand#32|STANDARD BURNISHED BRASS|36|4 +Brand#32|STANDARD BURNISHED BRASS|45|4 +Brand#32|STANDARD BURNISHED COPPER|3|4 +Brand#32|STANDARD BURNISHED COPPER|36|4 +Brand#32|STANDARD BURNISHED NICKEL|49|4 +Brand#32|STANDARD BURNISHED STEEL|49|4 +Brand#32|STANDARD BURNISHED TIN|23|4 +Brand#32|STANDARD PLATED BRASS|9|4 +Brand#32|STANDARD PLATED BRASS|45|4 +Brand#32|STANDARD PLATED STEEL|36|4 +Brand#32|STANDARD POLISHED BRASS|14|4 +Brand#32|STANDARD POLISHED COPPER|36|4 +Brand#32|STANDARD POLISHED STEEL|14|4 +Brand#33|ECONOMY ANODIZED BRASS|23|4 +Brand#33|ECONOMY ANODIZED COPPER|9|4 +Brand#33|ECONOMY ANODIZED NICKEL|3|4 +Brand#33|ECONOMY ANODIZED NICKEL|9|4 +Brand#33|ECONOMY ANODIZED NICKEL|23|4 +Brand#33|ECONOMY ANODIZED NICKEL|36|4 +Brand#33|ECONOMY BRUSHED BRASS|14|4 +Brand#33|ECONOMY BRUSHED COPPER|23|4 +Brand#33|ECONOMY BURNISHED BRASS|49|4 +Brand#33|ECONOMY BURNISHED COPPER|3|4 +Brand#33|ECONOMY BURNISHED COPPER|14|4 +Brand#33|ECONOMY BURNISHED STEEL|3|4 +Brand#33|ECONOMY BURNISHED TIN|36|4 +Brand#33|ECONOMY BURNISHED TIN|45|4 +Brand#33|ECONOMY PLATED COPPER|19|4 +Brand#33|ECONOMY PLATED COPPER|45|4 +Brand#33|ECONOMY PLATED NICKEL|14|4 +Brand#33|ECONOMY PLATED NICKEL|36|4 +Brand#33|ECONOMY PLATED STEEL|3|4 +Brand#33|ECONOMY PLATED STEEL|23|4 +Brand#33|ECONOMY PLATED STEEL|36|4 +Brand#33|ECONOMY POLISHED BRASS|14|4 +Brand#33|ECONOMY POLISHED NICKEL|19|4 +Brand#33|ECONOMY POLISHED TIN|9|4 +Brand#33|LARGE ANODIZED BRASS|36|4 +Brand#33|LARGE ANODIZED COPPER|19|4 +Brand#33|LARGE ANODIZED COPPER|45|4 +Brand#33|LARGE ANODIZED NICKEL|36|4 +Brand#33|LARGE ANODIZED NICKEL|45|4 +Brand#33|LARGE ANODIZED STEEL|3|4 +Brand#33|LARGE ANODIZED STEEL|45|4 +Brand#33|LARGE ANODIZED TIN|45|4 +Brand#33|LARGE BRUSHED BRASS|3|4 +Brand#33|LARGE BRUSHED BRASS|49|4 +Brand#33|LARGE BRUSHED STEEL|19|4 +Brand#33|LARGE BRUSHED TIN|36|4 +Brand#33|LARGE BURNISHED COPPER|45|4 +Brand#33|LARGE BURNISHED NICKEL|23|4 +Brand#33|LARGE BURNISHED STEEL|19|4 +Brand#33|LARGE PLATED BRASS|3|4 +Brand#33|LARGE PLATED COPPER|19|4 +Brand#33|LARGE PLATED STEEL|3|4 +Brand#33|LARGE PLATED STEEL|19|4 +Brand#33|LARGE PLATED TIN|45|4 +Brand#33|LARGE POLISHED BRASS|45|4 +Brand#33|LARGE POLISHED STEEL|14|4 +Brand#33|LARGE POLISHED STEEL|23|4 +Brand#33|LARGE POLISHED TIN|23|4 +Brand#33|MEDIUM ANODIZED BRASS|3|4 +Brand#33|MEDIUM ANODIZED COPPER|9|4 +Brand#33|MEDIUM ANODIZED COPPER|36|4 +Brand#33|MEDIUM ANODIZED COPPER|49|4 +Brand#33|MEDIUM ANODIZED NICKEL|3|4 +Brand#33|MEDIUM ANODIZED NICKEL|19|4 +Brand#33|MEDIUM BRUSHED BRASS|3|4 +Brand#33|MEDIUM BRUSHED STEEL|19|4 +Brand#33|MEDIUM BRUSHED TIN|14|4 +Brand#33|MEDIUM BURNISHED COPPER|14|4 +Brand#33|MEDIUM BURNISHED COPPER|49|4 +Brand#33|MEDIUM BURNISHED TIN|36|4 +Brand#33|MEDIUM PLATED BRASS|3|4 +Brand#33|MEDIUM PLATED STEEL|3|4 +Brand#33|MEDIUM PLATED STEEL|49|4 +Brand#33|PROMO ANODIZED BRASS|3|4 +Brand#33|PROMO BRUSHED BRASS|49|4 +Brand#33|PROMO BURNISHED COPPER|23|4 +Brand#33|PROMO BURNISHED NICKEL|14|4 +Brand#33|PROMO BURNISHED NICKEL|36|4 +Brand#33|PROMO BURNISHED TIN|19|4 +Brand#33|PROMO BURNISHED TIN|23|4 +Brand#33|PROMO PLATED COPPER|14|4 +Brand#33|PROMO PLATED STEEL|45|4 +Brand#33|PROMO PLATED STEEL|49|4 +Brand#33|PROMO PLATED TIN|49|4 +Brand#33|PROMO POLISHED COPPER|3|4 +Brand#33|PROMO POLISHED STEEL|3|4 +Brand#33|PROMO POLISHED STEEL|9|4 +Brand#33|PROMO POLISHED STEEL|23|4 +Brand#33|SMALL ANODIZED BRASS|19|4 +Brand#33|SMALL ANODIZED COPPER|23|4 +Brand#33|SMALL ANODIZED COPPER|49|4 +Brand#33|SMALL ANODIZED STEEL|9|4 +Brand#33|SMALL BRUSHED BRASS|3|4 +Brand#33|SMALL BRUSHED COPPER|3|4 +Brand#33|SMALL BRUSHED NICKEL|45|4 +Brand#33|SMALL BRUSHED STEEL|3|4 +Brand#33|SMALL BRUSHED TIN|9|4 +Brand#33|SMALL BURNISHED BRASS|19|4 +Brand#33|SMALL BURNISHED NICKEL|3|4 +Brand#33|SMALL PLATED BRASS|3|4 +Brand#33|SMALL PLATED STEEL|14|4 +Brand#33|SMALL PLATED STEEL|45|4 +Brand#33|SMALL PLATED TIN|23|4 +Brand#33|SMALL PLATED TIN|36|4 +Brand#33|SMALL POLISHED NICKEL|23|4 +Brand#33|SMALL POLISHED TIN|19|4 +Brand#33|SMALL POLISHED TIN|23|4 +Brand#33|SMALL POLISHED TIN|45|4 +Brand#33|STANDARD ANODIZED COPPER|49|4 +Brand#33|STANDARD ANODIZED STEEL|14|4 +Brand#33|STANDARD ANODIZED STEEL|45|4 +Brand#33|STANDARD ANODIZED STEEL|49|4 +Brand#33|STANDARD ANODIZED TIN|45|4 +Brand#33|STANDARD BRUSHED BRASS|9|4 +Brand#33|STANDARD BRUSHED NICKEL|45|4 +Brand#33|STANDARD BRUSHED STEEL|9|4 +Brand#33|STANDARD BRUSHED TIN|36|4 +Brand#33|STANDARD BURNISHED BRASS|9|4 +Brand#33|STANDARD BURNISHED BRASS|23|4 +Brand#33|STANDARD BURNISHED NICKEL|49|4 +Brand#33|STANDARD PLATED BRASS|49|4 +Brand#33|STANDARD PLATED COPPER|3|4 +Brand#33|STANDARD PLATED COPPER|14|4 +Brand#33|STANDARD PLATED NICKEL|36|4 +Brand#33|STANDARD PLATED STEEL|3|4 +Brand#33|STANDARD PLATED STEEL|36|4 +Brand#33|STANDARD PLATED TIN|14|4 +Brand#33|STANDARD POLISHED BRASS|9|4 +Brand#33|STANDARD POLISHED BRASS|19|4 +Brand#33|STANDARD POLISHED STEEL|3|4 +Brand#33|STANDARD POLISHED STEEL|9|4 +Brand#33|STANDARD POLISHED STEEL|14|4 +Brand#34|ECONOMY ANODIZED BRASS|9|4 +Brand#34|ECONOMY ANODIZED COPPER|3|4 +Brand#34|ECONOMY ANODIZED COPPER|14|4 +Brand#34|ECONOMY ANODIZED COPPER|19|4 +Brand#34|ECONOMY ANODIZED STEEL|9|4 +Brand#34|ECONOMY ANODIZED TIN|49|4 +Brand#34|ECONOMY BRUSHED BRASS|14|4 +Brand#34|ECONOMY BRUSHED NICKEL|49|4 +Brand#34|ECONOMY BURNISHED COPPER|9|4 +Brand#34|ECONOMY BURNISHED STEEL|19|4 +Brand#34|ECONOMY BURNISHED TIN|3|4 +Brand#34|ECONOMY BURNISHED TIN|23|4 +Brand#34|ECONOMY PLATED BRASS|9|4 +Brand#34|ECONOMY PLATED BRASS|14|4 +Brand#34|ECONOMY PLATED COPPER|3|4 +Brand#34|ECONOMY PLATED NICKEL|45|4 +Brand#34|ECONOMY PLATED TIN|14|4 +Brand#34|ECONOMY PLATED TIN|45|4 +Brand#34|ECONOMY POLISHED BRASS|45|4 +Brand#34|LARGE ANODIZED BRASS|14|4 +Brand#34|LARGE ANODIZED BRASS|23|4 +Brand#34|LARGE ANODIZED BRASS|36|4 +Brand#34|LARGE ANODIZED NICKEL|3|4 +Brand#34|LARGE ANODIZED TIN|49|4 +Brand#34|LARGE BRUSHED BRASS|49|4 +Brand#34|LARGE BRUSHED COPPER|23|4 +Brand#34|LARGE BRUSHED NICKEL|23|4 +Brand#34|LARGE BRUSHED STEEL|14|4 +Brand#34|LARGE BRUSHED STEEL|19|4 +Brand#34|LARGE BRUSHED TIN|9|4 +Brand#34|LARGE BURNISHED BRASS|23|4 +Brand#34|LARGE BURNISHED COPPER|3|4 +Brand#34|LARGE BURNISHED COPPER|36|4 +Brand#34|LARGE BURNISHED NICKEL|19|4 +Brand#34|LARGE PLATED BRASS|23|4 +Brand#34|LARGE PLATED BRASS|36|4 +Brand#34|LARGE PLATED BRASS|45|4 +Brand#34|LARGE PLATED COPPER|23|4 +Brand#34|LARGE PLATED COPPER|49|4 +Brand#34|LARGE PLATED STEEL|49|4 +Brand#34|LARGE POLISHED NICKEL|49|4 +Brand#34|MEDIUM ANODIZED COPPER|36|4 +Brand#34|MEDIUM ANODIZED TIN|3|4 +Brand#34|MEDIUM BRUSHED BRASS|49|4 +Brand#34|MEDIUM BRUSHED COPPER|9|4 +Brand#34|MEDIUM BRUSHED NICKEL|9|4 +Brand#34|MEDIUM BRUSHED NICKEL|23|4 +Brand#34|MEDIUM BRUSHED TIN|3|4 +Brand#34|MEDIUM BRUSHED TIN|14|4 +Brand#34|MEDIUM BURNISHED STEEL|45|4 +Brand#34|MEDIUM BURNISHED STEEL|49|4 +Brand#34|MEDIUM PLATED COPPER|36|4 +Brand#34|MEDIUM PLATED TIN|3|4 +Brand#34|MEDIUM PLATED TIN|14|4 +Brand#34|PROMO ANODIZED COPPER|45|4 +Brand#34|PROMO ANODIZED NICKEL|14|4 +Brand#34|PROMO ANODIZED STEEL|49|4 +Brand#34|PROMO ANODIZED TIN|14|4 +Brand#34|PROMO BRUSHED BRASS|9|4 +Brand#34|PROMO BRUSHED BRASS|23|4 +Brand#34|PROMO BRUSHED COPPER|36|4 +Brand#34|PROMO BRUSHED STEEL|36|4 +Brand#34|PROMO BURNISHED BRASS|49|4 +Brand#34|PROMO BURNISHED STEEL|3|4 +Brand#34|PROMO PLATED BRASS|9|4 +Brand#34|PROMO PLATED STEEL|49|4 +Brand#34|PROMO POLISHED BRASS|23|4 +Brand#34|PROMO POLISHED NICKEL|3|4 +Brand#34|PROMO POLISHED NICKEL|36|4 +Brand#34|SMALL ANODIZED BRASS|36|4 +Brand#34|SMALL ANODIZED COPPER|45|4 +Brand#34|SMALL ANODIZED NICKEL|14|4 +Brand#34|SMALL ANODIZED NICKEL|36|4 +Brand#34|SMALL ANODIZED STEEL|3|4 +Brand#34|SMALL ANODIZED STEEL|19|4 +Brand#34|SMALL ANODIZED STEEL|23|4 +Brand#34|SMALL ANODIZED STEEL|36|4 +Brand#34|SMALL BRUSHED BRASS|14|4 +Brand#34|SMALL BRUSHED BRASS|36|4 +Brand#34|SMALL BRUSHED NICKEL|14|4 +Brand#34|SMALL BRUSHED NICKEL|36|4 +Brand#34|SMALL BRUSHED NICKEL|45|4 +Brand#34|SMALL BRUSHED TIN|9|4 +Brand#34|SMALL BRUSHED TIN|23|4 +Brand#34|SMALL BRUSHED TIN|36|4 +Brand#34|SMALL BURNISHED COPPER|9|4 +Brand#34|SMALL BURNISHED TIN|36|4 +Brand#34|SMALL PLATED BRASS|14|4 +Brand#34|SMALL PLATED COPPER|36|4 +Brand#34|SMALL PLATED TIN|45|4 +Brand#34|SMALL POLISHED NICKEL|14|4 +Brand#34|SMALL POLISHED NICKEL|45|4 +Brand#34|SMALL POLISHED TIN|9|4 +Brand#34|SMALL POLISHED TIN|14|4 +Brand#34|SMALL POLISHED TIN|19|4 +Brand#34|STANDARD ANODIZED BRASS|23|4 +Brand#34|STANDARD ANODIZED BRASS|36|4 +Brand#34|STANDARD ANODIZED COPPER|45|4 +Brand#34|STANDARD ANODIZED NICKEL|36|4 +Brand#34|STANDARD ANODIZED STEEL|9|4 +Brand#34|STANDARD ANODIZED STEEL|49|4 +Brand#34|STANDARD ANODIZED TIN|9|4 +Brand#34|STANDARD BRUSHED BRASS|19|4 +Brand#34|STANDARD BRUSHED BRASS|23|4 +Brand#34|STANDARD BRUSHED NICKEL|23|4 +Brand#34|STANDARD BRUSHED STEEL|3|4 +Brand#34|STANDARD BRUSHED TIN|19|4 +Brand#34|STANDARD BURNISHED COPPER|45|4 +Brand#34|STANDARD BURNISHED NICKEL|19|4 +Brand#34|STANDARD BURNISHED NICKEL|45|4 +Brand#34|STANDARD BURNISHED STEEL|36|4 +Brand#34|STANDARD BURNISHED TIN|45|4 +Brand#34|STANDARD PLATED BRASS|9|4 +Brand#34|STANDARD PLATED COPPER|9|4 +Brand#34|STANDARD PLATED NICKEL|36|4 +Brand#35|ECONOMY ANODIZED COPPER|3|4 +Brand#35|ECONOMY ANODIZED STEEL|45|4 +Brand#35|ECONOMY BRUSHED BRASS|3|4 +Brand#35|ECONOMY BRUSHED NICKEL|49|4 +Brand#35|ECONOMY BRUSHED STEEL|23|4 +Brand#35|ECONOMY BRUSHED STEEL|45|4 +Brand#35|ECONOMY BRUSHED TIN|14|4 +Brand#35|ECONOMY BRUSHED TIN|23|4 +Brand#35|ECONOMY BURNISHED NICKEL|19|4 +Brand#35|ECONOMY BURNISHED STEEL|36|4 +Brand#35|ECONOMY BURNISHED TIN|9|4 +Brand#35|ECONOMY BURNISHED TIN|19|4 +Brand#35|ECONOMY BURNISHED TIN|49|4 +Brand#35|ECONOMY POLISHED COPPER|9|4 +Brand#35|ECONOMY POLISHED TIN|19|4 +Brand#35|LARGE ANODIZED BRASS|3|4 +Brand#35|LARGE ANODIZED BRASS|23|4 +Brand#35|LARGE ANODIZED COPPER|49|4 +Brand#35|LARGE ANODIZED STEEL|36|4 +Brand#35|LARGE ANODIZED TIN|9|4 +Brand#35|LARGE BRUSHED COPPER|9|4 +Brand#35|LARGE BRUSHED COPPER|23|4 +Brand#35|LARGE BRUSHED STEEL|3|4 +Brand#35|LARGE BRUSHED STEEL|9|4 +Brand#35|LARGE BURNISHED BRASS|36|4 +Brand#35|LARGE BURNISHED BRASS|45|4 +Brand#35|LARGE BURNISHED COPPER|23|4 +Brand#35|LARGE BURNISHED NICKEL|23|4 +Brand#35|LARGE PLATED BRASS|9|4 +Brand#35|LARGE PLATED COPPER|36|4 +Brand#35|LARGE POLISHED BRASS|49|4 +Brand#35|LARGE POLISHED STEEL|9|4 +Brand#35|LARGE POLISHED TIN|14|4 +Brand#35|MEDIUM ANODIZED BRASS|9|4 +Brand#35|MEDIUM ANODIZED BRASS|36|4 +Brand#35|MEDIUM ANODIZED COPPER|9|4 +Brand#35|MEDIUM BRUSHED BRASS|14|4 +Brand#35|MEDIUM BRUSHED COPPER|9|4 +Brand#35|MEDIUM BRUSHED COPPER|36|4 +Brand#35|MEDIUM BURNISHED BRASS|49|4 +Brand#35|MEDIUM BURNISHED NICKEL|45|4 +Brand#35|MEDIUM BURNISHED TIN|36|4 +Brand#35|MEDIUM PLATED BRASS|23|4 +Brand#35|MEDIUM PLATED COPPER|9|4 +Brand#35|MEDIUM PLATED NICKEL|45|4 +Brand#35|MEDIUM PLATED NICKEL|49|4 +Brand#35|MEDIUM PLATED STEEL|49|4 +Brand#35|PROMO ANODIZED COPPER|49|4 +Brand#35|PROMO ANODIZED NICKEL|19|4 +Brand#35|PROMO ANODIZED NICKEL|23|4 +Brand#35|PROMO ANODIZED TIN|3|4 +Brand#35|PROMO ANODIZED TIN|14|4 +Brand#35|PROMO BRUSHED BRASS|49|4 +Brand#35|PROMO BRUSHED NICKEL|14|4 +Brand#35|PROMO BRUSHED NICKEL|19|4 +Brand#35|PROMO BURNISHED BRASS|3|4 +Brand#35|PROMO BURNISHED STEEL|3|4 +Brand#35|PROMO PLATED BRASS|19|4 +Brand#35|PROMO PLATED COPPER|14|4 +Brand#35|PROMO PLATED STEEL|23|4 +Brand#35|PROMO PLATED STEEL|36|4 +Brand#35|PROMO PLATED TIN|19|4 +Brand#35|PROMO POLISHED BRASS|9|4 +Brand#35|PROMO POLISHED BRASS|36|4 +Brand#35|PROMO POLISHED NICKEL|36|4 +Brand#35|PROMO POLISHED STEEL|23|4 +Brand#35|PROMO POLISHED TIN|36|4 +Brand#35|PROMO POLISHED TIN|45|4 +Brand#35|SMALL ANODIZED COPPER|9|4 +Brand#35|SMALL ANODIZED STEEL|19|4 +Brand#35|SMALL ANODIZED TIN|19|4 +Brand#35|SMALL BRUSHED BRASS|36|4 +Brand#35|SMALL BRUSHED STEEL|49|4 +Brand#35|SMALL BRUSHED TIN|3|4 +Brand#35|SMALL BRUSHED TIN|19|4 +Brand#35|SMALL BRUSHED TIN|23|4 +Brand#35|SMALL BURNISHED BRASS|23|4 +Brand#35|SMALL BURNISHED STEEL|36|4 +Brand#35|SMALL BURNISHED TIN|3|4 +Brand#35|SMALL BURNISHED TIN|36|4 +Brand#35|SMALL BURNISHED TIN|49|4 +Brand#35|SMALL PLATED BRASS|23|4 +Brand#35|SMALL PLATED STEEL|14|4 +Brand#35|SMALL POLISHED BRASS|36|4 +Brand#35|SMALL POLISHED STEEL|3|4 +Brand#35|SMALL POLISHED STEEL|49|4 +Brand#35|SMALL POLISHED TIN|23|4 +Brand#35|SMALL POLISHED TIN|45|4 +Brand#35|STANDARD ANODIZED NICKEL|14|4 +Brand#35|STANDARD ANODIZED STEEL|23|4 +Brand#35|STANDARD ANODIZED STEEL|45|4 +Brand#35|STANDARD ANODIZED TIN|9|4 +Brand#35|STANDARD ANODIZED TIN|19|4 +Brand#35|STANDARD BRUSHED BRASS|3|4 +Brand#35|STANDARD BRUSHED BRASS|23|4 +Brand#35|STANDARD BRUSHED BRASS|36|4 +Brand#35|STANDARD BRUSHED COPPER|36|4 +Brand#35|STANDARD BRUSHED NICKEL|36|4 +Brand#35|STANDARD BRUSHED NICKEL|49|4 +Brand#35|STANDARD BRUSHED TIN|9|4 +Brand#35|STANDARD BURNISHED BRASS|9|4 +Brand#35|STANDARD BURNISHED BRASS|19|4 +Brand#35|STANDARD BURNISHED BRASS|23|4 +Brand#35|STANDARD BURNISHED COPPER|36|4 +Brand#35|STANDARD BURNISHED STEEL|14|4 +Brand#35|STANDARD PLATED COPPER|19|4 +Brand#35|STANDARD PLATED NICKEL|23|4 +Brand#35|STANDARD PLATED STEEL|14|4 +Brand#35|STANDARD PLATED STEEL|23|4 +Brand#35|STANDARD PLATED TIN|49|4 +Brand#35|STANDARD POLISHED NICKEL|23|4 +Brand#35|STANDARD POLISHED TIN|23|4 +Brand#35|STANDARD POLISHED TIN|45|4 +Brand#41|ECONOMY ANODIZED STEEL|49|4 +Brand#41|ECONOMY BRUSHED BRASS|3|4 +Brand#41|ECONOMY BRUSHED COPPER|36|4 +Brand#41|ECONOMY BRUSHED NICKEL|23|4 +Brand#41|ECONOMY BRUSHED STEEL|36|4 +Brand#41|ECONOMY BRUSHED STEEL|45|4 +Brand#41|ECONOMY BRUSHED TIN|14|4 +Brand#41|ECONOMY PLATED COPPER|3|4 +Brand#41|ECONOMY PLATED STEEL|3|4 +Brand#41|ECONOMY PLATED TIN|23|4 +Brand#41|ECONOMY POLISHED COPPER|19|4 +Brand#41|ECONOMY POLISHED NICKEL|9|4 +Brand#41|ECONOMY POLISHED NICKEL|14|4 +Brand#41|ECONOMY POLISHED NICKEL|23|4 +Brand#41|ECONOMY POLISHED NICKEL|49|4 +Brand#41|ECONOMY POLISHED STEEL|9|4 +Brand#41|ECONOMY POLISHED STEEL|19|4 +Brand#41|ECONOMY POLISHED STEEL|45|4 +Brand#41|ECONOMY POLISHED TIN|19|4 +Brand#41|LARGE ANODIZED BRASS|14|4 +Brand#41|LARGE ANODIZED BRASS|23|4 +Brand#41|LARGE ANODIZED COPPER|49|4 +Brand#41|LARGE ANODIZED STEEL|3|4 +Brand#41|LARGE ANODIZED STEEL|23|4 +Brand#41|LARGE BRUSHED COPPER|23|4 +Brand#41|LARGE BRUSHED COPPER|49|4 +Brand#41|LARGE BRUSHED STEEL|19|4 +Brand#41|LARGE BURNISHED BRASS|45|4 +Brand#41|LARGE BURNISHED COPPER|3|4 +Brand#41|LARGE BURNISHED NICKEL|23|4 +Brand#41|LARGE BURNISHED TIN|9|4 +Brand#41|LARGE PLATED NICKEL|3|4 +Brand#41|LARGE PLATED NICKEL|23|4 +Brand#41|LARGE PLATED STEEL|9|4 +Brand#41|LARGE PLATED STEEL|36|4 +Brand#41|LARGE PLATED TIN|9|4 +Brand#41|LARGE POLISHED BRASS|36|4 +Brand#41|LARGE POLISHED COPPER|19|4 +Brand#41|LARGE POLISHED COPPER|49|4 +Brand#41|LARGE POLISHED NICKEL|36|4 +Brand#41|LARGE POLISHED STEEL|14|4 +Brand#41|MEDIUM ANODIZED BRASS|9|4 +Brand#41|MEDIUM ANODIZED COPPER|14|4 +Brand#41|MEDIUM ANODIZED NICKEL|3|4 +Brand#41|MEDIUM ANODIZED NICKEL|9|4 +Brand#41|MEDIUM ANODIZED STEEL|14|4 +Brand#41|MEDIUM BRUSHED COPPER|3|4 +Brand#41|MEDIUM BRUSHED TIN|9|4 +Brand#41|MEDIUM BURNISHED COPPER|23|4 +Brand#41|MEDIUM BURNISHED STEEL|9|4 +Brand#41|MEDIUM BURNISHED STEEL|45|4 +Brand#41|MEDIUM BURNISHED TIN|3|4 +Brand#41|MEDIUM PLATED BRASS|19|4 +Brand#41|MEDIUM PLATED BRASS|45|4 +Brand#41|MEDIUM PLATED COPPER|19|4 +Brand#41|MEDIUM PLATED STEEL|19|4 +Brand#41|MEDIUM PLATED STEEL|23|4 +Brand#41|PROMO ANODIZED BRASS|19|4 +Brand#41|PROMO ANODIZED COPPER|9|4 +Brand#41|PROMO ANODIZED NICKEL|9|4 +Brand#41|PROMO BRUSHED BRASS|14|4 +Brand#41|PROMO BRUSHED COPPER|36|4 +Brand#41|PROMO BRUSHED NICKEL|14|4 +Brand#41|PROMO BURNISHED BRASS|49|4 +Brand#41|PROMO BURNISHED NICKEL|36|4 +Brand#41|PROMO BURNISHED TIN|3|4 +Brand#41|PROMO PLATED NICKEL|14|4 +Brand#41|PROMO PLATED NICKEL|45|4 +Brand#41|PROMO PLATED STEEL|3|4 +Brand#41|PROMO PLATED TIN|3|4 +Brand#41|PROMO POLISHED COPPER|23|4 +Brand#41|SMALL ANODIZED BRASS|3|4 +Brand#41|SMALL ANODIZED BRASS|14|4 +Brand#41|SMALL ANODIZED STEEL|45|4 +Brand#41|SMALL ANODIZED TIN|9|4 +Brand#41|SMALL BRUSHED TIN|19|4 +Brand#41|SMALL BURNISHED COPPER|9|4 +Brand#41|SMALL BURNISHED NICKEL|3|4 +Brand#41|SMALL BURNISHED TIN|45|4 +Brand#41|SMALL PLATED COPPER|14|4 +Brand#41|SMALL PLATED COPPER|36|4 +Brand#41|SMALL PLATED COPPER|49|4 +Brand#41|SMALL PLATED TIN|19|4 +Brand#41|SMALL POLISHED COPPER|14|4 +Brand#41|SMALL POLISHED COPPER|19|4 +Brand#41|SMALL POLISHED COPPER|36|4 +Brand#41|SMALL POLISHED TIN|45|4 +Brand#41|STANDARD ANODIZED COPPER|19|4 +Brand#41|STANDARD ANODIZED NICKEL|9|4 +Brand#41|STANDARD ANODIZED STEEL|49|4 +Brand#41|STANDARD ANODIZED TIN|9|4 +Brand#41|STANDARD ANODIZED TIN|36|4 +Brand#41|STANDARD ANODIZED TIN|49|4 +Brand#41|STANDARD BRUSHED BRASS|19|4 +Brand#41|STANDARD BRUSHED NICKEL|3|4 +Brand#41|STANDARD BRUSHED NICKEL|9|4 +Brand#41|STANDARD BRUSHED STEEL|45|4 +Brand#41|STANDARD BRUSHED TIN|45|4 +Brand#41|STANDARD BURNISHED BRASS|23|4 +Brand#41|STANDARD BURNISHED BRASS|36|4 +Brand#41|STANDARD BURNISHED COPPER|49|4 +Brand#41|STANDARD BURNISHED STEEL|45|4 +Brand#41|STANDARD PLATED BRASS|45|4 +Brand#41|STANDARD PLATED NICKEL|14|4 +Brand#41|STANDARD PLATED STEEL|45|4 +Brand#41|STANDARD PLATED TIN|49|4 +Brand#41|STANDARD POLISHED STEEL|9|4 +Brand#41|STANDARD POLISHED STEEL|19|4 +Brand#41|STANDARD POLISHED TIN|45|4 +Brand#42|ECONOMY ANODIZED NICKEL|19|4 +Brand#42|ECONOMY BRUSHED BRASS|14|4 +Brand#42|ECONOMY BRUSHED COPPER|3|4 +Brand#42|ECONOMY BRUSHED COPPER|14|4 +Brand#42|ECONOMY BRUSHED NICKEL|14|4 +Brand#42|ECONOMY BRUSHED STEEL|14|4 +Brand#42|ECONOMY BRUSHED TIN|19|4 +Brand#42|ECONOMY BRUSHED TIN|49|4 +Brand#42|ECONOMY BURNISHED BRASS|19|4 +Brand#42|ECONOMY BURNISHED COPPER|23|4 +Brand#42|ECONOMY BURNISHED NICKEL|14|4 +Brand#42|ECONOMY BURNISHED TIN|14|4 +Brand#42|ECONOMY PLATED COPPER|23|4 +Brand#42|ECONOMY POLISHED BRASS|3|4 +Brand#42|ECONOMY POLISHED COPPER|9|4 +Brand#42|ECONOMY POLISHED STEEL|9|4 +Brand#42|ECONOMY POLISHED STEEL|36|4 +Brand#42|ECONOMY POLISHED TIN|14|4 +Brand#42|LARGE ANODIZED BRASS|49|4 +Brand#42|LARGE ANODIZED COPPER|14|4 +Brand#42|LARGE ANODIZED COPPER|49|4 +Brand#42|LARGE ANODIZED NICKEL|45|4 +Brand#42|LARGE ANODIZED NICKEL|49|4 +Brand#42|LARGE ANODIZED TIN|45|4 +Brand#42|LARGE BRUSHED BRASS|49|4 +Brand#42|LARGE BURNISHED BRASS|45|4 +Brand#42|LARGE BURNISHED BRASS|49|4 +Brand#42|LARGE BURNISHED COPPER|9|4 +Brand#42|LARGE BURNISHED TIN|9|4 +Brand#42|LARGE PLATED BRASS|45|4 +Brand#42|LARGE PLATED COPPER|9|4 +Brand#42|LARGE PLATED NICKEL|36|4 +Brand#42|LARGE PLATED TIN|23|4 +Brand#42|LARGE POLISHED BRASS|9|4 +Brand#42|LARGE POLISHED NICKEL|3|4 +Brand#42|LARGE POLISHED NICKEL|23|4 +Brand#42|LARGE POLISHED STEEL|9|4 +Brand#42|MEDIUM ANODIZED BRASS|23|4 +Brand#42|MEDIUM ANODIZED COPPER|19|4 +Brand#42|MEDIUM ANODIZED NICKEL|14|4 +Brand#42|MEDIUM ANODIZED NICKEL|19|4 +Brand#42|MEDIUM ANODIZED NICKEL|23|4 +Brand#42|MEDIUM ANODIZED STEEL|9|4 +Brand#42|MEDIUM ANODIZED STEEL|14|4 +Brand#42|MEDIUM ANODIZED STEEL|23|4 +Brand#42|MEDIUM ANODIZED TIN|14|4 +Brand#42|MEDIUM ANODIZED TIN|19|4 +Brand#42|MEDIUM BRUSHED COPPER|45|4 +Brand#42|MEDIUM BRUSHED COPPER|49|4 +Brand#42|MEDIUM BRUSHED STEEL|36|4 +Brand#42|MEDIUM BURNISHED COPPER|49|4 +Brand#42|MEDIUM BURNISHED TIN|3|4 +Brand#42|MEDIUM BURNISHED TIN|49|4 +Brand#42|MEDIUM PLATED NICKEL|45|4 +Brand#42|MEDIUM PLATED STEEL|3|4 +Brand#42|MEDIUM PLATED STEEL|23|4 +Brand#42|MEDIUM PLATED STEEL|45|4 +Brand#42|PROMO ANODIZED NICKEL|3|4 +Brand#42|PROMO ANODIZED NICKEL|19|4 +Brand#42|PROMO ANODIZED STEEL|49|4 +Brand#42|PROMO BRUSHED COPPER|45|4 +Brand#42|PROMO BRUSHED STEEL|19|4 +Brand#42|PROMO BRUSHED TIN|45|4 +Brand#42|PROMO BURNISHED COPPER|45|4 +Brand#42|PROMO BURNISHED NICKEL|3|4 +Brand#42|PROMO BURNISHED STEEL|9|4 +Brand#42|PROMO BURNISHED TIN|49|4 +Brand#42|PROMO PLATED BRASS|45|4 +Brand#42|PROMO PLATED NICKEL|23|4 +Brand#42|PROMO PLATED STEEL|19|4 +Brand#42|PROMO PLATED STEEL|45|4 +Brand#42|PROMO POLISHED COPPER|36|4 +Brand#42|PROMO POLISHED NICKEL|3|4 +Brand#42|SMALL ANODIZED BRASS|23|4 +Brand#42|SMALL ANODIZED COPPER|14|4 +Brand#42|SMALL ANODIZED COPPER|19|4 +Brand#42|SMALL ANODIZED NICKEL|23|4 +Brand#42|SMALL BRUSHED TIN|49|4 +Brand#42|SMALL BURNISHED BRASS|3|4 +Brand#42|SMALL BURNISHED BRASS|36|4 +Brand#42|SMALL BURNISHED COPPER|9|4 +Brand#42|SMALL BURNISHED NICKEL|9|4 +Brand#42|SMALL BURNISHED TIN|9|4 +Brand#42|SMALL PLATED NICKEL|9|4 +Brand#42|SMALL PLATED TIN|36|4 +Brand#42|SMALL POLISHED BRASS|3|4 +Brand#42|SMALL POLISHED COPPER|36|4 +Brand#42|SMALL POLISHED NICKEL|23|4 +Brand#42|SMALL POLISHED STEEL|49|4 +Brand#42|SMALL POLISHED TIN|3|4 +Brand#42|STANDARD ANODIZED BRASS|49|4 +Brand#42|STANDARD ANODIZED COPPER|49|4 +Brand#42|STANDARD ANODIZED NICKEL|36|4 +Brand#42|STANDARD ANODIZED NICKEL|45|4 +Brand#42|STANDARD BRUSHED NICKEL|23|4 +Brand#42|STANDARD BURNISHED NICKEL|49|4 +Brand#42|STANDARD BURNISHED STEEL|3|4 +Brand#42|STANDARD BURNISHED TIN|19|4 +Brand#42|STANDARD PLATED BRASS|19|4 +Brand#42|STANDARD PLATED COPPER|9|4 +Brand#42|STANDARD PLATED NICKEL|45|4 +Brand#42|STANDARD PLATED STEEL|3|4 +Brand#42|STANDARD POLISHED BRASS|36|4 +Brand#42|STANDARD POLISHED BRASS|45|4 +Brand#42|STANDARD POLISHED COPPER|14|4 +Brand#42|STANDARD POLISHED NICKEL|45|4 +Brand#42|STANDARD POLISHED TIN|9|4 +Brand#42|STANDARD POLISHED TIN|19|4 +Brand#42|STANDARD POLISHED TIN|23|4 +Brand#42|STANDARD POLISHED TIN|36|4 +Brand#43|ECONOMY ANODIZED COPPER|19|4 +Brand#43|ECONOMY ANODIZED COPPER|45|4 +Brand#43|ECONOMY ANODIZED NICKEL|3|4 +Brand#43|ECONOMY ANODIZED NICKEL|49|4 +Brand#43|ECONOMY ANODIZED STEEL|23|4 +Brand#43|ECONOMY ANODIZED TIN|49|4 +Brand#43|ECONOMY BRUSHED BRASS|49|4 +Brand#43|ECONOMY BRUSHED COPPER|45|4 +Brand#43|ECONOMY BRUSHED NICKEL|9|4 +Brand#43|ECONOMY BURNISHED NICKEL|9|4 +Brand#43|ECONOMY BURNISHED TIN|19|4 +Brand#43|ECONOMY PLATED COPPER|36|4 +Brand#43|ECONOMY PLATED STEEL|9|4 +Brand#43|ECONOMY PLATED TIN|14|4 +Brand#43|ECONOMY PLATED TIN|19|4 +Brand#43|ECONOMY PLATED TIN|49|4 +Brand#43|ECONOMY POLISHED COPPER|19|4 +Brand#43|ECONOMY POLISHED NICKEL|36|4 +Brand#43|ECONOMY POLISHED TIN|14|4 +Brand#43|ECONOMY POLISHED TIN|45|4 +Brand#43|LARGE ANODIZED BRASS|14|4 +Brand#43|LARGE ANODIZED BRASS|36|4 +Brand#43|LARGE ANODIZED COPPER|45|4 +Brand#43|LARGE BRUSHED COPPER|3|4 +Brand#43|LARGE BRUSHED NICKEL|14|4 +Brand#43|LARGE BRUSHED NICKEL|19|4 +Brand#43|LARGE BRUSHED NICKEL|45|4 +Brand#43|LARGE BRUSHED NICKEL|49|4 +Brand#43|LARGE BURNISHED COPPER|3|4 +Brand#43|LARGE BURNISHED TIN|23|4 +Brand#43|LARGE BURNISHED TIN|45|4 +Brand#43|LARGE PLATED BRASS|45|4 +Brand#43|LARGE PLATED STEEL|14|4 +Brand#43|LARGE PLATED TIN|36|4 +Brand#43|LARGE PLATED TIN|45|4 +Brand#43|LARGE POLISHED BRASS|9|4 +Brand#43|LARGE POLISHED COPPER|9|4 +Brand#43|LARGE POLISHED COPPER|19|4 +Brand#43|LARGE POLISHED STEEL|14|4 +Brand#43|LARGE POLISHED TIN|45|4 +Brand#43|MEDIUM ANODIZED BRASS|14|4 +Brand#43|MEDIUM ANODIZED COPPER|36|4 +Brand#43|MEDIUM ANODIZED COPPER|49|4 +Brand#43|MEDIUM ANODIZED STEEL|19|4 +Brand#43|MEDIUM ANODIZED STEEL|36|4 +Brand#43|MEDIUM BRUSHED BRASS|9|4 +Brand#43|MEDIUM BRUSHED BRASS|49|4 +Brand#43|MEDIUM BRUSHED COPPER|3|4 +Brand#43|MEDIUM BRUSHED NICKEL|9|4 +Brand#43|MEDIUM BRUSHED STEEL|23|4 +Brand#43|MEDIUM BURNISHED COPPER|14|4 +Brand#43|MEDIUM BURNISHED COPPER|45|4 +Brand#43|MEDIUM BURNISHED TIN|23|4 +Brand#43|MEDIUM PLATED BRASS|3|4 +Brand#43|MEDIUM PLATED COPPER|14|4 +Brand#43|MEDIUM PLATED NICKEL|36|4 +Brand#43|MEDIUM PLATED NICKEL|45|4 +Brand#43|MEDIUM PLATED TIN|49|4 +Brand#43|PROMO ANODIZED NICKEL|45|4 +Brand#43|PROMO ANODIZED TIN|14|4 +Brand#43|PROMO BRUSHED NICKEL|14|4 +Brand#43|PROMO BRUSHED STEEL|14|4 +Brand#43|PROMO BRUSHED TIN|45|4 +Brand#43|PROMO BURNISHED BRASS|49|4 +Brand#43|PROMO BURNISHED NICKEL|9|4 +Brand#43|PROMO BURNISHED STEEL|3|4 +Brand#43|PROMO BURNISHED STEEL|36|4 +Brand#43|PROMO BURNISHED TIN|36|4 +Brand#43|PROMO PLATED BRASS|19|4 +Brand#43|PROMO PLATED COPPER|45|4 +Brand#43|PROMO PLATED COPPER|49|4 +Brand#43|PROMO PLATED TIN|3|4 +Brand#43|PROMO POLISHED BRASS|19|4 +Brand#43|PROMO POLISHED BRASS|23|4 +Brand#43|PROMO POLISHED NICKEL|49|4 +Brand#43|PROMO POLISHED STEEL|14|4 +Brand#43|PROMO POLISHED STEEL|19|4 +Brand#43|PROMO POLISHED STEEL|23|4 +Brand#43|PROMO POLISHED STEEL|36|4 +Brand#43|SMALL ANODIZED BRASS|19|4 +Brand#43|SMALL ANODIZED NICKEL|9|4 +Brand#43|SMALL BRUSHED NICKEL|3|4 +Brand#43|SMALL BRUSHED NICKEL|9|4 +Brand#43|SMALL BURNISHED BRASS|49|4 +Brand#43|SMALL BURNISHED STEEL|23|4 +Brand#43|SMALL PLATED BRASS|14|4 +Brand#43|SMALL PLATED BRASS|36|4 +Brand#43|SMALL PLATED COPPER|23|4 +Brand#43|SMALL PLATED COPPER|49|4 +Brand#43|SMALL PLATED NICKEL|36|4 +Brand#43|SMALL PLATED NICKEL|49|4 +Brand#43|SMALL PLATED STEEL|14|4 +Brand#43|SMALL PLATED TIN|49|4 +Brand#43|SMALL POLISHED STEEL|19|4 +Brand#43|STANDARD ANODIZED BRASS|3|4 +Brand#43|STANDARD ANODIZED COPPER|49|4 +Brand#43|STANDARD ANODIZED NICKEL|14|4 +Brand#43|STANDARD BRUSHED TIN|14|4 +Brand#43|STANDARD BURNISHED BRASS|23|4 +Brand#43|STANDARD BURNISHED STEEL|19|4 +Brand#43|STANDARD BURNISHED STEEL|23|4 +Brand#43|STANDARD PLATED BRASS|9|4 +Brand#43|STANDARD PLATED BRASS|19|4 +Brand#43|STANDARD PLATED BRASS|49|4 +Brand#43|STANDARD PLATED COPPER|36|4 +Brand#43|STANDARD PLATED NICKEL|14|4 +Brand#43|STANDARD PLATED NICKEL|19|4 +Brand#43|STANDARD PLATED TIN|14|4 +Brand#43|STANDARD POLISHED BRASS|23|4 +Brand#43|STANDARD POLISHED TIN|9|4 +Brand#44|ECONOMY ANODIZED BRASS|3|4 +Brand#44|ECONOMY ANODIZED BRASS|45|4 +Brand#44|ECONOMY ANODIZED NICKEL|36|4 +Brand#44|ECONOMY ANODIZED STEEL|19|4 +Brand#44|ECONOMY BRUSHED COPPER|23|4 +Brand#44|ECONOMY BRUSHED TIN|49|4 +Brand#44|ECONOMY BURNISHED COPPER|19|4 +Brand#44|ECONOMY BURNISHED STEEL|45|4 +Brand#44|ECONOMY PLATED STEEL|19|4 +Brand#44|ECONOMY PLATED STEEL|23|4 +Brand#44|ECONOMY PLATED TIN|23|4 +Brand#44|ECONOMY POLISHED BRASS|23|4 +Brand#44|ECONOMY POLISHED COPPER|9|4 +Brand#44|ECONOMY POLISHED COPPER|45|4 +Brand#44|ECONOMY POLISHED NICKEL|14|4 +Brand#44|ECONOMY POLISHED NICKEL|23|4 +Brand#44|ECONOMY POLISHED STEEL|49|4 +Brand#44|ECONOMY POLISHED TIN|23|4 +Brand#44|ECONOMY POLISHED TIN|36|4 +Brand#44|LARGE ANODIZED BRASS|19|4 +Brand#44|LARGE ANODIZED TIN|3|4 +Brand#44|LARGE ANODIZED TIN|14|4 +Brand#44|LARGE BRUSHED TIN|3|4 +Brand#44|LARGE BRUSHED TIN|23|4 +Brand#44|LARGE BURNISHED BRASS|23|4 +Brand#44|LARGE BURNISHED BRASS|49|4 +Brand#44|LARGE BURNISHED COPPER|3|4 +Brand#44|LARGE BURNISHED COPPER|19|4 +Brand#44|LARGE BURNISHED COPPER|36|4 +Brand#44|LARGE BURNISHED TIN|14|4 +Brand#44|LARGE PLATED BRASS|9|4 +Brand#44|LARGE PLATED BRASS|49|4 +Brand#44|LARGE PLATED NICKEL|14|4 +Brand#44|LARGE PLATED STEEL|14|4 +Brand#44|LARGE PLATED TIN|19|4 +Brand#44|LARGE PLATED TIN|23|4 +Brand#44|LARGE POLISHED STEEL|23|4 +Brand#44|LARGE POLISHED STEEL|49|4 +Brand#44|MEDIUM ANODIZED COPPER|45|4 +Brand#44|MEDIUM ANODIZED NICKEL|45|4 +Brand#44|MEDIUM BRUSHED BRASS|49|4 +Brand#44|MEDIUM BRUSHED COPPER|3|4 +Brand#44|MEDIUM BRUSHED COPPER|45|4 +Brand#44|MEDIUM BRUSHED STEEL|19|4 +Brand#44|MEDIUM BRUSHED TIN|49|4 +Brand#44|MEDIUM BURNISHED COPPER|45|4 +Brand#44|MEDIUM BURNISHED NICKEL|23|4 +Brand#44|MEDIUM BURNISHED TIN|23|4 +Brand#44|MEDIUM PLATED COPPER|14|4 +Brand#44|PROMO ANODIZED COPPER|23|4 +Brand#44|PROMO ANODIZED STEEL|36|4 +Brand#44|PROMO BRUSHED COPPER|23|4 +Brand#44|PROMO BRUSHED COPPER|36|4 +Brand#44|PROMO BRUSHED TIN|19|4 +Brand#44|PROMO PLATED BRASS|3|4 +Brand#44|PROMO PLATED COPPER|36|4 +Brand#44|PROMO PLATED STEEL|3|4 +Brand#44|PROMO PLATED STEEL|36|4 +Brand#44|PROMO PLATED STEEL|49|4 +Brand#44|PROMO POLISHED BRASS|3|4 +Brand#44|PROMO POLISHED BRASS|19|4 +Brand#44|PROMO POLISHED COPPER|45|4 +Brand#44|PROMO POLISHED STEEL|36|4 +Brand#44|PROMO POLISHED TIN|9|4 +Brand#44|SMALL ANODIZED COPPER|23|4 +Brand#44|SMALL ANODIZED STEEL|23|4 +Brand#44|SMALL ANODIZED TIN|45|4 +Brand#44|SMALL BRUSHED COPPER|14|4 +Brand#44|SMALL BRUSHED STEEL|45|4 +Brand#44|SMALL BURNISHED COPPER|14|4 +Brand#44|SMALL BURNISHED COPPER|49|4 +Brand#44|SMALL BURNISHED NICKEL|14|4 +Brand#44|SMALL BURNISHED STEEL|23|4 +Brand#44|SMALL BURNISHED TIN|49|4 +Brand#44|SMALL PLATED BRASS|36|4 +Brand#44|SMALL PLATED COPPER|19|4 +Brand#44|SMALL PLATED NICKEL|3|4 +Brand#44|SMALL POLISHED COPPER|3|4 +Brand#44|SMALL POLISHED COPPER|49|4 +Brand#44|SMALL POLISHED STEEL|3|4 +Brand#44|STANDARD ANODIZED BRASS|3|4 +Brand#44|STANDARD ANODIZED COPPER|3|4 +Brand#44|STANDARD ANODIZED NICKEL|3|4 +Brand#44|STANDARD ANODIZED NICKEL|36|4 +Brand#44|STANDARD ANODIZED STEEL|14|4 +Brand#44|STANDARD ANODIZED TIN|3|4 +Brand#44|STANDARD ANODIZED TIN|9|4 +Brand#44|STANDARD ANODIZED TIN|36|4 +Brand#44|STANDARD BRUSHED COPPER|36|4 +Brand#44|STANDARD BRUSHED COPPER|45|4 +Brand#44|STANDARD BRUSHED TIN|9|4 +Brand#44|STANDARD BRUSHED TIN|49|4 +Brand#44|STANDARD BURNISHED COPPER|9|4 +Brand#44|STANDARD BURNISHED STEEL|23|4 +Brand#44|STANDARD PLATED BRASS|14|4 +Brand#44|STANDARD PLATED BRASS|23|4 +Brand#44|STANDARD PLATED BRASS|49|4 +Brand#44|STANDARD PLATED COPPER|14|4 +Brand#44|STANDARD POLISHED NICKEL|19|4 +Brand#44|STANDARD POLISHED TIN|9|4 +Brand#51|ECONOMY ANODIZED BRASS|9|4 +Brand#51|ECONOMY ANODIZED BRASS|23|4 +Brand#51|ECONOMY ANODIZED NICKEL|3|4 +Brand#51|ECONOMY ANODIZED NICKEL|23|4 +Brand#51|ECONOMY ANODIZED STEEL|19|4 +Brand#51|ECONOMY ANODIZED STEEL|23|4 +Brand#51|ECONOMY ANODIZED STEEL|49|4 +Brand#51|ECONOMY BRUSHED BRASS|3|4 +Brand#51|ECONOMY BRUSHED BRASS|49|4 +Brand#51|ECONOMY BRUSHED NICKEL|14|4 +Brand#51|ECONOMY BRUSHED STEEL|45|4 +Brand#51|ECONOMY BRUSHED TIN|36|4 +Brand#51|ECONOMY BURNISHED BRASS|14|4 +Brand#51|ECONOMY BURNISHED COPPER|45|4 +Brand#51|ECONOMY PLATED NICKEL|49|4 +Brand#51|ECONOMY PLATED TIN|36|4 +Brand#51|ECONOMY POLISHED COPPER|9|4 +Brand#51|ECONOMY POLISHED STEEL|14|4 +Brand#51|ECONOMY POLISHED STEEL|49|4 +Brand#51|LARGE ANODIZED COPPER|9|4 +Brand#51|LARGE ANODIZED COPPER|49|4 +Brand#51|LARGE ANODIZED NICKEL|14|4 +Brand#51|LARGE ANODIZED STEEL|36|4 +Brand#51|LARGE BRUSHED NICKEL|3|4 +Brand#51|LARGE BRUSHED NICKEL|9|4 +Brand#51|LARGE BURNISHED BRASS|19|4 +Brand#51|LARGE BURNISHED BRASS|36|4 +Brand#51|LARGE BURNISHED COPPER|14|4 +Brand#51|LARGE BURNISHED NICKEL|14|4 +Brand#51|LARGE PLATED BRASS|36|4 +Brand#51|LARGE POLISHED COPPER|14|4 +Brand#51|LARGE POLISHED NICKEL|23|4 +Brand#51|LARGE POLISHED NICKEL|36|4 +Brand#51|LARGE POLISHED STEEL|19|4 +Brand#51|MEDIUM ANODIZED COPPER|9|4 +Brand#51|MEDIUM ANODIZED STEEL|3|4 +Brand#51|MEDIUM BRUSHED BRASS|36|4 +Brand#51|MEDIUM BRUSHED BRASS|45|4 +Brand#51|MEDIUM BRUSHED STEEL|3|4 +Brand#51|MEDIUM BRUSHED TIN|36|4 +Brand#51|MEDIUM BURNISHED NICKEL|3|4 +Brand#51|MEDIUM BURNISHED NICKEL|36|4 +Brand#51|MEDIUM BURNISHED STEEL|14|4 +Brand#51|MEDIUM BURNISHED TIN|9|4 +Brand#51|MEDIUM PLATED STEEL|19|4 +Brand#51|MEDIUM PLATED TIN|3|4 +Brand#51|PROMO ANODIZED NICKEL|14|4 +Brand#51|PROMO ANODIZED STEEL|23|4 +Brand#51|PROMO ANODIZED TIN|19|4 +Brand#51|PROMO BRUSHED BRASS|23|4 +Brand#51|PROMO BRUSHED COPPER|45|4 +Brand#51|PROMO BRUSHED STEEL|45|4 +Brand#51|PROMO BRUSHED TIN|9|4 +Brand#51|PROMO BURNISHED BRASS|19|4 +Brand#51|PROMO BURNISHED BRASS|23|4 +Brand#51|PROMO BURNISHED NICKEL|14|4 +Brand#51|PROMO PLATED BRASS|3|4 +Brand#51|PROMO PLATED BRASS|23|4 +Brand#51|PROMO PLATED TIN|19|4 +Brand#51|PROMO PLATED TIN|23|4 +Brand#51|PROMO POLISHED BRASS|23|4 +Brand#51|PROMO POLISHED COPPER|9|4 +Brand#51|PROMO POLISHED NICKEL|9|4 +Brand#51|PROMO POLISHED STEEL|49|4 +Brand#51|SMALL ANODIZED STEEL|14|4 +Brand#51|SMALL BRUSHED BRASS|23|4 +Brand#51|SMALL BRUSHED TIN|19|4 +Brand#51|SMALL BURNISHED NICKEL|23|4 +Brand#51|SMALL PLATED COPPER|49|4 +Brand#51|SMALL PLATED NICKEL|3|4 +Brand#51|SMALL PLATED NICKEL|14|4 +Brand#51|SMALL PLATED STEEL|45|4 +Brand#51|SMALL POLISHED NICKEL|14|4 +Brand#51|SMALL POLISHED NICKEL|23|4 +Brand#51|SMALL POLISHED STEEL|3|4 +Brand#51|SMALL POLISHED STEEL|19|4 +Brand#51|SMALL POLISHED STEEL|49|4 +Brand#51|STANDARD ANODIZED NICKEL|3|4 +Brand#51|STANDARD ANODIZED NICKEL|49|4 +Brand#51|STANDARD BRUSHED BRASS|3|4 +Brand#51|STANDARD BRUSHED COPPER|3|4 +Brand#51|STANDARD BRUSHED NICKEL|19|4 +Brand#51|STANDARD BRUSHED STEEL|36|4 +Brand#51|STANDARD BURNISHED COPPER|19|4 +Brand#51|STANDARD BURNISHED NICKEL|49|4 +Brand#51|STANDARD BURNISHED STEEL|23|4 +Brand#51|STANDARD BURNISHED STEEL|36|4 +Brand#51|STANDARD BURNISHED TIN|45|4 +Brand#51|STANDARD PLATED BRASS|36|4 +Brand#51|STANDARD PLATED BRASS|49|4 +Brand#51|STANDARD PLATED COPPER|14|4 +Brand#51|STANDARD PLATED COPPER|23|4 +Brand#51|STANDARD POLISHED BRASS|14|4 +Brand#51|STANDARD POLISHED BRASS|45|4 +Brand#51|STANDARD POLISHED STEEL|36|4 +Brand#51|STANDARD POLISHED STEEL|49|4 +Brand#51|STANDARD POLISHED TIN|45|4 +Brand#52|ECONOMY ANODIZED BRASS|14|4 +Brand#52|ECONOMY ANODIZED BRASS|23|4 +Brand#52|ECONOMY ANODIZED COPPER|36|4 +Brand#52|ECONOMY ANODIZED NICKEL|49|4 +Brand#52|ECONOMY ANODIZED STEEL|19|4 +Brand#52|ECONOMY BRUSHED COPPER|49|4 +Brand#52|ECONOMY BURNISHED BRASS|36|4 +Brand#52|ECONOMY BURNISHED COPPER|19|4 +Brand#52|ECONOMY BURNISHED COPPER|45|4 +Brand#52|ECONOMY BURNISHED NICKEL|19|4 +Brand#52|ECONOMY BURNISHED STEEL|36|4 +Brand#52|ECONOMY PLATED TIN|14|4 +Brand#52|ECONOMY PLATED TIN|23|4 +Brand#52|ECONOMY POLISHED BRASS|23|4 +Brand#52|ECONOMY POLISHED BRASS|45|4 +Brand#52|ECONOMY POLISHED NICKEL|36|4 +Brand#52|ECONOMY POLISHED STEEL|49|4 +Brand#52|LARGE ANODIZED COPPER|14|4 +Brand#52|LARGE ANODIZED NICKEL|3|4 +Brand#52|LARGE ANODIZED NICKEL|45|4 +Brand#52|LARGE ANODIZED TIN|45|4 +Brand#52|LARGE BRUSHED COPPER|19|4 +Brand#52|LARGE BRUSHED NICKEL|3|4 +Brand#52|LARGE BRUSHED NICKEL|19|4 +Brand#52|LARGE BRUSHED NICKEL|23|4 +Brand#52|LARGE BRUSHED STEEL|49|4 +Brand#52|LARGE BRUSHED TIN|14|4 +Brand#52|LARGE BURNISHED NICKEL|9|4 +Brand#52|LARGE BURNISHED TIN|23|4 +Brand#52|LARGE BURNISHED TIN|45|4 +Brand#52|LARGE PLATED BRASS|14|4 +Brand#52|LARGE PLATED COPPER|14|4 +Brand#52|LARGE PLATED COPPER|19|4 +Brand#52|LARGE PLATED NICKEL|45|4 +Brand#52|LARGE PLATED STEEL|9|4 +Brand#52|LARGE PLATED TIN|9|4 +Brand#52|LARGE POLISHED NICKEL|19|4 +Brand#52|LARGE POLISHED NICKEL|23|4 +Brand#52|LARGE POLISHED NICKEL|36|4 +Brand#52|LARGE POLISHED TIN|9|4 +Brand#52|MEDIUM ANODIZED COPPER|36|4 +Brand#52|MEDIUM ANODIZED STEEL|14|4 +Brand#52|MEDIUM ANODIZED TIN|3|4 +Brand#52|MEDIUM ANODIZED TIN|49|4 +Brand#52|MEDIUM BRUSHED COPPER|9|4 +Brand#52|MEDIUM BRUSHED NICKEL|9|4 +Brand#52|MEDIUM BRUSHED STEEL|23|4 +Brand#52|MEDIUM BRUSHED STEEL|49|4 +Brand#52|MEDIUM BURNISHED STEEL|23|4 +Brand#52|MEDIUM BURNISHED TIN|45|4 +Brand#52|MEDIUM BURNISHED TIN|49|4 +Brand#52|MEDIUM PLATED BRASS|36|4 +Brand#52|MEDIUM PLATED STEEL|9|4 +Brand#52|MEDIUM PLATED STEEL|49|4 +Brand#52|MEDIUM PLATED TIN|9|4 +Brand#52|MEDIUM PLATED TIN|49|4 +Brand#52|PROMO ANODIZED BRASS|9|4 +Brand#52|PROMO ANODIZED BRASS|23|4 +Brand#52|PROMO ANODIZED BRASS|36|4 +Brand#52|PROMO ANODIZED NICKEL|45|4 +Brand#52|PROMO ANODIZED STEEL|36|4 +Brand#52|PROMO BRUSHED COPPER|3|4 +Brand#52|PROMO BRUSHED NICKEL|3|4 +Brand#52|PROMO BRUSHED NICKEL|49|4 +Brand#52|PROMO BRUSHED STEEL|14|4 +Brand#52|PROMO BRUSHED TIN|3|4 +Brand#52|PROMO BRUSHED TIN|19|4 +Brand#52|PROMO BRUSHED TIN|36|4 +Brand#52|PROMO BURNISHED COPPER|49|4 +Brand#52|PROMO BURNISHED NICKEL|9|4 +Brand#52|PROMO BURNISHED STEEL|9|4 +Brand#52|PROMO BURNISHED STEEL|23|4 +Brand#52|PROMO BURNISHED TIN|19|4 +Brand#52|PROMO BURNISHED TIN|36|4 +Brand#52|PROMO PLATED BRASS|19|4 +Brand#52|PROMO PLATED BRASS|45|4 +Brand#52|PROMO PLATED BRASS|49|4 +Brand#52|PROMO PLATED COPPER|9|4 +Brand#52|PROMO PLATED NICKEL|3|4 +Brand#52|PROMO PLATED NICKEL|23|4 +Brand#52|PROMO POLISHED NICKEL|14|4 +Brand#52|PROMO POLISHED NICKEL|49|4 +Brand#52|PROMO POLISHED TIN|36|4 +Brand#52|SMALL ANODIZED BRASS|3|4 +Brand#52|SMALL ANODIZED BRASS|14|4 +Brand#52|SMALL ANODIZED COPPER|3|4 +Brand#52|SMALL ANODIZED NICKEL|36|4 +Brand#52|SMALL ANODIZED STEEL|9|4 +Brand#52|SMALL ANODIZED STEEL|19|4 +Brand#52|SMALL BRUSHED NICKEL|19|4 +Brand#52|SMALL BRUSHED STEEL|23|4 +Brand#52|SMALL BRUSHED TIN|14|4 +Brand#52|SMALL BRUSHED TIN|19|4 +Brand#52|SMALL BURNISHED NICKEL|14|4 +Brand#52|SMALL BURNISHED NICKEL|49|4 +Brand#52|SMALL BURNISHED TIN|9|4 +Brand#52|SMALL POLISHED BRASS|36|4 +Brand#52|SMALL POLISHED BRASS|49|4 +Brand#52|SMALL POLISHED TIN|45|4 +Brand#52|STANDARD ANODIZED BRASS|45|4 +Brand#52|STANDARD BRUSHED BRASS|23|4 +Brand#52|STANDARD BRUSHED COPPER|14|4 +Brand#52|STANDARD BRUSHED TIN|36|4 +Brand#52|STANDARD BURNISHED BRASS|49|4 +Brand#52|STANDARD BURNISHED STEEL|19|4 +Brand#52|STANDARD BURNISHED TIN|9|4 +Brand#52|STANDARD BURNISHED TIN|19|4 +Brand#52|STANDARD PLATED NICKEL|36|4 +Brand#52|STANDARD PLATED STEEL|36|4 +Brand#52|STANDARD POLISHED BRASS|36|4 +Brand#52|STANDARD POLISHED COPPER|45|4 +Brand#52|STANDARD POLISHED STEEL|19|4 +Brand#52|STANDARD POLISHED TIN|19|4 +Brand#53|ECONOMY ANODIZED BRASS|45|4 +Brand#53|ECONOMY ANODIZED COPPER|9|4 +Brand#53|ECONOMY ANODIZED NICKEL|3|4 +Brand#53|ECONOMY ANODIZED NICKEL|19|4 +Brand#53|ECONOMY ANODIZED STEEL|45|4 +Brand#53|ECONOMY ANODIZED TIN|14|4 +Brand#53|ECONOMY ANODIZED TIN|36|4 +Brand#53|ECONOMY BRUSHED TIN|45|4 +Brand#53|ECONOMY BURNISHED BRASS|14|4 +Brand#53|ECONOMY BURNISHED COPPER|45|4 +Brand#53|ECONOMY BURNISHED NICKEL|3|4 +Brand#53|ECONOMY BURNISHED NICKEL|49|4 +Brand#53|ECONOMY BURNISHED TIN|45|4 +Brand#53|ECONOMY PLATED BRASS|3|4 +Brand#53|ECONOMY PLATED NICKEL|14|4 +Brand#53|ECONOMY PLATED STEEL|23|4 +Brand#53|ECONOMY PLATED STEEL|36|4 +Brand#53|ECONOMY POLISHED TIN|36|4 +Brand#53|LARGE ANODIZED NICKEL|49|4 +Brand#53|LARGE ANODIZED STEEL|19|4 +Brand#53|LARGE BRUSHED COPPER|3|4 +Brand#53|LARGE BRUSHED COPPER|14|4 +Brand#53|LARGE BRUSHED NICKEL|23|4 +Brand#53|LARGE BRUSHED NICKEL|36|4 +Brand#53|LARGE BRUSHED TIN|36|4 +Brand#53|LARGE BURNISHED BRASS|45|4 +Brand#53|LARGE BURNISHED COPPER|19|4 +Brand#53|LARGE BURNISHED COPPER|36|4 +Brand#53|LARGE BURNISHED NICKEL|23|4 +Brand#53|LARGE BURNISHED STEEL|19|4 +Brand#53|LARGE BURNISHED STEEL|23|4 +Brand#53|LARGE PLATED BRASS|9|4 +Brand#53|LARGE PLATED BRASS|45|4 +Brand#53|LARGE PLATED BRASS|49|4 +Brand#53|LARGE PLATED COPPER|23|4 +Brand#53|LARGE PLATED NICKEL|23|4 +Brand#53|LARGE PLATED NICKEL|49|4 +Brand#53|LARGE PLATED STEEL|49|4 +Brand#53|LARGE PLATED TIN|14|4 +Brand#53|LARGE POLISHED COPPER|49|4 +Brand#53|LARGE POLISHED STEEL|36|4 +Brand#53|LARGE POLISHED TIN|9|4 +Brand#53|MEDIUM ANODIZED BRASS|23|4 +Brand#53|MEDIUM ANODIZED STEEL|14|4 +Brand#53|MEDIUM ANODIZED STEEL|36|4 +Brand#53|MEDIUM ANODIZED TIN|3|4 +Brand#53|MEDIUM ANODIZED TIN|9|4 +Brand#53|MEDIUM BRUSHED BRASS|3|4 +Brand#53|MEDIUM BRUSHED COPPER|3|4 +Brand#53|MEDIUM BRUSHED NICKEL|14|4 +Brand#53|MEDIUM BRUSHED NICKEL|36|4 +Brand#53|MEDIUM BRUSHED NICKEL|49|4 +Brand#53|MEDIUM BRUSHED STEEL|45|4 +Brand#53|MEDIUM BURNISHED BRASS|3|4 +Brand#53|MEDIUM BURNISHED BRASS|36|4 +Brand#53|MEDIUM BURNISHED TIN|9|4 +Brand#53|MEDIUM BURNISHED TIN|14|4 +Brand#53|MEDIUM BURNISHED TIN|36|4 +Brand#53|MEDIUM PLATED BRASS|23|4 +Brand#53|MEDIUM PLATED COPPER|14|4 +Brand#53|MEDIUM PLATED NICKEL|45|4 +Brand#53|MEDIUM PLATED TIN|19|4 +Brand#53|MEDIUM PLATED TIN|45|4 +Brand#53|PROMO ANODIZED BRASS|36|4 +Brand#53|PROMO ANODIZED NICKEL|3|4 +Brand#53|PROMO ANODIZED NICKEL|19|4 +Brand#53|PROMO BRUSHED BRASS|45|4 +Brand#53|PROMO BRUSHED COPPER|3|4 +Brand#53|PROMO BRUSHED COPPER|23|4 +Brand#53|PROMO BRUSHED COPPER|45|4 +Brand#53|PROMO BURNISHED BRASS|23|4 +Brand#53|PROMO BURNISHED BRASS|36|4 +Brand#53|PROMO BURNISHED NICKEL|23|4 +Brand#53|PROMO BURNISHED STEEL|23|4 +Brand#53|PROMO BURNISHED STEEL|49|4 +Brand#53|PROMO PLATED TIN|19|4 +Brand#53|PROMO PLATED TIN|23|4 +Brand#53|PROMO PLATED TIN|36|4 +Brand#53|PROMO POLISHED STEEL|23|4 +Brand#53|PROMO POLISHED TIN|3|4 +Brand#53|SMALL ANODIZED COPPER|23|4 +Brand#53|SMALL ANODIZED COPPER|36|4 +Brand#53|SMALL ANODIZED COPPER|49|4 +Brand#53|SMALL ANODIZED NICKEL|36|4 +Brand#53|SMALL BRUSHED BRASS|36|4 +Brand#53|SMALL BRUSHED COPPER|3|4 +Brand#53|SMALL BRUSHED TIN|3|4 +Brand#53|SMALL BRUSHED TIN|36|4 +Brand#53|SMALL BURNISHED BRASS|9|4 +Brand#53|SMALL BURNISHED BRASS|49|4 +Brand#53|SMALL BURNISHED COPPER|19|4 +Brand#53|SMALL BURNISHED COPPER|45|4 +Brand#53|SMALL PLATED BRASS|9|4 +Brand#53|SMALL PLATED COPPER|3|4 +Brand#53|SMALL PLATED NICKEL|14|4 +Brand#53|SMALL POLISHED NICKEL|19|4 +Brand#53|SMALL POLISHED STEEL|36|4 +Brand#53|SMALL POLISHED TIN|23|4 +Brand#53|STANDARD ANODIZED BRASS|14|4 +Brand#53|STANDARD ANODIZED NICKEL|9|4 +Brand#53|STANDARD ANODIZED NICKEL|23|4 +Brand#53|STANDARD ANODIZED NICKEL|45|4 +Brand#53|STANDARD ANODIZED STEEL|45|4 +Brand#53|STANDARD BRUSHED COPPER|3|4 +Brand#53|STANDARD BRUSHED NICKEL|23|4 +Brand#53|STANDARD BRUSHED TIN|14|4 +Brand#53|STANDARD BURNISHED NICKEL|49|4 +Brand#53|STANDARD BURNISHED STEEL|9|4 +Brand#53|STANDARD PLATED BRASS|36|4 +Brand#53|STANDARD PLATED COPPER|45|4 +Brand#53|STANDARD PLATED NICKEL|36|4 +Brand#53|STANDARD PLATED STEEL|3|4 +Brand#53|STANDARD PLATED STEEL|49|4 +Brand#53|STANDARD PLATED TIN|23|4 +Brand#53|STANDARD POLISHED STEEL|3|4 +Brand#54|ECONOMY ANODIZED BRASS|9|4 +Brand#54|ECONOMY ANODIZED BRASS|45|4 +Brand#54|ECONOMY ANODIZED COPPER|9|4 +Brand#54|ECONOMY ANODIZED STEEL|19|4 +Brand#54|ECONOMY BRUSHED BRASS|45|4 +Brand#54|ECONOMY BRUSHED NICKEL|19|4 +Brand#54|ECONOMY BRUSHED STEEL|3|4 +Brand#54|ECONOMY BRUSHED TIN|19|4 +Brand#54|ECONOMY BURNISHED BRASS|45|4 +Brand#54|ECONOMY BURNISHED COPPER|14|4 +Brand#54|ECONOMY BURNISHED NICKEL|9|4 +Brand#54|ECONOMY BURNISHED NICKEL|36|4 +Brand#54|ECONOMY BURNISHED STEEL|36|4 +Brand#54|ECONOMY BURNISHED TIN|9|4 +Brand#54|ECONOMY BURNISHED TIN|14|4 +Brand#54|ECONOMY BURNISHED TIN|23|4 +Brand#54|ECONOMY PLATED TIN|23|4 +Brand#54|ECONOMY POLISHED BRASS|9|4 +Brand#54|ECONOMY POLISHED BRASS|19|4 +Brand#54|ECONOMY POLISHED COPPER|23|4 +Brand#54|ECONOMY POLISHED STEEL|23|4 +Brand#54|ECONOMY POLISHED TIN|3|4 +Brand#54|LARGE ANODIZED BRASS|14|4 +Brand#54|LARGE ANODIZED BRASS|49|4 +Brand#54|LARGE ANODIZED TIN|9|4 +Brand#54|LARGE BRUSHED BRASS|14|4 +Brand#54|LARGE BRUSHED STEEL|9|4 +Brand#54|LARGE BRUSHED STEEL|23|4 +Brand#54|LARGE BRUSHED TIN|14|4 +Brand#54|LARGE BURNISHED BRASS|49|4 +Brand#54|LARGE BURNISHED COPPER|19|4 +Brand#54|LARGE BURNISHED NICKEL|14|4 +Brand#54|LARGE BURNISHED TIN|14|4 +Brand#54|LARGE PLATED BRASS|19|4 +Brand#54|LARGE PLATED BRASS|23|4 +Brand#54|LARGE POLISHED BRASS|19|4 +Brand#54|LARGE POLISHED BRASS|23|4 +Brand#54|LARGE POLISHED NICKEL|3|4 +Brand#54|LARGE POLISHED NICKEL|14|4 +Brand#54|LARGE POLISHED STEEL|19|4 +Brand#54|LARGE POLISHED TIN|3|4 +Brand#54|LARGE POLISHED TIN|9|4 +Brand#54|LARGE POLISHED TIN|36|4 +Brand#54|MEDIUM ANODIZED NICKEL|9|4 +Brand#54|MEDIUM ANODIZED NICKEL|14|4 +Brand#54|MEDIUM ANODIZED NICKEL|36|4 +Brand#54|MEDIUM BRUSHED NICKEL|9|4 +Brand#54|MEDIUM BRUSHED NICKEL|19|4 +Brand#54|MEDIUM BURNISHED STEEL|3|4 +Brand#54|MEDIUM BURNISHED STEEL|19|4 +Brand#54|MEDIUM BURNISHED STEEL|23|4 +Brand#54|MEDIUM PLATED BRASS|3|4 +Brand#54|MEDIUM PLATED NICKEL|45|4 +Brand#54|PROMO ANODIZED NICKEL|45|4 +Brand#54|PROMO BRUSHED BRASS|3|4 +Brand#54|PROMO BRUSHED STEEL|23|4 +Brand#54|PROMO BRUSHED TIN|14|4 +Brand#54|PROMO BURNISHED COPPER|49|4 +Brand#54|PROMO BURNISHED TIN|9|4 +Brand#54|PROMO PLATED BRASS|14|4 +Brand#54|PROMO PLATED NICKEL|3|4 +Brand#54|PROMO PLATED STEEL|19|4 +Brand#54|PROMO PLATED TIN|23|4 +Brand#54|PROMO PLATED TIN|49|4 +Brand#54|PROMO POLISHED BRASS|3|4 +Brand#54|PROMO POLISHED NICKEL|9|4 +Brand#54|PROMO POLISHED TIN|49|4 +Brand#54|SMALL ANODIZED COPPER|49|4 +Brand#54|SMALL ANODIZED NICKEL|9|4 +Brand#54|SMALL ANODIZED NICKEL|36|4 +Brand#54|SMALL ANODIZED TIN|19|4 +Brand#54|SMALL BRUSHED BRASS|14|4 +Brand#54|SMALL BRUSHED BRASS|19|4 +Brand#54|SMALL BRUSHED BRASS|36|4 +Brand#54|SMALL BRUSHED COPPER|3|4 +Brand#54|SMALL BRUSHED COPPER|9|4 +Brand#54|SMALL BRUSHED COPPER|19|4 +Brand#54|SMALL BRUSHED TIN|9|4 +Brand#54|SMALL BRUSHED TIN|36|4 +Brand#54|SMALL BURNISHED COPPER|9|4 +Brand#54|SMALL BURNISHED COPPER|36|4 +Brand#54|SMALL BURNISHED STEEL|14|4 +Brand#54|SMALL BURNISHED STEEL|19|4 +Brand#54|SMALL BURNISHED TIN|9|4 +Brand#54|SMALL BURNISHED TIN|36|4 +Brand#54|SMALL PLATED BRASS|23|4 +Brand#54|SMALL PLATED COPPER|9|4 +Brand#54|SMALL PLATED COPPER|36|4 +Brand#54|SMALL PLATED COPPER|49|4 +Brand#54|SMALL PLATED NICKEL|9|4 +Brand#54|SMALL PLATED TIN|23|4 +Brand#54|SMALL PLATED TIN|36|4 +Brand#54|SMALL POLISHED BRASS|9|4 +Brand#54|SMALL POLISHED COPPER|9|4 +Brand#54|SMALL POLISHED TIN|9|4 +Brand#54|STANDARD ANODIZED BRASS|3|4 +Brand#54|STANDARD ANODIZED BRASS|9|4 +Brand#54|STANDARD ANODIZED COPPER|3|4 +Brand#54|STANDARD ANODIZED TIN|3|4 +Brand#54|STANDARD BRUSHED COPPER|3|4 +Brand#54|STANDARD BRUSHED NICKEL|45|4 +Brand#54|STANDARD BRUSHED TIN|36|4 +Brand#54|STANDARD BURNISHED BRASS|23|4 +Brand#54|STANDARD BURNISHED BRASS|49|4 +Brand#54|STANDARD BURNISHED COPPER|19|4 +Brand#54|STANDARD BURNISHED NICKEL|23|4 +Brand#54|STANDARD BURNISHED STEEL|45|4 +Brand#54|STANDARD PLATED BRASS|3|4 +Brand#54|STANDARD PLATED BRASS|45|4 +Brand#54|STANDARD PLATED BRASS|49|4 +Brand#54|STANDARD PLATED STEEL|3|4 +Brand#54|STANDARD POLISHED BRASS|36|4 +Brand#54|STANDARD POLISHED STEEL|3|4 +Brand#54|STANDARD POLISHED STEEL|14|4 +Brand#54|STANDARD POLISHED STEEL|45|4 +Brand#55|ECONOMY ANODIZED BRASS|3|4 +Brand#55|ECONOMY BRUSHED BRASS|19|4 +Brand#55|ECONOMY BRUSHED COPPER|9|4 +Brand#55|ECONOMY BRUSHED COPPER|23|4 +Brand#55|ECONOMY BRUSHED COPPER|45|4 +Brand#55|ECONOMY BRUSHED STEEL|23|4 +Brand#55|ECONOMY BURNISHED NICKEL|36|4 +Brand#55|ECONOMY BURNISHED NICKEL|45|4 +Brand#55|ECONOMY BURNISHED TIN|45|4 +Brand#55|ECONOMY PLATED NICKEL|19|4 +Brand#55|ECONOMY POLISHED NICKEL|9|4 +Brand#55|LARGE BRUSHED BRASS|23|4 +Brand#55|LARGE BRUSHED BRASS|45|4 +Brand#55|LARGE BRUSHED COPPER|49|4 +Brand#55|LARGE BRUSHED NICKEL|9|4 +Brand#55|LARGE BRUSHED NICKEL|14|4 +Brand#55|LARGE BURNISHED BRASS|3|4 +Brand#55|LARGE BURNISHED COPPER|14|4 +Brand#55|LARGE BURNISHED COPPER|36|4 +Brand#55|LARGE PLATED BRASS|45|4 +Brand#55|LARGE PLATED COPPER|19|4 +Brand#55|LARGE PLATED NICKEL|9|4 +Brand#55|LARGE PLATED STEEL|9|4 +Brand#55|LARGE PLATED TIN|9|4 +Brand#55|LARGE PLATED TIN|14|4 +Brand#55|LARGE PLATED TIN|23|4 +Brand#55|LARGE POLISHED NICKEL|3|4 +Brand#55|LARGE POLISHED STEEL|36|4 +Brand#55|LARGE POLISHED STEEL|45|4 +Brand#55|MEDIUM ANODIZED COPPER|9|4 +Brand#55|MEDIUM BRUSHED BRASS|3|4 +Brand#55|MEDIUM BRUSHED NICKEL|23|4 +Brand#55|MEDIUM BRUSHED TIN|45|4 +Brand#55|MEDIUM BURNISHED BRASS|23|4 +Brand#55|MEDIUM BURNISHED COPPER|36|4 +Brand#55|MEDIUM BURNISHED NICKEL|3|4 +Brand#55|MEDIUM BURNISHED STEEL|14|4 +Brand#55|MEDIUM BURNISHED STEEL|36|4 +Brand#55|MEDIUM PLATED NICKEL|23|4 +Brand#55|PROMO ANODIZED COPPER|14|4 +Brand#55|PROMO ANODIZED COPPER|49|4 +Brand#55|PROMO ANODIZED STEEL|36|4 +Brand#55|PROMO ANODIZED TIN|23|4 +Brand#55|PROMO BRUSHED NICKEL|36|4 +Brand#55|PROMO BRUSHED STEEL|3|4 +Brand#55|PROMO BRUSHED STEEL|36|4 +Brand#55|PROMO BRUSHED TIN|9|4 +Brand#55|PROMO BURNISHED COPPER|3|4 +Brand#55|PROMO BURNISHED STEEL|14|4 +Brand#55|PROMO BURNISHED TIN|23|4 +Brand#55|PROMO BURNISHED TIN|49|4 +Brand#55|PROMO PLATED COPPER|3|4 +Brand#55|PROMO PLATED NICKEL|3|4 +Brand#55|PROMO PLATED NICKEL|14|4 +Brand#55|PROMO PLATED NICKEL|23|4 +Brand#55|PROMO PLATED TIN|3|4 +Brand#55|PROMO POLISHED COPPER|3|4 +Brand#55|SMALL ANODIZED BRASS|19|4 +Brand#55|SMALL ANODIZED NICKEL|45|4 +Brand#55|SMALL BRUSHED COPPER|14|4 +Brand#55|SMALL BRUSHED COPPER|45|4 +Brand#55|SMALL BURNISHED BRASS|14|4 +Brand#55|SMALL BURNISHED TIN|3|4 +Brand#55|SMALL BURNISHED TIN|49|4 +Brand#55|SMALL PLATED BRASS|45|4 +Brand#55|SMALL PLATED COPPER|23|4 +Brand#55|SMALL PLATED COPPER|36|4 +Brand#55|SMALL PLATED COPPER|45|4 +Brand#55|SMALL PLATED COPPER|49|4 +Brand#55|SMALL PLATED NICKEL|9|4 +Brand#55|SMALL PLATED STEEL|9|4 +Brand#55|SMALL PLATED TIN|14|4 +Brand#55|SMALL PLATED TIN|36|4 +Brand#55|SMALL POLISHED NICKEL|45|4 +Brand#55|SMALL POLISHED STEEL|19|4 +Brand#55|SMALL POLISHED TIN|19|4 +Brand#55|STANDARD ANODIZED BRASS|36|4 +Brand#55|STANDARD ANODIZED BRASS|49|4 +Brand#55|STANDARD ANODIZED STEEL|19|4 +Brand#55|STANDARD ANODIZED TIN|36|4 +Brand#55|STANDARD ANODIZED TIN|49|4 +Brand#55|STANDARD BRUSHED BRASS|36|4 +Brand#55|STANDARD BRUSHED COPPER|3|4 +Brand#55|STANDARD BRUSHED COPPER|9|4 +Brand#55|STANDARD BRUSHED COPPER|23|4 +Brand#55|STANDARD BRUSHED STEEL|19|4 +Brand#55|STANDARD BRUSHED TIN|23|4 +Brand#55|STANDARD BRUSHED TIN|45|4 +Brand#55|STANDARD BURNISHED BRASS|19|4 +Brand#55|STANDARD BURNISHED NICKEL|3|4 +Brand#55|STANDARD BURNISHED NICKEL|36|4 +Brand#55|STANDARD BURNISHED STEEL|19|4 +Brand#55|STANDARD PLATED BRASS|23|4 +Brand#55|STANDARD PLATED NICKEL|9|4 +Brand#55|STANDARD PLATED TIN|36|4 +Brand#55|STANDARD POLISHED BRASS|3|4 +Brand#55|STANDARD POLISHED BRASS|49|4 +Brand#55|STANDARD POLISHED COPPER|19|4 +Brand#55|STANDARD POLISHED COPPER|36|4 +Brand#55|STANDARD POLISHED NICKEL|14|4 +Brand#55|STANDARD POLISHED STEEL|9|4 +Brand#55|STANDARD POLISHED STEEL|36|4 +Brand#12|LARGE BURNISHED NICKEL|14|3 +Brand#12|PROMO POLISHED TIN|3|3 +Brand#21|MEDIUM ANODIZED TIN|9|3 +Brand#22|PROMO BRUSHED BRASS|19|3 +Brand#22|PROMO BURNISHED COPPER|14|3 +Brand#43|STANDARD BRUSHED BRASS|23|3 +Brand#44|MEDIUM ANODIZED NICKEL|9|3 +Brand#53|MEDIUM BURNISHED BRASS|49|3 From 5d54b1c3d8283417b31edceb1daf36a7609e28ef Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 27 May 2024 10:37:15 +0800 Subject: [PATCH 157/229] enable pushdown --- .github/workflows/polystore-benchmark-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index fb9180e817..1201487789 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -66,6 +66,7 @@ jobs: with: DB-name: "Parquet" Set-Filter-Fragment-OFF: "true" + Push-Down: "true" # start udf path test first to avoid being effected - name: Start IGinX @@ -146,6 +147,7 @@ jobs: DB-name: "Parquet" Set-Filter-Fragment-OFF: "true" Root-Dir-Path: "IGinX" + Push-Down: "true" - name: Start IGinX shell: bash From 5f0d68794a04e763a516046359464f265f7d95ab Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 27 May 2024 15:25:27 +0800 Subject: [PATCH 158/229] disable pushdown --- .github/workflows/polystore-benchmark-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 1201487789..fb9180e817 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -66,7 +66,6 @@ jobs: with: DB-name: "Parquet" Set-Filter-Fragment-OFF: "true" - Push-Down: "true" # start udf path test first to avoid being effected - name: Start IGinX @@ -147,7 +146,6 @@ jobs: DB-name: "Parquet" Set-Filter-Fragment-OFF: "true" Root-Dir-Path: "IGinX" - Push-Down: "true" - name: Start IGinX shell: bash From 75f5fa145c6bbfbebc8a1c0c3a4aa7295b697b8a Mon Sep 17 00:00:00 2001 From: Janet731 Date: Mon, 27 May 2024 20:14:41 +0800 Subject: [PATCH 159/229] show os info --- .../workflows/polystore-benchmark-test.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index fb9180e817..80761431b6 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -30,6 +30,32 @@ jobs: python-version: ${{ matrix.python-version }} java: ${{ matrix.java }} + - name: Display System Info + shell: bash + run: | + echo "Operating System: $(uname -a 2>/dev/null || ver)" + echo "Architecture: $(uname -m 2>/dev/null || echo %PROCESSOR_ARCHITECTURE%)" + echo "Java Version:" + java -version + echo "Python Version:" + python --version + echo "CPU Info:" + if [ "$(uname)" = "Linux" ]; then + lscpu + elif [ "$(uname)" = "Darwin" ]; then + sysctl -n machdep.cpu.brand_string + else + wmic cpu get name + fi + echo "Memory Info:" + if [ "$(uname)" = "Linux" ]; then + free -h + elif [ "$(uname)" = "Darwin" ]; then + vm_stat + else + systeminfo | findstr /C:"Total Physical Memory" + fi + - name: generate tpch data shell: bash run: | From 56b94d311edd71b624f2361f32abceb7cc502f2e Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 28 May 2024 01:13:31 +0800 Subject: [PATCH 160/229] add regression standard --- .../workflows/polystore-benchmark-test.yml | 44 +++++++++---------- .../integration/polybench/TPCHRunner.java | 13 +++++- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index 80761431b6..5315c7db62 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -33,28 +33,28 @@ jobs: - name: Display System Info shell: bash run: | - echo "Operating System: $(uname -a 2>/dev/null || ver)" - echo "Architecture: $(uname -m 2>/dev/null || echo %PROCESSOR_ARCHITECTURE%)" - echo "Java Version:" - java -version - echo "Python Version:" - python --version - echo "CPU Info:" - if [ "$(uname)" = "Linux" ]; then - lscpu - elif [ "$(uname)" = "Darwin" ]; then - sysctl -n machdep.cpu.brand_string - else - wmic cpu get name - fi - echo "Memory Info:" - if [ "$(uname)" = "Linux" ]; then - free -h - elif [ "$(uname)" = "Darwin" ]; then - vm_stat - else - systeminfo | findstr /C:"Total Physical Memory" - fi + echo "Operating System: $(uname -a 2>/dev/null || ver)" + echo "Architecture: $(uname -m 2>/dev/null || echo %PROCESSOR_ARCHITECTURE%)" + echo "Java Version:" + java -version + echo "Python Version:" + python --version + echo "CPU Info:" + if [ "$(uname)" = "Linux" ]; then + lscpu + elif [ "$(uname)" = "Darwin" ]; then + sysctl -n machdep.cpu.brand_string + else + wmic cpu get name + fi + echo "Memory Info:" + if [ "$(uname)" = "Linux" ]; then + free -h + elif [ "$(uname)" = "Darwin" ]; then + vm_stat + else + systeminfo | findstr /C:"Total Physical Memory" + fi - name: generate tpch data shell: bash diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 4d1a3cf002..54ff42711b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -221,8 +221,19 @@ public void test() { + ", new average time cost: " + newAvgTimeCosts.get(i) + "ms"); + // TODO 如果相差超过15%？，则报错 + if (Math.abs(newAvgTimeCosts.get(i) - avgTimeCosts.get(i)) > 0.15 * avgTimeCosts.get(i)) { + System.out.println( + "query " + + queryIds.get(i) + + ", new branch average time cost: " + + newAvgTimeCosts.get(i) + + "ms"); + System.out.println( + "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + throw new RuntimeException("time cost not equal"); + } } - // TODO 如果相差超过15%？，则报错 } else { writeToFile(avgTimeCosts, fileName); } From ea72ba1e2383c93fb3c64df3fce7e20e2d129585 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 28 May 2024 13:55:47 +0800 Subject: [PATCH 161/229] add memory profiler --- .../integration/polybench/TPCHRunner.java | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 54ff42711b..ba74b4822b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -81,6 +81,14 @@ public void test() { // 输出所有存储引擎 String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); System.out.println(clusterInfo); + // 获取当前JVM的Runtime实例 + Runtime runtime = Runtime.getRuntime(); + + // 执行垃圾回收，尽量释放内存 + runtime.gc(); + + // 获取执行语句前的内存使用情况 + long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); // 添加存储引擎 System.out.println("start adding storage engine"); @@ -206,23 +214,20 @@ public void test() { System.out.println( "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); } - // write avg time cost to file - for (int i = 0; i < queryIds.size(); i++) { - System.out.println( - "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); - } String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; - if (Files.exists(Paths.get(fileName))) { - List newAvgTimeCosts = readFromFile(fileName); + if (Files.exists(Paths.get(fileName))) { // 如果文件存在，即此次跑的是主分支代码，需要读取文件进行比较 + List newAvgTimeCosts = readFromFile(fileName); // 文件中存的是新分支的运行时间 for (int i = 0; i < queryIds.size(); i++) { + System.out.println( + "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); System.out.println( "query " + queryIds.get(i) + ", new average time cost: " + newAvgTimeCosts.get(i) + "ms"); - // TODO 如果相差超过15%？，则报错 - if (Math.abs(newAvgTimeCosts.get(i) - avgTimeCosts.get(i)) > 0.15 * avgTimeCosts.get(i)) { + // TODO 如果相差超过30%？，则报错 + if (Math.abs(newAvgTimeCosts.get(i) - avgTimeCosts.get(i)) > 0.3 * avgTimeCosts.get(i)) { System.out.println( "query " + queryIds.get(i) @@ -237,6 +242,17 @@ public void test() { } else { writeToFile(avgTimeCosts, fileName); } + // 再次执行垃圾回收 + runtime.gc(); + + // 获取执行语句后的内存使用情况 + long usedMemoryAfter = runtime.totalMemory() - runtime.freeMemory(); + + // 计算内存使用的变化 + long memoryUsed = usedMemoryAfter - usedMemoryBefore; + + // 输出内存使用情况 + System.out.println("Memory used by the statement: " + memoryUsed + " bytes"); // 关闭会话 conn.closeSession(); From a3a2c1aecc3231a3845d452d1c4557577f52b682 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Tue, 28 May 2024 14:29:06 +0800 Subject: [PATCH 162/229] add memory profiler --- .../integration/polybench/TPCHRunner.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index ba74b4822b..0f2962b397 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -138,6 +138,14 @@ public void test() { result = conn.executeSql(sql); result.print(false, ""); } + // 再次执行垃圾回收 + runtime.gc(); + // 获取执行语句后的内存使用情况 + long usedMemoryAfter = runtime.totalMemory() - runtime.freeMemory(); + // 计算内存使用的变化 + long memoryUsed = usedMemoryAfter - usedMemoryBefore; + // 输出内存使用情况 + System.out.println("Memory used by the statement: " + memoryUsed + " bytes"); // 验证 Long timeCost = System.currentTimeMillis() - startTime; @@ -242,17 +250,6 @@ public void test() { } else { writeToFile(avgTimeCosts, fileName); } - // 再次执行垃圾回收 - runtime.gc(); - - // 获取执行语句后的内存使用情况 - long usedMemoryAfter = runtime.totalMemory() - runtime.freeMemory(); - - // 计算内存使用的变化 - long memoryUsed = usedMemoryAfter - usedMemoryBefore; - - // 输出内存使用情况 - System.out.println("Memory used by the statement: " + memoryUsed + " bytes"); // 关闭会话 conn.closeSession(); From 769f45b5fa0a47d2d461b8b177e80984c7aef8a3 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 20 Jun 2024 01:56:28 +0800 Subject: [PATCH 163/229] fix --- .github/actions/confWriter/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 120a6ec313..7bcd6f0338 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -28,7 +28,7 @@ inputs: Root-Dir-Path: description: "the path of IGinX root directory" required: false - default: "${{ inputs.Root-Dir-Path }}" + default: "${GITHUB_WORKSPACE}" runs: using: "composite" # Mandatory parameter From 40421b610f70cf73f5ab1a411604fdae82d8f716 Mon Sep 17 00:00:00 2001 From: Janet731 Date: Thu, 20 Jun 2024 08:39:35 +0800 Subject: [PATCH 164/229] fix --- .github/scripts/benchmarks/parquet.sh | 8 ++++---- .github/scripts/iginx/iginx_regression_udf_path.sh | 8 ++++---- .github/workflows/polystore-benchmark-test.yml | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh index 4b0e10f66d..30d554bb93 100755 --- a/.github/scripts/benchmarks/parquet.sh +++ b/.github/scripts/benchmarks/parquet.sh @@ -29,14 +29,14 @@ cat "$output_file" # 插入数据 COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' -SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" +SCRIPT_COMMAND="bash client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" -bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" +bash -c "chmod +x client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh" if [ "$RUNNER_OS" = "Linux" ]; then bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" + bash -c "client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" fi diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh index dd5ebfb45e..752ba80b2a 100644 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ b/.github/scripts/iginx/iginx_regression_udf_path.sh @@ -4,7 +4,7 @@ cd IGinX pwd set -e -cd core/target/iginx-core-0.6.0-SNAPSHOT/ +cd core/target/iginx-core-0.7.0-SNAPSHOT/ iginx_home_path=$PWD @@ -17,13 +17,13 @@ if [ -n "$MSYSTEM" ]; then export IGINX_HOME=$windows_path - powershell -Command "Start-Process -FilePath 'iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.bat' -NoNewWindow -RedirectStandardOutput '../../iginx-udf.log' -RedirectStandardError '../../iginx-udf-error.log'" + powershell -Command "Start-Process -FilePath 'iginx-core-0.7.0-SNAPSHOT/sbin/start_iginx.bat' -NoNewWindow -RedirectStandardOutput '../../iginx-udf.log' -RedirectStandardError '../../iginx-udf-error.log'" else export IGINX_HOME=$iginx_home_path echo "Iginx home path: $IGINX_HOME" pwd - sh -c "chmod +x iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh" + sh -c "chmod +x iginx-core-0.7.0-SNAPSHOT/sbin/start_iginx.sh" - sh -c "nohup iginx-core-0.6.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &" + sh -c "nohup iginx-core-0.7.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &" fi diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml index a7ef53b6d9..b49ab5e3a4 100644 --- a/.github/workflows/polystore-benchmark-test.yml +++ b/.github/workflows/polystore-benchmark-test.yml @@ -5,7 +5,7 @@ on: types: [opened, reopened] env: - VERSION: 0.6.0-SNAPSHOT + VERSION: 0.7.0-SNAPSHOT concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -192,7 +192,7 @@ jobs: exit 1 fi cd .. - rm ./core/target/iginx-core-0.6.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py + rm ./core/target/iginx-core-0.7.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py chmod +x ".github/scripts/iginx/iginx_regression_udf_path.sh" ".github/scripts/iginx/iginx_regression_udf_path.sh" @@ -235,15 +235,15 @@ jobs: mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" chmod 777 "udtf_extract_year.py" COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' - SCRIPT_COMMAND="bash client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - bash -c "chmod +x client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh" + SCRIPT_COMMAND="bash client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" + bash -c "chmod +x client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh" if [ "$RUNNER_OS" = "Linux" ]; then bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" + bash -c "client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.6.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" + sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" fi - name: Run session test From 98f96523f8f94c5cc49f163af30e3a82f6a4e53f Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Mon, 1 Jul 2024 10:22:23 +0800 Subject: [PATCH 165/229] format --- .../edu/tsinghua/iginx/integration/polybench/TPCHRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 0f2962b397..be7fcac01d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -224,10 +224,10 @@ public void test() { } String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; if (Files.exists(Paths.get(fileName))) { // 如果文件存在，即此次跑的是主分支代码，需要读取文件进行比较 - List newAvgTimeCosts = readFromFile(fileName); // 文件中存的是新分支的运行时间 + List newAvgTimeCosts = readFromFile(fileName); // 文件中存的是新分支的运行时间 for (int i = 0; i < queryIds.size(); i++) { System.out.println( - "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); System.out.println( "query " + queryIds.get(i) From 9ae088facd894747b18b03970991ffce9b644327 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Wed, 3 Jul 2024 08:30:31 +0800 Subject: [PATCH 166/229] increase test number --- .../cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index be7fcac01d..33317c9e28 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -206,7 +206,7 @@ public void test() { SessionExecuteSqlResult result = null; String[] sqls = sqlString.split(";"); List timeCosts = new ArrayList<>(); - for (int i = 0; i < 2; i++) { // TODO: 5 + for (int i = 0; i < 10; i++) { // TODO: 5 startTime = System.currentTimeMillis(); if (sqls.length == 1) // TODO: error From ec45313f8d5ea3b3574599944cb90b3ca80fb42a Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 4 Jul 2024 10:13:49 +0800 Subject: [PATCH 167/229] reduce test number --- .../integration/polybench/TPCHRunner.java | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java index 33317c9e28..59d18f31e8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java @@ -148,7 +148,7 @@ public void test() { System.out.println("Memory used by the statement: " + memoryUsed + " bytes"); // 验证 - Long timeCost = System.currentTimeMillis() - startTime; + long timeCost = System.currentTimeMillis() - startTime; System.out.println("end tpch query, time cost: " + timeCost + "ms"); List> values = result.getValues(); List> answers = @@ -185,7 +185,7 @@ public void test() { new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); String answerString = answers.get(i).get(j); if (!resultString.equals(answerString)) { - System.out.println("Result string： " + resultString); + System.out.println("Result string: " + resultString); System.out.println("Answer string: " + answerString); } assert resultString.equals(answerString); @@ -206,12 +206,14 @@ public void test() { SessionExecuteSqlResult result = null; String[] sqls = sqlString.split(";"); List timeCosts = new ArrayList<>(); - for (int i = 0; i < 10; i++) { // TODO: 5 + for (int i = 0; i < 3; i++) { // TODO: 5 startTime = System.currentTimeMillis(); - if (sqls.length == 1) + if (sqls.length == 1) { // TODO: error System.out.println("wrong input"); - else result = conn.executeSql(sqls[sqls.length - 2] + ";"); + } else { + result = conn.executeSql(sqls[sqls.length - 2] + ";"); + } Long timeCost = System.currentTimeMillis() - startTime; timeCosts.add(timeCost); System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); @@ -226,24 +228,17 @@ public void test() { if (Files.exists(Paths.get(fileName))) { // 如果文件存在，即此次跑的是主分支代码，需要读取文件进行比较 List newAvgTimeCosts = readFromFile(fileName); // 文件中存的是新分支的运行时间 for (int i = 0; i < queryIds.size(); i++) { - System.out.println( - "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); - System.out.println( - "query " - + queryIds.get(i) - + ", new average time cost: " - + newAvgTimeCosts.get(i) - + "ms"); + System.out.printf( + "query %d, old time cost: %2fms\n", queryIds.get(i), avgTimeCosts.get(i)); + System.out.printf( + "query %d, new average time cost: %2fms\n", queryIds.get(i), newAvgTimeCosts.get(i)); // TODO 如果相差超过30%？，则报错 if (Math.abs(newAvgTimeCosts.get(i) - avgTimeCosts.get(i)) > 0.3 * avgTimeCosts.get(i)) { - System.out.println( - "query " - + queryIds.get(i) - + ", new branch average time cost: " - + newAvgTimeCosts.get(i) - + "ms"); - System.out.println( - "query " + queryIds.get(i) + ", average time cost: " + avgTimeCosts.get(i) + "ms"); + System.out.printf( + "query %d, old average time cost: %2fms\n", queryIds.get(i), avgTimeCosts.get(i)); + System.out.printf( + "query %d, new average time cost: %2fms\n", + queryIds.get(i), newAvgTimeCosts.get(i)); throw new RuntimeException("time cost not equal"); } } @@ -253,9 +248,7 @@ public void test() { // 关闭会话 conn.closeSession(); - } catch (SessionException e) { - throw new RuntimeException(e); - } catch (IOException e) { + } catch (SessionException | IOException e) { throw new RuntimeException(e); } } From f2774fb2de10dc9b5384a428968d36da024ea55f Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 17:07:24 +0800 Subject: [PATCH 168/229] use single db storage --- .github/actions/iginxRunner/action.yml | 33 +- .github/scripts/benchmarks/mongodb.sh | 29 -- .github/scripts/benchmarks/parquet.sh | 42 -- .github/scripts/benchmarks/postgres.sh | 70 --- .github/scripts/benchmarks/tpch.sh | 2 +- .../iginx/iginx_regression_udf_path.sh | 29 -- .github/workflows/standard-test-suite.yml | 5 + .github/workflows/tpc-h.yml | 166 +++++++ .../func/sql/tpch/TPCHRegressionIT.java | 443 ++++++++++++++++++ .../polybench/TPCHDataInsertionIT.java | 296 ------------ .../integration/polybench/TPCHRunner.java | 255 ---------- .../test/resources/polybench/queries/q1.sql | 31 -- .../test/resources/polybench/queries/q10.sql | 51 -- .../test/resources/polybench/queries/q13.sql | 25 - .../test/resources/polybench/queries/q16.sql | 49 -- .../test/resources/polybench/queries/q17.sql | 29 -- .../test/resources/polybench/queries/q18.sql | 35 -- .../test/resources/polybench/queries/q19.sql | 48 -- .../test/resources/polybench/queries/q2.sql | 44 -- .../test/resources/polybench/queries/q20.sql | 45 -- .../test/resources/polybench/queries/q3.sql | 31 -- .../test/resources/polybench/queries/q5.sql | 28 -- .../test/resources/polybench/queries/q6.sql | 12 - .../test/resources/polybench/queries/q9.sql | 32 -- .../polybench/udf/udtf_extract_year.py | 28 -- test/src/test/resources/tpch/queries/q1.sql | 31 ++ test/src/test/resources/tpch/queries/q10.sql | 51 ++ test/src/test/resources/tpch/queries/q13.sql | 25 + test/src/test/resources/tpch/queries/q16.sql | 49 ++ test/src/test/resources/tpch/queries/q17.sql | 29 ++ test/src/test/resources/tpch/queries/q18.sql | 35 ++ test/src/test/resources/tpch/queries/q19.sql | 48 ++ test/src/test/resources/tpch/queries/q2.sql | 44 ++ test/src/test/resources/tpch/queries/q20.sql | 45 ++ test/src/test/resources/tpch/queries/q3.sql | 31 ++ test/src/test/resources/tpch/queries/q5.sql | 28 ++ test/src/test/resources/tpch/queries/q6.sql | 12 + test/src/test/resources/tpch/queries/q9.sql | 32 ++ .../{polybench => tpch}/sf0.1/q1.csv | 0 .../{polybench => tpch}/sf0.1/q10.csv | 0 .../{polybench => tpch}/sf0.1/q13.csv | 0 .../{polybench => tpch}/sf0.1/q16.csv | 0 .../{polybench => tpch}/sf0.1/q17.csv | 0 .../{polybench => tpch}/sf0.1/q18.csv | 0 .../{polybench => tpch}/sf0.1/q19.csv | 0 .../{polybench => tpch}/sf0.1/q2.csv | 0 .../{polybench => tpch}/sf0.1/q20.csv | 0 .../{polybench => tpch}/sf0.1/q3.csv | 0 .../{polybench => tpch}/sf0.1/q5.csv | 0 .../{polybench => tpch}/sf0.1/q6.csv | 0 .../{polybench => tpch}/sf0.1/q9.csv | 0 51 files changed, 1094 insertions(+), 1224 deletions(-) delete mode 100644 .github/scripts/benchmarks/mongodb.sh delete mode 100755 .github/scripts/benchmarks/parquet.sh delete mode 100644 .github/scripts/benchmarks/postgres.sh delete mode 100644 .github/scripts/iginx/iginx_regression_udf_path.sh create mode 100644 .github/workflows/tpc-h.yml create mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java delete mode 100644 test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java delete mode 100644 test/src/test/resources/polybench/queries/q1.sql delete mode 100644 test/src/test/resources/polybench/queries/q10.sql delete mode 100644 test/src/test/resources/polybench/queries/q13.sql delete mode 100644 test/src/test/resources/polybench/queries/q16.sql delete mode 100644 test/src/test/resources/polybench/queries/q17.sql delete mode 100644 test/src/test/resources/polybench/queries/q18.sql delete mode 100644 test/src/test/resources/polybench/queries/q19.sql delete mode 100644 test/src/test/resources/polybench/queries/q2.sql delete mode 100644 test/src/test/resources/polybench/queries/q20.sql delete mode 100644 test/src/test/resources/polybench/queries/q3.sql delete mode 100644 test/src/test/resources/polybench/queries/q5.sql delete mode 100644 test/src/test/resources/polybench/queries/q6.sql delete mode 100644 test/src/test/resources/polybench/queries/q9.sql delete mode 100644 test/src/test/resources/polybench/udf/udtf_extract_year.py create mode 100644 test/src/test/resources/tpch/queries/q1.sql create mode 100644 test/src/test/resources/tpch/queries/q10.sql create mode 100644 test/src/test/resources/tpch/queries/q13.sql create mode 100644 test/src/test/resources/tpch/queries/q16.sql create mode 100644 test/src/test/resources/tpch/queries/q17.sql create mode 100644 test/src/test/resources/tpch/queries/q18.sql create mode 100644 test/src/test/resources/tpch/queries/q19.sql create mode 100644 test/src/test/resources/tpch/queries/q2.sql create mode 100644 test/src/test/resources/tpch/queries/q20.sql create mode 100644 test/src/test/resources/tpch/queries/q3.sql create mode 100644 test/src/test/resources/tpch/queries/q5.sql create mode 100644 test/src/test/resources/tpch/queries/q6.sql create mode 100644 test/src/test/resources/tpch/queries/q9.sql rename test/src/test/resources/{polybench => tpch}/sf0.1/q1.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q10.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q13.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q16.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q17.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q18.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q19.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q2.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q20.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q3.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q5.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q6.csv (100%) rename test/src/test/resources/{polybench => tpch}/sf0.1/q9.csv (100%) diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index 859354faf2..d8927c81ca 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -9,6 +9,11 @@ inputs: description: "to test UDF path detection" required: false default: "false" + Root-Dir-Path: + description: "the path of IGinX root directory" + required: false + default: "${GITHUB_WORKSPACE}" + runs: using: "composite" # Mandatory parameter steps: @@ -17,37 +22,37 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 fi - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" ${VERSION} + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_udf_path.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_udf_path.sh" ${VERSION} - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' name: Start IGinX shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx.sh" 6888 6666 elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_windows.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_macos.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' name: Stop IGinX shell: bash run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_kill.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/scripts/benchmarks/mongodb.sh b/.github/scripts/benchmarks/mongodb.sh deleted file mode 100644 index 2a6af66ef7..0000000000 --- a/.github/scripts/benchmarks/mongodb.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -if [ "$RUNNER_OS" = "Linux" ]; then - set -e - sh -c "wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" - sh -c "tar -zxf mongodb-linux-x86_64-ubuntu2204-6.0.4.tgz" - sudo sh -c "cp -r mongodb-linux-x86_64-ubuntu2204-6.0.4/ mongodb-linux-x86_64-ubuntu2204-6.0.4-27017/" - sudo sh -c "cd mongodb-linux-x86_64-ubuntu2204-6.0.4-27017/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port 27017 --dbpath data/db --logpath data/log/mongo.log &" -elif [ "$RUNNER_OS" = "Windows" ]; then - set -e - echo "Downloading zip archive. This may take a few minutes..." - sh -c "curl -LJO https://fastdl.mongodb.org/windows/mongodb-windows-x86_64-6.0.12.zip -o mongodb-windows-x86_64-6.0.12.zip" - sh -c "unzip -qq mongodb-windows-x86_64-6.0.12.zip" - echo "Download finished." - sh -c "ls mongodb-win32-x86_64-windows-6.0.12/bin" - sh -c "cp -r mongodb-win32-x86_64-windows-6.0.12/ mongodb-win32-x86_64-windows-6.0.12-27017/" - sh -c "cd mongodb-win32-x86_64-windows-6.0.12-27017/; mkdir -p data/db; mkdir -p logs; " - filePrefix="mongodb-win32-x86_64-windows-6.0.12-27017" - arguments="-ArgumentList '--port', '27017', '--dbpath', '$filePrefix/data/db', '--logpath', '$filePrefix/logs/db.log'" - powershell -command "Start-Process -FilePath '$filePrefix/bin/mongod' $arguments" -elif [ "$RUNNER_OS" = "macOS" ]; then - set -e - sh -c "wget https://fastdl.mongodb.org/osx/mongodb-macos-x86_64-6.0.4.tgz" - sh -c "tar -zxf mongodb-macos-x86_64-6.0.4.tgz" - sudo sh -c "cp -r mongodb-macos-x86_64-6.0.4/ mongodb-macos-x86_64-6.0.4-27017/" - sudo sh -c "cd mongodb-macos-x86_64-6.0.4-27017/; mkdir -p data/db; mkdir -p data/log; nohup ./bin/mongod --port 27017 --dbpath data/db --logpath data/log/mongo.log &" -else - echo "$RUNNER_OS is not supported" - exit 1 -fi \ No newline at end of file diff --git a/.github/scripts/benchmarks/parquet.sh b/.github/scripts/benchmarks/parquet.sh deleted file mode 100755 index 30d554bb93..0000000000 --- a/.github/scripts/benchmarks/parquet.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# 读取 tbl 文件 -input_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.tbl" -output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - -# 如果文件已存在，则删除 -if [ -f "$output_file" ]; then - rm "$output_file" -fi - -# 处理每一行数据 -line_number=1 -while IFS='|' read -r fields; do - # 将 "|" 分隔符替换为 "," - fields=$(echo "$fields" | tr '|' ',') - # 删除每行末尾的逗号 - fields=$(echo "$fields" | sed 's/,$//') - # 添加行号并将处理后的数据写入 CSV 文件 - echo "$line_number,$fields" >> "$output_file" - ((line_number++)) -done < "$input_file" - -cat "$output_file" - -echo "Conversion completed. CSV file: $output_file" - -cat "$output_file" - -# 插入数据 -COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment);CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "test/src/test/resources/polybench/udf/udtf_extract_year.py";' -SCRIPT_COMMAND="bash client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - -bash -c "chmod +x client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh" - -if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" -elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" -elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" -fi diff --git a/.github/scripts/benchmarks/postgres.sh b/.github/scripts/benchmarks/postgres.sh deleted file mode 100644 index aec0571b9d..0000000000 --- a/.github/scripts/benchmarks/postgres.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/sh -set -e -if [ "$RUNNER_OS" = "Linux" ]; then - sh -c "sudo sh -c 'echo \"deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main\" > /etc/apt/sources.list.d/pgdg.list'" - sh -c "wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -" - sh -c "sudo apt-get update" - sh -c "sudo apt-get -y install postgresql-15" - sh -c "sudo rm -rf /var/lib/postgresql/15/main" - sh -c "sudo mkdir -p /var/lib/postgresql/15/main" - sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" - sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" - sh -c "sudo chmod -R 777 /usr/lib/postgresql/15" - sh -c "sudo mkdir -p /usr/lib/postgresql-5432" - sh -c "sudo chmod -R 777 /usr/lib/postgresql-5432" - sh -c "sudo cp -R /usr/lib/postgresql/15 /usr/lib/postgresql-5432" - sh -c "sudo mkdir -p /var/lib/postgresql-5432/15/main" - sh -c "sudo chown -R postgres /var/lib/postgresql-5432/15/main" - sh -c "sudo chmod -R 777 /var/lib/postgresql-5432/15/main" - sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/initdb -D /var/lib/postgresql-5432/15/main --auth trust --no-instructions'" - sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/pg_ctl -D /var/lib/postgresql-5432/15/main -o \"-F -p 5432\" start'" - sh -c "sudo su - postgres -c '/usr/lib/postgresql-5432/15/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" -elif [ "$RUNNER_OS" = "Windows" ]; then - echo "Downloading zip archive. This may take a few minutes..." - sh -c "curl -LJO https://get.enterprisedb.com/postgresql/postgresql-15.5-1-windows-x64-binaries.zip -o postgresql-15.5-1-windows-x64-binaries.zip" - sh -c "unzip -qq postgresql-15.5-1-windows-x64-binaries.zip" - echo "Download finished." - sh -c "ls pgsql/bin" - sh -c "cp -R pgsql pgsql-5432" - filePrefix="pgsql-5432" - sh -c "mkdir -p $filePrefix/data" - sh -c "mkdir -p $filePrefix/logs" - arguments="-ArgumentList '-D', '$filePrefix/data', '--username=postgres', '--auth', 'trust', '--no-instructions', '-E', 'UTF8'" - redirect="-RedirectStandardOutput '$filePrefix/logs/db.log' -RedirectStandardError '$filePrefix/logs/db-error.log'" - powershell -command "Start-Process -FilePath '$filePrefix/bin/initdb' -NoNewWindow $arguments $redirect" - ctlarguments="-ArgumentList '-D', '$filePrefix/data', '-o', '\"-F -p 5432\"', 'start'" - ctlredirect="-RedirectStandardOutput '$filePrefix/logs/pg_ctl.log' -RedirectStandardError '$filePrefix/logs/pg_ctl-error.log'" - sh -c "sleep 6" - powershell -command "Start-Process -FilePath '$filePrefix/bin/pg_ctl' -NoNewWindow $ctlarguments $ctlredirect" - sql='"ALTER USER postgres WITH PASSWORD '\'\''postgres'\'\'';"' - sqlarguments="-ArgumentList '-c', $sql" - sqlredirect="-RedirectStandardOutput '$filePrefix/logs/psql.log' -RedirectStandardError '$filePrefix/logs/psql-error.log'" - powershell -command "Start-Process -FilePath '$filePrefix/bin/psql' -NoNewWindow $sqlarguments $sqlredirect" -elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "wget --quiet https://get.enterprisedb.com/postgresql/postgresql-15.2-1-osx-binaries.zip" - sh -c "sudo unzip -q postgresql-15.2-1-osx-binaries.zip" - sh -c "sudo dscl . -create /Users/postgres UniqueID 666" - sh -c "sudo dscl . -create /Groups/postgres PrimaryGroupID 777" - sh -c "sudo dscl . -create /Users/postgres PrimaryGroupID 777" - sh -c "sudo dscl . -create /Users/postgres UserShell /bin/bash" - sh -c "sudo dscl . -create /Users/postgres RealName \"PostgreSQL\"" - sh -c "sudo dscl . create /Groups/postgres passwd '*'" - sh -c "sudo mkdir /Users/postgres" - sh -c "sudo chown -R postgres:postgres /Users/postgres" - sh -c "sudo dscl . -create /Users/postgres NFSHomeDirectory /Users/postgres" - sh -c "sudo mkdir -p /var/lib/postgresql/15/main" - sh -c "sudo chown -R postgres /var/lib/postgresql/15/main" - sh -c "sudo chmod -R 777 /var/lib/postgresql/15/main" - sh -c "sudo mkdir -p /Users/postgres/pgsql-5432" - sh -c "sudo chmod -R 777 /Users/postgres/pgsql-5432" - sh -c "sudo cp -R pgsql /Users/postgres/pgsql-5432" - sh -c "sudo mkdir -p /var/lib/postgresql-5432/15/main" - sh -c "sudo chown -R postgres /var/lib/postgresql-5432/15/main" - sh -c "sudo chmod -R 777 /var/lib/postgresql-5432/15/main" - sh -c "sudo su - postgres -c '/Users/postgres/pgsql-5432/pgsql/bin/initdb -D /var/lib/postgresql-5432/15/main --auth trust --no-instructions'" - sh -c "sudo su - postgres -c '/Users/postgres/pgsql-5432/pgsql/bin/pg_ctl -D /var/lib/postgresql-5432/15/main -o \"-F -p 5432\" start'" - sh -c "sudo su - postgres -c '/Users/postgres/pgsql-5432/pgsql/bin/psql -c \"ALTER USER postgres WITH PASSWORD '\''postgres'\'';\"'" -else - echo "$RUNNER_OS is not supported" - exit 1 -fi \ No newline at end of file diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index f02744fa58..3e20b9aa31 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -11,7 +11,7 @@ else fi cd tpchdata # 目标文件夹路径 - destination_folder="../tpc/TPC-H V3.0.1/data" + destination_folder="../../tpc/TPC-H V3.0.1/data" # 确保目标文件夹存在，如果不存在则创建 mkdir -p "$destination_folder" diff --git a/.github/scripts/iginx/iginx_regression_udf_path.sh b/.github/scripts/iginx/iginx_regression_udf_path.sh deleted file mode 100644 index 752ba80b2a..0000000000 --- a/.github/scripts/iginx/iginx_regression_udf_path.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -cd IGinX -pwd -set -e - -cd core/target/iginx-core-0.7.0-SNAPSHOT/ - -iginx_home_path=$PWD - -echo "Iginx home path: $iginx_home_path" - -cd .. - -if [ -n "$MSYSTEM" ]; then - windows_path=$(cygpath -w "$iginx_home_path") - - export IGINX_HOME=$windows_path - - powershell -Command "Start-Process -FilePath 'iginx-core-0.7.0-SNAPSHOT/sbin/start_iginx.bat' -NoNewWindow -RedirectStandardOutput '../../iginx-udf.log' -RedirectStandardError '../../iginx-udf-error.log'" -else - export IGINX_HOME=$iginx_home_path - echo "Iginx home path: $IGINX_HOME" - pwd - - sh -c "chmod +x iginx-core-0.7.0-SNAPSHOT/sbin/start_iginx.sh" - - sh -c "nohup iginx-core-0.7.0-SNAPSHOT/sbin/start_iginx.sh > ../../iginx.log 2>&1 &" -fi diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 29f327370c..3ce96b4682 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -37,3 +37,8 @@ jobs: metadata-matrix: '["zookeeper"]' assemebly-test: uses: ./.github/workflows/assembly-test.yml + tpc-h: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml new file mode 100644 index 0000000000..cc16b9f0bc --- /dev/null +++ b/.github/workflows/tpc-h.yml @@ -0,0 +1,166 @@ +name: "TPC-H Test" + +on: + workflow_call: + inputs: + java-matrix: + description: "The java version to run the test on" + type: string + required: false + default: '["8"]' + python-matrix: + description: "The python version to run the test on" + type: string + required: false + default: '["3.9"]' + os-matrix: + description: "The operating system to run the test on" + type: string + required: false + default: '["ubuntu-latest", "macos-13", "windows-latest"]' + metadata-matrix: + description: "The metadata to run the test on" + type: string + required: false + default: '["zookeeper", "etcd"]' + db-matrix: + description: "The database to run the test on" + type: string + required: false + default: '["FileSystem", "IoTDB12", "InfluxDB", "PostgreSQL", "Redis", "MongoDB", "Parquet", "MySQL"]' + +jobs: + TPC-H-Test: + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + java: ${{ fromJSON(inputs.java-matrix) }} + python-version: ${{ fromJSON(inputs.python-matrix) }} + os: ${{ fromJSON(inputs.os-matrix) }} + metadata: ${{ fromJSON(inputs.metadata-matrix) }} + DB-name: ${{ fromJSON(inputs.db-matrix) }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Environment dependence + uses: ./.github/actions/dependence + with: + python-version: ${{ matrix.python-version }} + java: ${{ matrix.java }} + + - name: Display System Info + shell: bash + run: | + echo "Operating System: $(uname -a 2>/dev/null || ver)" + echo "Architecture: $(uname -m 2>/dev/null || echo %PROCESSOR_ARCHITECTURE%)" + echo "Java Version:" + java -version + echo "Python Version:" + python --version + echo "CPU Info:" + if [ "$(uname)" = "Linux" ]; then + lscpu + elif [ "$(uname)" = "Darwin" ]; then + sysctl -n machdep.cpu.brand_string + else + wmic cpu get name + fi + echo "Memory Info:" + if [ "$(uname)" = "Linux" ]; then + free -h + elif [ "$(uname)" = "Darwin" ]; then + vm_stat + else + systeminfo | findstr /C:"Total Physical Memory" + fi + + - name: Generate TPC-H Data + shell: bash + run: | + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" + + - name: Run ZooKeeper + uses: ./.github/actions/zookeeperRunner + + - name: Run DB + uses: ./.github/actions/dbRunner + with: + DB-name: ${{ matrix.DB-name }} + + - name: Switch to Old Version + shell: bash + run: | + cd .. + mkdir "new" + mkdir "new/IGinX" + cp -r "IGinX" "new/IGinX" + mkdir "old" + cd old + git clone https://github.com/IGinX-THU/IGinX.git + cd IGinX + + - name: Install IGinX with Maven + shell: bash + run: | + mvn clean package -DskipTests -P-format -q + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "../old/IGinX" + + - name: Start Old IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + Root-Dir-Path: "../old/IGinX" + + - name: Run Regression Test on Old IGinX + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format + + - name: Stop ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-stop: "true" + + - name: Stop Old IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-stop: "true" + + - name: Rerun ZooKeeper + uses: ./.github/actions/zookeeperRunner + with: + if-rerun: "true" + + - name: Switch to New Version + shell: bash + run: | + cd "../../new/IGinX" + + - name: Change IGinX config + uses: ./.github/actions/confWriter + with: + DB-name: ${{ matrix.DB-name }} + Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "../new/IGinX" + + - name: Start New IGinX + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + Root-Dir-Path: "../new/IGinX" + + - name: Run Regression Test on New IGinX + if: always() + shell: bash + run: | + mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java new file mode 100644 index 0000000000..f818b31389 --- /dev/null +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -0,0 +1,443 @@ +package cn.edu.tsinghua.iginx.integration.func.sql.tpch; + +import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; +import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.DATE; +import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.NUM; +import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.STR; + +import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.tool.MultiConnection; +import cn.edu.tsinghua.iginx.session.Session; +import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TPCHRegressionIT { + + // host info + protected static String defaultTestHost = "127.0.0.1"; + protected static int defaultTestPort = 6888; + protected static String defaultTestUser = "root"; + protected static String defaultTestPass = "root"; + + // .tbl文件所在目录 + private static final String dataPath = + System.getProperty("user.dir") + "/../../tpc/TPC-H V3.0.1/data"; + + // 最大重复测试次数 + private static final int MAX_REPETITIONS_NUM = 5; + + // 回归阈值 + private static final double REGRESSION_THRESHOLD_PERCENTAGE = 0.1; + + protected static MultiConnection conn; + + enum FieldType { + NUM, + STR, + DATE + } + + public TPCHRegressionIT() {} + + @BeforeClass + public static void setUp() throws SessionException { + conn = + new MultiConnection( + new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); + conn.openSession(); + } + + @AfterClass + public static void tearDown() throws SessionException { + clearAllData(conn); + conn.closeSession(); + } + + @Before + public void insertData() { + List tableList = + Arrays.asList( + "region", "nation", "supplier", "part", "partsupp", "customer", "orders", "lineitem"); + + List> fieldsList = new ArrayList<>(); + List> typesList = new ArrayList<>(); + + // region表 + fieldsList.add(Arrays.asList("r_regionkey", "r_name", "r_comment")); + typesList.add(Arrays.asList(NUM, STR, STR)); + + // nation表 + fieldsList.add(Arrays.asList("n_nationkey", "n_name", "n_regionkey", "n_comment")); + typesList.add(Arrays.asList(NUM, STR, NUM, STR)); + + // supplier表 + fieldsList.add( + Arrays.asList( + "s_suppkey", + "s_name", + "s_address", + "s_nationkey", + "s_phone", + "s_acctbal", + "s_comment")); + typesList.add(Arrays.asList(NUM, STR, STR, NUM, STR, NUM, STR)); + + // part表 + fieldsList.add( + Arrays.asList( + "p_partkey", + "p_name", + "p_mfgr", + "p_brand", + "p_type", + "p_size", + "p_container", + "p_retailprice", + "p_comment")); + typesList.add(Arrays.asList(NUM, STR, STR, STR, STR, NUM, STR, NUM, STR)); + + // partsupp表 + fieldsList.add( + Arrays.asList("ps_partkey", "ps_suppkey", "ps_availqty", "ps_supplycost", "ps_comment")); + typesList.add(Arrays.asList(NUM, NUM, NUM, NUM, STR)); + + // customer表 + fieldsList.add( + Arrays.asList( + "c_custkey", + "c_name", + "c_address", + "c_nationkey", + "c_phone", + "c_acctbal", + "c_mktsegment", + "c_comment")); + typesList.add(Arrays.asList(NUM, STR, STR, NUM, STR, NUM, STR, STR)); + + // orders表 + fieldsList.add( + Arrays.asList( + "o_orderkey", + "o_custkey", + "o_orderstatus", + "o_totalprice", + "o_orderdate", + "o_orderpriority", + "o_clerk", + "o_shippriority", + "o_comment")); + typesList.add(Arrays.asList(NUM, NUM, STR, NUM, DATE, STR, STR, NUM, STR)); + + // lineitem表 + fieldsList.add( + Arrays.asList( + "l_orderkey", + "l_partkey", + "l_suppkey", + "l_linenumber", + "l_quantity", + "l_extendedprice", + "l_discount", + "l_tax", + "l_returnflag", + "l_linestatus", + "l_shipdate", + "l_commitdate", + "l_receiptdate", + "l_shipinstruct", + "l_shipmode", + "l_comment")); + typesList.add( + Arrays.asList( + NUM, NUM, NUM, NUM, NUM, NUM, NUM, NUM, STR, STR, DATE, DATE, DATE, STR, STR, STR)); + + for (int i = 0; i < 8; i++) { + insertTable(tableList.get(i), fieldsList.get(i), typesList.get(i)); + } + } + + private void insertTable(String table, List fields, List types) { + StringBuilder builder = new StringBuilder("INSERT INTO "); + builder.append(table); + builder.append("(key, "); + for (String field : fields) { + builder.append(field); + builder.append(", "); + } + builder.setLength(builder.length() - 2); + builder.append(") VALUES "); + String insertPrefix = builder.toString(); + + long count = 0; + try (BufferedReader br = + new BufferedReader(new FileReader(String.format("%s/%s.tbl", dataPath, table)))) { + StringBuilder sb = new StringBuilder(insertPrefix); + String line; + while ((line = br.readLine()) != null) { + String[] items = line.split("\\|"); + sb.append("("); + count++; + sb.append(count); // 插入自增key列 + sb.append(", "); + assert fields.size() == items.length; + for (int i = 0; i < items.length; i++) { + switch (types.get(i)) { + case NUM: + sb.append(items[i]); + sb.append(", "); + break; + case STR: // 字符串类型在外面需要包一层引号 + sb.append("\""); + sb.append(items[i]); + sb.append("\", "); + break; + case DATE: // 日期类型需要转为时间戳 + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + sb.append(dateFormat.parse(items[i]).getTime()); + sb.append(", "); + break; + default: + break; + } + } + sb.setLength(sb.length() - 2); + sb.append("), "); + + // 每次最多插入10000条数据 + if (count % 10000 == 0) { + sb.setLength(sb.length() - 2); + sb.append(";"); + conn.executeSql(sb.toString()); + sb = new StringBuilder(insertPrefix); + } + } + } catch (IOException | ParseException | SessionException e) { + throw new RuntimeException(e); + } + } + + @After + public void clearData() throws SessionException { + conn.executeSql("CLEAR DATA;"); + } + + @Test + public void test() { + System.out.println("start"); + try { + // 获取当前JVM的Runtime实例 + Runtime runtime = Runtime.getRuntime(); + // 执行垃圾回收，尽量释放内存 + runtime.gc(); + // 获取执行语句前的内存使用情况 + long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); + long startTime; + // 13有问题 + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + for (int queryId : queryIds) { + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/tpch/queries/q" + queryId + ".sql"); + // 开始 tpc-h 查询 + System.out.println("start tpc-h query " + queryId); + startTime = System.currentTimeMillis(); + // 执行查询语句, split by ; 最后一句为执行结果 + SessionExecuteSqlResult result = null; + String[] sqls = sqlString.split(";"); + for (String sql : sqls) { + if (sql.trim().isEmpty()) { + continue; + } + sql += ";"; + result = conn.executeSql(sql); + result.print(false, ""); + } + // 再次执行垃圾回收 + runtime.gc(); + // 获取执行语句后的内存使用情况 + long usedMemoryAfter = runtime.totalMemory() - runtime.freeMemory(); + // 计算内存使用的变化 + long memoryUsed = usedMemoryAfter - usedMemoryBefore; + // 输出内存使用情况 + System.out.println("Memory used by the statement: " + memoryUsed + " bytes"); + long timeCost = System.currentTimeMillis() - startTime; + System.out.println("end tpc-h query, time cost: " + timeCost + "ms"); + + // 验证 + List> values = result.getValues(); + List> answers = + csvReader("src/test/resources/tpch/sf0.1/q" + queryId + ".csv"); + if (values.size() != answers.size()) { + System.out.println("values.size() = " + values.size()); + System.out.println("answers.size() = " + answers.size()); + throw new RuntimeException("size not equal"); + } + for (int i = 0; i < values.size(); i++) { + for (int j = 0; j < values.get(i).size(); j++) { + if (result.getPaths().get(j).contains("address") + || result.getPaths().get(j).contains("comment") + || result.getPaths().get(j).contains("orderdate")) { + // TODO change unix time to date + continue; + } + // if only contains number and dot, then parse to double + if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { + double number = Double.parseDouble(values.get(i).get(j).toString()); + double answerNumber = Double.parseDouble(answers.get(i).get(j)); + if (answerNumber - number >= 1e-3 || number - answerNumber >= 1e-3) { + System.out.println("Number: " + number); + System.out.println("Answer number: " + answerNumber); + } + assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; + } else { + String resultString = + new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); + String answerString = answers.get(i).get(j); + if (!resultString.equals(answerString)) { + System.out.println("Result string: " + resultString); + System.out.println("Answer string: " + answerString); + } + assert resultString.equals(answerString); + } + } + } + } + + String fileName = "src/test/resources/tpch/oldTimeCosts.txt"; + if (!Files.exists(Paths.get(fileName))) { // 文件不存在，即此次跑的是主分支代码，需要将查询时间写入文件 + List timeCosts = new ArrayList<>(); + for (int queryId : queryIds) { + // 开始 tpc-h 查询 + System.out.printf("start tpc-h query %d in main branch%n", queryId); + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/tpch/queries/q" + queryId + ".sql"); + // 执行查询语句, split by ; 最后一句为执行结果 + String[] sqls = sqlString.split(";"); + long timeCost = executeSQL(sqls[sqls.length - 2] + ";"); + timeCosts.add(timeCost); + System.out.printf( + "end tpc-h query %d in main branch, time cost: %dms%n", queryId, timeCost); + } + writeToFile(timeCosts, fileName); + } else { // 文件存在，即此次跑的是新分支代码，需要读取文件进行比较 + List oldTimeCosts = readFromFile(fileName); + double multiplyPercentage = 1 + REGRESSION_THRESHOLD_PERCENTAGE; + for (int i = 0; i < queryIds.size(); i++) { + double oldTimeCost = (double) oldTimeCosts.get(i); + double newTimeCost = 0; + List newTimeCosts = new ArrayList<>(); + boolean regressionDetected = true; + + while (newTimeCosts.size() < MAX_REPETITIONS_NUM) { + // 开始 tpc-h 查询 + System.out.printf("start tpc-h query %d in new branch%n", queryIds.get(i)); + // read from sql file + String sqlString = + readSqlFileAsString("src/test/resources/tpch/queries/q" + queryIds.get(i) + ".sql"); + // 执行查询语句, split by ; 最后一句为执行结果 + String[] sqls = sqlString.split(";"); + long timeCost = executeSQL(sqls[sqls.length - 2] + ";"); + System.out.printf( + "end tpc-h query %d in new branch, time cost: %dms%n", queryIds.get(i), timeCost); + newTimeCosts.add(timeCost); + newTimeCost = getMedian(newTimeCosts); + if (oldTimeCost * multiplyPercentage >= newTimeCost) { + regressionDetected = false; + break; + } + } + + // 重复5次后耗时仍超过阈值 + if (regressionDetected) { + System.out.printf( + "performance degradation of query %d exceeds %f%n", + queryIds.get(i), REGRESSION_THRESHOLD_PERCENTAGE); + System.out.printf("old timing: %.3fms%n", oldTimeCost); + System.out.printf("new timing: %.3fms%n", newTimeCost); + throw new RuntimeException("performance degradation exceeds the threshold"); + } + } + } + + } catch (SessionException | IOException e) { + throw new RuntimeException(e); + } + } + + private static long executeSQL(String sql) throws SessionException { + long startTime = System.currentTimeMillis(); + SessionExecuteSqlResult result = conn.executeSql(sql); + return System.currentTimeMillis() - startTime; + } + + private static double getMedian(List array) { + Collections.sort(array); + int middle = array.size() / 2; + return array.size() % 2 == 0 + ? (array.get(middle - 1) + array.get(middle)) / 2.0 + : (double) array.get(middle); + } + + private static void writeToFile(List timeCosts, String fileName) throws IOException { + Path filePath = Paths.get(fileName); + List lines = timeCosts.stream().map(String::valueOf).collect(Collectors.toList()); + Files.write(filePath, lines); + } + + private static List readFromFile(String fileName) throws IOException { + Path filePath = Paths.get(fileName); + List lines = Files.readAllLines(filePath); + return lines.stream().map(Long::valueOf).collect(Collectors.toList()); + } + + private static List> csvReader(String filePath) { + List> data = new ArrayList<>(); + boolean skipHeader = true; + try (Scanner scanner = new Scanner(Paths.get(filePath))) { + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + if (skipHeader) { + skipHeader = false; + continue; + } + List row = Arrays.asList(line.split("\\|")); + data.add(row); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + System.out.println(data); + return data; + } + + private static String readSqlFileAsString(String filePath) throws IOException { + StringBuilder contentBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + while ((line = br.readLine()) != null) { + contentBuilder.append(line).append("\n"); + } + } + return contentBuilder.toString(); + } +} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java deleted file mode 100644 index 04c062864e..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHDataInsertionIT.java +++ /dev/null @@ -1,296 +0,0 @@ -package cn.edu.tsinghua.iginx.integration.polybench; - -import com.mongodb.MongoClientSettings; -import com.mongodb.ServerAddress; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoCollection; -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.sql.*; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.Date; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import org.bson.Document; -import org.junit.Test; -import org.postgresql.copy.CopyManager; -import org.postgresql.core.BaseConnection; - -public class TPCHDataInsertionIT { - private static final String dataPath = - System.getProperty("user.dir") + "/../tpc/TPC-H V3.0.1/data"; - - public void TPCHDataInsertionIT() {} - - private static MongoClient connect(int port) { - ServerAddress address = new ServerAddress("127.0.0.1", port); - MongoClientSettings settings = - MongoClientSettings.builder() - .applyToClusterSettings(builder -> builder.hosts(Collections.singletonList(address))) - .build(); - - return MongoClients.create(settings); - } - - @Test - public void insertDataIntoMongoDB() { - System.out.println(System.getProperty("user.dir")); - int port = 27017; - String databaseName = "mongotpch"; - try (MongoClient client = connect(port)) { - ExecutorService executor = Executors.newFixedThreadPool(2); // 使用2个线程 - List> futures = new ArrayList<>(); - - // 读取 lineitem.tbl 文件并插入数据 - futures.add(executor.submit(() -> insertLineItemFromFile(client, databaseName))); - - // 读取 orders.tbl 文件并插入数据 - futures.add(executor.submit(() -> insertOrdersFromFile(client, databaseName))); - - // 等待所有任务完成 - for (Future future : futures) { - future.get(); - } - - executor.shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void insertLineItemFromFile(MongoClient client, String databaseName) { - try { - String collectionName = "lineitem"; - String fileName = "lineitem.tbl"; - Long start_time = System.currentTimeMillis(); - List documents = new ArrayList<>(); - try (BufferedReader br = - new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { - String line; - while ((line = br.readLine()) != null) { - String[] fields = line.split("\\|"); - // 解析数据并构建 Document 对象 - // 将字符型字段转换为相应类型 - int l_orderkey = Integer.parseInt(fields[0]); - int l_partkey = Integer.parseInt(fields[1]); - int l_suppkey = Integer.parseInt(fields[2]); - int l_linenumber = Integer.parseInt(fields[3]); - double l_quantity = Double.parseDouble(fields[4]); - double l_extendedprice = Double.parseDouble(fields[5]); - double l_discount = Double.parseDouble(fields[6]); - double l_tax = Double.parseDouble(fields[7]); - String l_returnflag = fields[8]; - String l_linestatus = fields[9]; - // 解析日期字段 - Date l_shipdate = null, l_commitdate = null, l_receiptdate = null; - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - l_shipdate = dateFormat.parse(fields[10]); - l_commitdate = dateFormat.parse(fields[11]); - l_receiptdate = dateFormat.parse(fields[12]); - } catch (ParseException e) { - e.printStackTrace(); - } - String l_shipinstruct = fields[13]; - String l_shipmode = fields[14]; - String l_comment = fields[15]; - - // 构建 MongoDB 文档 - Document document = - new Document() - .append("l_orderkey", l_orderkey) - .append("l_partkey", l_partkey) - .append("l_suppkey", l_suppkey) - .append("l_linenumber", l_linenumber) - .append("l_quantity", l_quantity) - .append("l_extendedprice", l_extendedprice) - .append("l_discount", l_discount) - .append("l_tax", l_tax) - .append("l_returnflag", l_returnflag) - .append("l_linestatus", l_linestatus) - .append("l_shipdate", l_shipdate) - .append("l_commitdate", l_commitdate) - .append("l_receiptdate", l_receiptdate) - .append("l_shipinstruct", l_shipinstruct) - .append("l_shipmode", l_shipmode) - .append("l_comment", l_comment); - // 将 Document 对象添加到列表中 - documents.add(document); - if (documents.size() >= 10000) { - // 每次插入 10000 条数据 - MongoCollection collection = - client.getDatabase(databaseName).getCollection(collectionName); - collection.insertMany(documents); - documents.clear(); - System.out.println("Inserted 10000 records into " + collectionName); - } - } - } catch (IOException e) { - e.printStackTrace(); - return; - } - // 插入数据到 MongoDB - MongoCollection collection = - client.getDatabase(databaseName).getCollection(collectionName); - collection.insertMany(documents); - System.out.println( - "Data loaded successfully to collection " - + collectionName - + " in " - + (System.currentTimeMillis() - start_time) - + "ms"); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void insertOrdersFromFile(MongoClient client, String databaseName) { - try { - String collectionName = "orders"; - String fileName = "orders.tbl"; - Long start_time = System.currentTimeMillis(); - List documents = new ArrayList<>(); - try (BufferedReader br = - new BufferedReader(new FileReader(String.format("%s/%s", dataPath, fileName)))) { - String line; - while ((line = br.readLine()) != null) { - String[] fields = line.split("\\|"); - // 将字符型字段转换为相应类型 - int o_orderkey = Integer.parseInt(fields[0]); - int o_custkey = Integer.parseInt(fields[1]); - String o_orderstatus = fields[2]; - double o_totalprice = Double.parseDouble(fields[3]); - Date o_orderdate = null; - try { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - o_orderdate = dateFormat.parse(fields[4]); - } catch (ParseException e) { - e.printStackTrace(); - } - String o_orderpriority = fields[5]; - String o_clerk = fields[6]; - int o_shippriority = Integer.parseInt(fields[7]); - String o_comment = fields[8]; - - // 构建 MongoDB 文档 - Document document = - new Document() - .append("o_orderkey", o_orderkey) - .append("o_custkey", o_custkey) - .append("o_orderstatus", o_orderstatus) - .append("o_totalprice", o_totalprice) - .append("o_orderdate", o_orderdate) - .append("o_orderpriority", o_orderpriority) - .append("o_clerk", o_clerk) - .append("o_shippriority", o_shippriority) - .append("o_comment", o_comment); - // 将 Document 对象添加到列表中 - documents.add(document); - if (documents.size() >= 10000) { - // 每次插入 10000 条数据 - MongoCollection collection = - client.getDatabase(databaseName).getCollection(collectionName); - collection.insertMany(documents); - documents.clear(); - System.out.println("Inserted 10000 records into " + collectionName); - } - } - } catch (IOException e) { - e.printStackTrace(); - return; - } - // 插入数据到 MongoDB - MongoCollection collection = - client.getDatabase(databaseName).getCollection(collectionName); - collection.insertMany(documents); - System.out.println( - "Data loaded successfully to collection " - + collectionName - + " in " - + (System.currentTimeMillis() - start_time) - + "ms"); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void insertDataIntoPostgreSQL() { - int port = 5432; - String databaseName = "tpchdata"; - // PostgreSQL连接参数 - String url = String.format("jdbc:postgresql://localhost:%s/postgres", port); - String user = "postgres"; - String password = "postgres"; - - // 待执行的 SQL 语句 - String[] sqlStatements = { - "DROP TABLE IF EXISTS customer", - "DROP TABLE IF EXISTS supplier", - "DROP TABLE IF EXISTS region", - "DROP TABLE IF EXISTS part", - "DROP TABLE IF EXISTS partsupp", - "CREATE TABLE IF NOT EXISTS customer ( c_custkey INT, c_name VARCHAR(25), c_address VARCHAR(40), c_nationkey INT, c_phone VARCHAR(15), c_acctbal FLOAT, c_mktsegment VARCHAR(10), c_comment VARCHAR(117), c_dummy VARCHAR(2), PRIMARY KEY (c_custkey))", - "CREATE TABLE IF NOT EXISTS region ( r_regionkey INT, r_name VARCHAR(25), r_comment VARCHAR(152), r_dummy VARCHAR(2), PRIMARY KEY (r_regionkey))", - "CREATE TABLE IF NOT EXISTS supplier ( s_suppkey INT, s_name VARCHAR(25), s_address VARCHAR(40), s_nationkey INT, s_phone VARCHAR(15), s_acctbal FLOAT, s_comment VARCHAR(101), s_dummy VARCHAR(2), PRIMARY KEY (s_suppkey))", - "CREATE TABLE IF NOT EXISTS part ( p_partkey INT, p_name VARCHAR(55), p_mfgr VARCHAR(25), p_brand VARCHAR(10), p_type VARCHAR(25), p_size INT, p_container VARCHAR(10), p_retailprice FLOAT , p_comment VARCHAR(23) , p_dummy VARCHAR(2), PRIMARY KEY (p_partkey))", - "CREATE TABLE IF NOT EXISTS partsupp ( ps_partkey INT, ps_suppkey INT, ps_availqty INT, ps_supplycost FLOAT, ps_comment VARCHAR(199), ps_dummy VARCHAR(2))", - }; - - try (Connection conn = DriverManager.getConnection(url, user, password); - Statement stmt = conn.createStatement()) { - if (conn != null) { - System.out.println("Connected to the PostgreSQL server successfully."); - stmt.executeUpdate(String.format("DROP DATABASE IF EXISTS %s", databaseName)); - stmt.executeUpdate(String.format("CREATE DATABASE %s", databaseName)); - System.out.println(String.format("Database '%s' created successfully.", databaseName)); - // 关闭当前连接 - stmt.close(); - conn.close(); - } - } catch (SQLException e) { - System.out.println("SQLException: " + e.getMessage()); - e.printStackTrace(); - } - // 连接到新创建的数据库 - String newDbUrl = String.format("jdbc:postgresql://localhost:5432/%s", databaseName); - try (Connection conn = DriverManager.getConnection(newDbUrl, user, password); - Statement stmt = conn.createStatement()) { - if (conn != null) { - System.out.println(String.format("Connected to '%s' successfully.", databaseName)); - - // 依次执行每条 SQL 语句 - for (String sql : sqlStatements) { - stmt.execute(sql); - System.out.println("Executed SQL statement: " + sql); - } - CopyManager copyManager = new CopyManager((BaseConnection) conn); - List tableNames = - Arrays.asList("customer", "supplier", "region", "part", "partsupp"); - for (String tableName : tableNames) { - String filePath = String.format("%s/%s.tbl", dataPath, tableName); - FileReader fileReader = new FileReader(filePath); - // 使用 CopyManager 执行 COPY 命令将数据从 CSV 文件加载到数据库表中 - copyManager.copyIn( - "COPY " + tableName + " FROM STDIN WITH DELIMITER '|' CSV", fileReader); - System.out.println("Data loaded successfully from CSV to table " + tableName); - } - } else { - System.out.println("Failed to make connection to the PostgreSQL server."); - } - } catch (SQLException e) { - System.out.println("SQLException: " + e.getMessage()); - e.printStackTrace(); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java deleted file mode 100644 index 59d18f31e8..0000000000 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/polybench/TPCHRunner.java +++ /dev/null @@ -1,255 +0,0 @@ -package cn.edu.tsinghua.iginx.integration.polybench; - -import cn.edu.tsinghua.iginx.exception.SessionException; -import cn.edu.tsinghua.iginx.integration.tool.MultiConnection; -import cn.edu.tsinghua.iginx.session.Session; -import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; -import cn.edu.tsinghua.iginx.thrift.*; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.stream.Collectors; -import org.junit.Test; - -public class TPCHRunner { - // host info - protected static String defaultTestHost = "127.0.0.1"; - protected static int defaultTestPort = 6888; - protected static String defaultTestUser = "root"; - protected static String defaultTestPass = "root"; - protected static MultiConnection conn; - - public static void TPCRunner(String[] args) {} - - public static void writeToFile(List avgTimeCosts, String fileName) throws IOException { - Path filePath = Paths.get(fileName); - List lines = avgTimeCosts.stream().map(String::valueOf).collect(Collectors.toList()); - Files.write(filePath, lines); - } - - public static List readFromFile(String fileName) throws IOException { - Path filePath = Paths.get(fileName); - List lines = Files.readAllLines(filePath); - return lines.stream().map(Double::valueOf).collect(Collectors.toList()); - } - - private List> csvReader(String filePath) { - List> data = new ArrayList<>(); - boolean skipHeader = true; - try (Scanner scanner = new Scanner(Paths.get(filePath))) { - while (scanner.hasNextLine()) { - String line = scanner.nextLine(); - if (skipHeader) { - skipHeader = false; - continue; - } - List row = Arrays.asList(line.split("\\|")); - data.add(row); - } - } catch (IOException e) { - e.printStackTrace(); - } - System.out.println(data); - return data; - } - - public static String readSqlFileAsString(String filePath) throws IOException { - StringBuilder contentBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { - String line; - while ((line = br.readLine()) != null) { - contentBuilder.append(line).append("\n"); - } - } - return contentBuilder.toString(); - } - - @Test - public void test() { - System.out.println("start"); - try { - conn = - new MultiConnection( - new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); - conn.openSession(); - - // 输出所有存储引擎 - String clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); - // 获取当前JVM的Runtime实例 - Runtime runtime = Runtime.getRuntime(); - - // 执行垃圾回收，尽量释放内存 - runtime.gc(); - - // 获取执行语句前的内存使用情况 - long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); - - // 添加存储引擎 - System.out.println("start adding storage engine"); - long startTime = System.currentTimeMillis(); - Map pgMap = new HashMap<>(); - pgMap.put("engine", "postgresql"); - pgMap.put("has_data", "true"); - pgMap.put("is_read_only", "true"); - pgMap.put("username", "postgres"); - pgMap.put("password", "postgres"); - pgMap = Collections.unmodifiableMap(pgMap); - conn.addStorageEngine("127.0.0.1", 5432, StorageEngineType.relational, pgMap); - Map mongoMap = new HashMap<>(); - mongoMap.put("has_data", "true"); - mongoMap.put("is_read_only", "true"); - mongoMap.put("schema.sample.size", "1000"); - mongoMap.put("dummy.sample.size", "0"); - conn.addStorageEngine("127.0.0.1", 27017, StorageEngineType.mongodb, mongoMap); - System.out.println( - "end adding storage engine, time cost: " - + (System.currentTimeMillis() - startTime) - + "ms"); - - // 输出所有存储引擎 - clusterInfo = conn.executeSql("SHOW CLUSTER INFO;").getResultInString(false, ""); - System.out.println(clusterInfo); - // Long startTime; - // 13有问题 - List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); - for (int queryId : queryIds) { - // read from sql file - String sqlString = - readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); - - // 开始 tpch 查询 - System.out.println("start tpch query " + queryId); - startTime = System.currentTimeMillis(); - - // 执行查询语句, split by ; 最后一句为执行结果 - SessionExecuteSqlResult result = null; - String[] sqls = sqlString.split(";"); - for (String sql : sqls) { - if (sql.trim().length() == 0) { - continue; - } - sql += ";"; - result = conn.executeSql(sql); - result.print(false, ""); - } - // 再次执行垃圾回收 - runtime.gc(); - // 获取执行语句后的内存使用情况 - long usedMemoryAfter = runtime.totalMemory() - runtime.freeMemory(); - // 计算内存使用的变化 - long memoryUsed = usedMemoryAfter - usedMemoryBefore; - // 输出内存使用情况 - System.out.println("Memory used by the statement: " + memoryUsed + " bytes"); - - // 验证 - long timeCost = System.currentTimeMillis() - startTime; - System.out.println("end tpch query, time cost: " + timeCost + "ms"); - List> values = result.getValues(); - List> answers = - csvReader("src/test/resources/polybench/sf0.1/q" + queryId + ".csv"); - if (values.size() != answers.size()) { - System.out.println("values.size() = " + values.size()); - System.out.println("answers.size() = " + answers.size()); - throw new RuntimeException("size not equal"); - } - for (int i = 0; i < values.size(); i++) { - if (values.get(i).size() != answers.get(i).size()) { - System.out.println("values.get(i).size() = " + values.get(i).size()); - System.out.println("answers.get(i).size() = " + answers.get(i).size()); - throw new RuntimeException("size not equal"); - } - for (int j = 0; j < values.get(i).size(); j++) { - if (result.getPaths().get(j).contains("address") - || result.getPaths().get(j).contains("comment") - || result.getPaths().get(j).contains("orderdate")) { - // TODO change unix time to date - continue; - } - // if only contains number and dot, then parse to double - if (values.get(i).get(j).toString().matches("-?[0-9]+.*[0-9]*")) { - double number = Double.parseDouble(values.get(i).get(j).toString()); - double answerNumber = Double.parseDouble(answers.get(i).get(j)); - if (answerNumber - number >= 1e-3 || number - answerNumber >= 1e-3) { - System.out.println("Number: " + number); - System.out.println("Answer number: " + answerNumber); - } - assert answerNumber - number < 1e-3 && number - answerNumber < 1e-3; - } else { - String resultString = - new String((byte[]) values.get(i).get(j), StandardCharsets.UTF_8); - String answerString = answers.get(i).get(j); - if (!resultString.equals(answerString)) { - System.out.println("Result string: " + resultString); - System.out.println("Answer string: " + answerString); - } - assert resultString.equals(answerString); - } - } - } - } - List avgTimeCosts = new ArrayList<>(); - for (int queryId : queryIds) { - // read from sql file - String sqlString = - readSqlFileAsString("src/test/resources/polybench/queries/q" + queryId + ".sql"); - - // 开始 tpch 查询 - System.out.println("start tpch query " + queryId); - - // 执行查询语句, split by ; 最后一句为执行结果 - SessionExecuteSqlResult result = null; - String[] sqls = sqlString.split(";"); - List timeCosts = new ArrayList<>(); - for (int i = 0; i < 3; i++) { // TODO: 5 - startTime = System.currentTimeMillis(); - if (sqls.length == 1) { - // TODO: error - System.out.println("wrong input"); - } else { - result = conn.executeSql(sqls[sqls.length - 2] + ";"); - } - Long timeCost = System.currentTimeMillis() - startTime; - timeCosts.add(timeCost); - System.out.println("query " + queryId + ", time cost: " + timeCost + "ms"); - } - Double averageTimeCost = - timeCosts.stream().mapToLong(Long::longValue).average().getAsDouble(); - avgTimeCosts.add(averageTimeCost); - System.out.println( - "end tpch query " + queryId + ", average time cost: " + averageTimeCost + "ms"); - } - String fileName = "src/test/resources/polybench/avgTimeCosts.txt"; - if (Files.exists(Paths.get(fileName))) { // 如果文件存在，即此次跑的是主分支代码，需要读取文件进行比较 - List newAvgTimeCosts = readFromFile(fileName); // 文件中存的是新分支的运行时间 - for (int i = 0; i < queryIds.size(); i++) { - System.out.printf( - "query %d, old time cost: %2fms\n", queryIds.get(i), avgTimeCosts.get(i)); - System.out.printf( - "query %d, new average time cost: %2fms\n", queryIds.get(i), newAvgTimeCosts.get(i)); - // TODO 如果相差超过30%？，则报错 - if (Math.abs(newAvgTimeCosts.get(i) - avgTimeCosts.get(i)) > 0.3 * avgTimeCosts.get(i)) { - System.out.printf( - "query %d, old average time cost: %2fms\n", queryIds.get(i), avgTimeCosts.get(i)); - System.out.printf( - "query %d, new average time cost: %2fms\n", - queryIds.get(i), newAvgTimeCosts.get(i)); - throw new RuntimeException("time cost not equal"); - } - } - } else { - writeToFile(avgTimeCosts, fileName); - } - - // 关闭会话 - conn.closeSession(); - } catch (SessionException | IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/test/src/test/resources/polybench/queries/q1.sql b/test/src/test/resources/polybench/queries/q1.sql deleted file mode 100644 index ca0cd2f221..0000000000 --- a/test/src/test/resources/polybench/queries/q1.sql +++ /dev/null @@ -1,31 +0,0 @@ -select - mongotpch.lineitem.l_returnflag as l_returnflag, - mongotpch.lineitem.l_linestatus as l_linestatus, - sum(mongotpch.lineitem.l_quantity) as sum_qty, - sum(mongotpch.lineitem.l_extendedprice) as sum_base_price, - sum(tmp1) as sum_disc_price, - sum(tmp2) as sum_charge, - avg(mongotpch.lineitem.l_quantity) as avg_qty, - avg(mongotpch.lineitem.l_extendedprice) as avg_price, - avg(mongotpch.lineitem.l_discount) as avg_disc, - count(mongotpch.lineitem.l_returnflag) as count_order -from ( - select - l_returnflag, - l_linestatus, - l_quantity, - l_extendedprice, - l_discount, - l_extendedprice * (1 - l_discount) as tmp1, - l_extendedprice * (1 - l_discount) * (1 + l_tax) as tmp2 - from - mongotpch.lineitem - where - mongotpch.lineitem.l_shipdate <= 904694400000 - ) -group by - mongotpch.lineitem.l_returnflag, - mongotpch.lineitem.l_linestatus -order by - mongotpch.lineitem.l_returnflag, - mongotpch.lineitem.l_linestatus; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q10.sql b/test/src/test/resources/polybench/queries/q10.sql deleted file mode 100644 index d145f6413b..0000000000 --- a/test/src/test/resources/polybench/queries/q10.sql +++ /dev/null @@ -1,51 +0,0 @@ -select - tpchdata.customer.c_custkey, - tpchdata.customer.c_name, - revenue, - tpchdata.customer.c_acctbal, - nation.n_name, - tpchdata.customer.c_address, - tpchdata.customer.c_phone, - tpchdata.customer.c_comment -from ( - select - tpchdata.customer.c_custkey, - tpchdata.customer.c_name, - sum(tmp) as revenue, - tpchdata.customer.c_acctbal, - nation.n_name, - tpchdata.customer.c_address, - tpchdata.customer.c_phone, - tpchdata.customer.c_comment - from ( - select - tpchdata.customer.c_custkey, - tpchdata.customer.c_name, - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp, - tpchdata.customer.c_acctbal, - nation.n_name, - tpchdata.customer.c_address, - tpchdata.customer.c_phone, - tpchdata.customer.c_comment - from - tpchdata.customer - join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey - join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey - join nation on tpchdata.customer.c_nationkey = nation.n_nationkey - where - mongotpch.orders.o_orderdate >= 749404800000 - and mongotpch.orders.o_orderdate < 757353600000 - and mongotpch.lineitem.l_returnflag = 'R' - ) - group by - tpchdata.customer.c_custkey, - tpchdata.customer.c_name, - tpchdata.customer.c_acctbal, - tpchdata.customer.c_phone, - nation.n_name, - tpchdata.customer.c_address, - tpchdata.customer.c_comment - ) -order by - revenue desc -limit 20; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q13.sql b/test/src/test/resources/polybench/queries/q13.sql deleted file mode 100644 index b6becd6182..0000000000 --- a/test/src/test/resources/polybench/queries/q13.sql +++ /dev/null @@ -1,25 +0,0 @@ -select - c_count, - custdist -from ( - select - c_count, - count(c_custkey) as custdist - from ( - select - tpchdata.customer.c_custkey as c_custkey, - count(mongotpch.orders.o_orderkey) as c_count - from - tpchdata.customer - left outer join mongotpch.orders - on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey - and !(mongotpch.orders.o_comment like '.*pending.*') - group by - tpchdata.customer.c_custkey - ) - group by - c_count - ) -order by - custdist, - c_count desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q16.sql b/test/src/test/resources/polybench/queries/q16.sql deleted file mode 100644 index e2878cb645..0000000000 --- a/test/src/test/resources/polybench/queries/q16.sql +++ /dev/null @@ -1,49 +0,0 @@ -select - p_brand, - p_type, - p_size, - supplier_cnt -from ( - select - tpchdata.part.p_brand as p_brand, - tpchdata.part.p_type as p_type, - tpchdata.part.p_size as p_size, - count(distinct tpchdata.partsupp.ps_suppkey) as supplier_cnt - from - tpchdata.partsupp - join tpchdata.part on tpchdata.part.p_partkey = tpchdata.partsupp.ps_partkey - where - tpchdata.part.p_brand <> 'Brand#45' - and tpchdata.part.p_partkey not in( - select p_partkey from tpchdata.part - where tpchdata.part.p_type like '.*MEDIUM POLISHED.*' - ) - and ( - tpchdata.part.p_size = 3 - or tpchdata.part.p_size = 9 - or tpchdata.part.p_size = 14 - or tpchdata.part.p_size = 19 - or tpchdata.part.p_size = 23 - or tpchdata.part.p_size = 36 - or tpchdata.part.p_size = 45 - or tpchdata.part.p_size = 49 - ) - and tpchdata.partsupp.ps_suppkey not in ( - select - s_suppkey - from - tpchdata.supplier - where - tpchdata.supplier.s_comment like '.*Customer.*Complaints.*' - ) - group by - tpchdata.part.p_brand, - tpchdata.part.p_type, - tpchdata.part.p_size - order by - tpchdata.part.p_brand, - tpchdata.part.p_type, - tpchdata.part.p_size - ) -order by - supplier_cnt desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q17.sql b/test/src/test/resources/polybench/queries/q17.sql deleted file mode 100644 index 02be404aa5..0000000000 --- a/test/src/test/resources/polybench/queries/q17.sql +++ /dev/null @@ -1,29 +0,0 @@ -insert into tmpTableB(key, p_partkey, val) values ( - select - tpchdata.part.p_partkey, - 0.2 * tmp - from ( - select - tpchdata.part.p_partkey, - avg(mongotpch.lineitem.l_quantity) as tmp - from - mongotpch.lineitem - join tpchdata.part on mongotpch.lineitem.l_partkey = tpchdata.part.p_partkey - group by tpchdata.part.p_partkey - ) -); - -select - tmp2 / 7 as avg_yearly -from ( - select - sum(mongotpch.lineitem.l_extendedprice) as tmp2 - from - mongotpch.lineitem - join tpchdata.part on tpchdata.part.p_partkey = mongotpch.lineitem.l_partkey - join tmpTableB on tmpTableB.p_partkey = mongotpch.lineitem.l_partkey - where - tpchdata.part.p_brand = 'Brand#23' - and tpchdata.part.p_container = 'MED BOX' - and mongotpch.lineitem.l_quantity < tmpTableB.val - ); \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q18.sql b/test/src/test/resources/polybench/queries/q18.sql deleted file mode 100644 index 9225cedade..0000000000 --- a/test/src/test/resources/polybench/queries/q18.sql +++ /dev/null @@ -1,35 +0,0 @@ -select - tpchdata.customer.c_name, - tpchdata.customer.c_custkey, - mongotpch.orders.o_orderkey, - mongotpch.orders.o_orderdate, - mongotpch.orders.o_totalprice, - sum(mongotpch.lineitem.l_quantity) -from - tpchdata.customer - join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey - join mongotpch.lineitem on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey -where - mongotpch.orders.o_orderkey in ( - select - mongotpch.lineitem.l_orderkey - from ( - select - l_orderkey, - sum(l_quantity) - from - mongotpch.lineitem - group by - l_orderkey - having - sum(mongotpch.lineitem.l_quantity) > 300 - ) - ) -group by - tpchdata.customer.c_name, - tpchdata.customer.c_custkey, - mongotpch.orders.o_orderkey, - mongotpch.orders.o_orderdate, - mongotpch.orders.o_totalprice -order by - mongotpch.orders.o_totalprice desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q19.sql b/test/src/test/resources/polybench/queries/q19.sql deleted file mode 100644 index 8a3fc95899..0000000000 --- a/test/src/test/resources/polybench/queries/q19.sql +++ /dev/null @@ -1,48 +0,0 @@ -select - sum(tmp) as revenue -from ( - select - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp - from - mongotpch.lineitem - join tpchdata.part on tpchdata.part.p_partkey = mongotpch.lineitem.l_partkey - where ( - tpchdata.part.p_brand = 'Brand#12' - and ( - tpchdata.part.p_container = 'SM CASE' - or tpchdata.part.p_container = 'SM BOX' - or tpchdata.part.p_container = 'SM PACK' - or tpchdata.part.p_container = 'SM PKG' - ) - and mongotpch.lineitem.l_quantity >= 1 and mongotpch.lineitem.l_quantity <= 11 - and tpchdata.part.p_size >= 1 and tpchdata.part.p_size <= 5 - and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') - and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' - ) - or ( - tpchdata.part.p_brand = 'Brand#23' - and ( - tpchdata.part.p_container = 'MED PKG' - or tpchdata.part.p_container = 'MED BOX' - or tpchdata.part.p_container = 'MED BAG' - or tpchdata.part.p_container = 'MED PACK' - ) - and mongotpch.lineitem.l_quantity >= 10 and mongotpch.lineitem.l_quantity <= 20 - and tpchdata.part.p_size >= 1 and tpchdata.part.p_size <= 10 - and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') - and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' - ) - or ( - tpchdata.part.p_brand = 'Brand#34' - and ( - tpchdata.part.p_container = 'LG PACK' - or tpchdata.part.p_container = 'LG BOX' - or tpchdata.part.p_container = 'LG CASE' - or tpchdata.part.p_container = 'LG PKG' - ) - and mongotpch.lineitem.l_quantity >= 20 and mongotpch.lineitem.l_quantity <= 30 - and tpchdata.part.p_size >= 1 and tpchdata.part.p_size <= 15 - and (mongotpch.lineitem.l_shipmode = 'AIR' or mongotpch.lineitem.l_shipmode = 'AIR REG') - and mongotpch.lineitem.l_shipinstruct = 'DELIVER IN PERSON' - ) - ); \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q2.sql b/test/src/test/resources/polybench/queries/q2.sql deleted file mode 100644 index 67c76217fc..0000000000 --- a/test/src/test/resources/polybench/queries/q2.sql +++ /dev/null @@ -1,44 +0,0 @@ -insert into tmpTable(key, p_key, minCost) values ( -select - tpchdata.part.p_partkey as p_key, - min(tpchdata.partsupp.ps_supplycost) as minCost -from - tpchdata.partsupp - join tpchdata.supplier on tpchdata.supplier.s_suppkey = tpchdata.partsupp.ps_suppkey - join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey - join tpchdata.region on nation.n_regionkey = tpchdata.region.r_regionkey - join tpchdata.part on tpchdata.part.p_partkey = tpchdata.partsupp.ps_partkey -where - tpchdata.region.r_name = 'EUROPE' - and tpchdata.part.p_size = 15 - and tpchdata.part.p_type like "^.*BRASS" -group by tpchdata.part.p_partkey -); - -select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from - (select - tpchdata.supplier.s_acctbal as s_acctbal, - tpchdata.supplier.s_name as s_name, - nation.n_name as n_name, - tpchdata.part.p_partkey as p_partkey, - tpchdata.part.p_mfgr as p_mfgr, - tpchdata.supplier.s_address as s_address, - tpchdata.supplier.s_phone as s_phone, - tpchdata.supplier.s_comment as s_comment - from - tpchdata.part - join tpchdata.partsupp on tpchdata.part.p_partkey = tpchdata.partsupp.ps_partkey - join tpchdata.supplier on tpchdata.supplier.s_suppkey = tpchdata.partsupp.ps_suppkey - join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey - join tpchdata.region on nation.n_regionkey = tpchdata.region.r_regionkey - join tmpTable on tmpTable.p_key = tpchdata.part.p_partkey and tpchdata.partsupp.ps_supplycost = tmpTable.minCost - where - tpchdata.part.p_size = 15 - and tpchdata.region.r_name = 'EUROPE' - and tpchdata.part.p_type like "^.*BRASS" - order by - nation.n_name, - tpchdata.supplier.s_name, - tpchdata.part.p_partkey - ) -order by s_acctbal desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q20.sql b/test/src/test/resources/polybench/queries/q20.sql deleted file mode 100644 index 8f3167414a..0000000000 --- a/test/src/test/resources/polybench/queries/q20.sql +++ /dev/null @@ -1,45 +0,0 @@ -insert into tmpTableA(key, partkey, suppkey, val) values ( - select - partkey, - suppkey, - 0.5 * tmp from( - select - l_partkey as partkey, - l_suppkey as suppkey, - sum(l_quantity) as tmp - from - mongotpch.lineitem - where - mongotpch.lineitem.l_shipdate >= 757353600000 - and mongotpch.lineitem.l_shipdate < 788889600000 - group by l_partkey, l_suppkey - ) -); - -select - tpchdata.supplier.s_name, - tpchdata.supplier.s_address -from - tpchdata.supplier - join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey -where - tpchdata.supplier.s_suppkey in ( - select - tpchdata.partsupp.ps_suppkey - from - tpchdata.partsupp - join tmpTableA on tmpTableA.suppkey = tpchdata.partsupp.ps_suppkey and tmpTableA.partkey = tpchdata.partsupp.ps_partkey - where - tpchdata.partsupp.ps_partkey in ( - select - p_partkey - from - tpchdata.part - where - tpchdata.part.p_name like 'forest.*' - ) - and tpchdata.partsupp.ps_availqty > tmpTableA.val - ) - and nation.n_name = 'CANADA' -order by - tpchdata.supplier.s_name; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q3.sql b/test/src/test/resources/polybench/queries/q3.sql deleted file mode 100644 index 468d0f7114..0000000000 --- a/test/src/test/resources/polybench/queries/q3.sql +++ /dev/null @@ -1,31 +0,0 @@ -select l_orderkey, - revenue, - o_orderdate, - o_shippriority from - (select - l_orderkey, - o_orderdate, - o_shippriority, - sum(tmp) as revenue - from( - select - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp, - mongotpch.lineitem.l_orderkey as l_orderkey, - mongotpch.orders.o_orderdate as o_orderdate, - mongotpch.orders.o_shippriority as o_shippriority - from - tpchdata.customer - join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey - join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey - where - tpchdata.customer.c_mktsegment = 'BUILDING' - and mongotpch.orders.o_orderdate < 795196800000 - and mongotpch.lineitem.l_shipdate > 795225600000 - ) - group by - l_orderkey, - o_orderdate, - o_shippriority - ) -order by revenue desc - limit 10; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q5.sql b/test/src/test/resources/polybench/queries/q5.sql deleted file mode 100644 index 8fd43778d2..0000000000 --- a/test/src/test/resources/polybench/queries/q5.sql +++ /dev/null @@ -1,28 +0,0 @@ -select - nation.n_name, - revenue -from ( - select - nation.n_name, - sum(tmp) as revenue - from ( - select - nation.n_name, - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) as tmp - from - tpchdata.customer - join mongotpch.orders on tpchdata.customer.c_custkey = mongotpch.orders.o_custkey - join mongotpch.lineitem on mongotpch.lineitem.l_orderkey = mongotpch.orders.o_orderkey - join tpchdata.supplier on mongotpch.lineitem.l_suppkey = tpchdata.supplier.s_suppkey and tpchdata.customer.c_nationkey = tpchdata.supplier.s_nationkey - join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey - join tpchdata.region on nation.n_regionkey = tpchdata.region.r_regionkey - where - tpchdata.region.r_name = "ASIA" - and mongotpch.orders.o_orderdate >= 757353600000 - and mongotpch.orders.o_orderdate < 788889600000 - ) - group by - nation.n_name - ) -order by - revenue desc; \ No newline at end of file diff --git a/test/src/test/resources/polybench/queries/q6.sql b/test/src/test/resources/polybench/queries/q6.sql deleted file mode 100644 index 5408a94ff8..0000000000 --- a/test/src/test/resources/polybench/queries/q6.sql +++ /dev/null @@ -1,12 +0,0 @@ -select sum(tmp) from ( - select - l_extendedprice * l_discount as tmp - from - mongotpch.lineitem - where - mongotpch.lineitem.l_shipdate >= 757353600000 - and mongotpch.lineitem.l_shipdate < 788889600000 - and mongotpch.lineitem.l_discount >= 0.05 - and mongotpch.lineitem.l_discount <= 0.07 - and mongotpch.lineitem.l_quantity < 24 - ); diff --git a/test/src/test/resources/polybench/queries/q9.sql b/test/src/test/resources/polybench/queries/q9.sql deleted file mode 100644 index 319ba55ef9..0000000000 --- a/test/src/test/resources/polybench/queries/q9.sql +++ /dev/null @@ -1,32 +0,0 @@ -insert into tmpTableC(key, orderkey, year) values ( - select o_orderkey, extractYear(o_orderdate) from mongotpch.orders -); - -select * from ( - select - nation, - o_year, - sum(amount) as sum_profit - from ( - select - nation.n_name as nation, - tmpTableC.year as o_year, - mongotpch.lineitem.l_extendedprice * (1 - mongotpch.lineitem.l_discount) - tpchdata.partsupp.ps_supplycost * mongotpch.lineitem.l_quantity as amount - from - tpchdata.part - join mongotpch.lineitem on tpchdata.part.p_partkey = mongotpch.lineitem.l_partkey - join tpchdata.supplier on tpchdata.supplier.s_suppkey = mongotpch.lineitem.l_suppkey - join tpchdata.partsupp on tpchdata.partsupp.ps_suppkey = mongotpch.lineitem.l_suppkey and tpchdata.partsupp.ps_partkey = mongotpch.lineitem.l_partkey - join mongotpch.orders on mongotpch.orders.o_orderkey = mongotpch.lineitem.l_orderkey - join nation on tpchdata.supplier.s_nationkey = nation.n_nationkey - join tmpTableC on mongotpch.orders.o_orderkey = tmpTableC.orderkey - where - tpchdata.part.p_name like '.*green.*' - ) - group by - o_year, - nation - order by - o_year desc -) -order by nation; \ No newline at end of file diff --git a/test/src/test/resources/polybench/udf/udtf_extract_year.py b/test/src/test/resources/polybench/udf/udtf_extract_year.py deleted file mode 100644 index 8efda32e04..0000000000 --- a/test/src/test/resources/polybench/udf/udtf_extract_year.py +++ /dev/null @@ -1,28 +0,0 @@ -from datetime import datetime, timezone -import pytz -class UDFExtractYear: - def __init__(self): - pass - - def extractYear(self, num): - # Unix timestamp is in milliseconds - timestamp_in_seconds = num / 1000 - # TODO 直接将timestamp增加8小时 - tz = pytz.timezone('Asia/Shanghai') - dt = datetime.fromtimestamp(timestamp_in_seconds, tz=tz) - return float(dt.year) - def transform(self, data, args, kvargs): - res = self.buildHeader(data) - dateRow = [] - for num in data[2][1:]: - dateRow.append(self.extractYear(num)) - res.append(dateRow) - return res - - def buildHeader(self, data): - colNames = [] - colTypes = [] - for name in data[0][1:]: - colNames.append("extractYear(" + name + ")") - colTypes.append("DOUBLE") - return [colNames, colTypes] diff --git a/test/src/test/resources/tpch/queries/q1.sql b/test/src/test/resources/tpch/queries/q1.sql new file mode 100644 index 0000000000..c77305f28d --- /dev/null +++ b/test/src/test/resources/tpch/queries/q1.sql @@ -0,0 +1,31 @@ +select + lineitem.l_returnflag as l_returnflag, + lineitem.l_linestatus as l_linestatus, + sum(lineitem.l_quantity) as sum_qty, + sum(lineitem.l_extendedprice) as sum_base_price, + sum(tmp1) as sum_disc_price, + sum(tmp2) as sum_charge, + avg(lineitem.l_quantity) as avg_qty, + avg(lineitem.l_extendedprice) as avg_price, + avg(lineitem.l_discount) as avg_disc, + count(lineitem.l_returnflag) as count_order +from ( + select + l_returnflag, + l_linestatus, + l_quantity, + l_extendedprice, + l_discount, + l_extendedprice * (1 - l_discount) as tmp1, + l_extendedprice * (1 - l_discount) * (1 + l_tax) as tmp2 + from + lineitem + where + lineitem.l_shipdate <= 904694400000 + ) +group by + lineitem.l_returnflag, + lineitem.l_linestatus +order by + lineitem.l_returnflag, + lineitem.l_linestatus; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q10.sql b/test/src/test/resources/tpch/queries/q10.sql new file mode 100644 index 0000000000..2fdd79af91 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q10.sql @@ -0,0 +1,51 @@ +select + customer.c_custkey, + customer.c_name, + revenue, + customer.c_acctbal, + nation.n_name, + customer.c_address, + customer.c_phone, + customer.c_comment +from ( + select + customer.c_custkey, + customer.c_name, + sum(tmp) as revenue, + customer.c_acctbal, + nation.n_name, + customer.c_address, + customer.c_phone, + customer.c_comment + from ( + select + customer.c_custkey, + customer.c_name, + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp, + customer.c_acctbal, + nation.n_name, + customer.c_address, + customer.c_phone, + customer.c_comment + from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on lineitem.l_orderkey = orders.o_orderkey + join nation on customer.c_nationkey = nation.n_nationkey + where + orders.o_orderdate >= 749404800000 + and orders.o_orderdate < 757353600000 + and lineitem.l_returnflag = 'R' + ) + group by + customer.c_custkey, + customer.c_name, + customer.c_acctbal, + customer.c_phone, + nation.n_name, + customer.c_address, + customer.c_comment + ) +order by + revenue desc +limit 20; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q13.sql b/test/src/test/resources/tpch/queries/q13.sql new file mode 100644 index 0000000000..c40d34162c --- /dev/null +++ b/test/src/test/resources/tpch/queries/q13.sql @@ -0,0 +1,25 @@ +select + c_count, + custdist +from ( + select + c_count, + count(c_custkey) as custdist + from ( + select + customer.c_custkey as c_custkey, + count(orders.o_orderkey) as c_count + from + customer + left outer join orders + on customer.c_custkey = orders.o_custkey + and !(orders.o_comment like '.*pending.*') + group by + customer.c_custkey + ) + group by + c_count + ) +order by + custdist, + c_count desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q16.sql b/test/src/test/resources/tpch/queries/q16.sql new file mode 100644 index 0000000000..a19922c7c9 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q16.sql @@ -0,0 +1,49 @@ +select + p_brand, + p_type, + p_size, + supplier_cnt +from ( + select + part.p_brand as p_brand, + part.p_type as p_type, + part.p_size as p_size, + count(distinct partsupp.ps_suppkey) as supplier_cnt + from + partsupp + join part on part.p_partkey = partsupp.ps_partkey + where + part.p_brand <> 'Brand#45' + and part.p_partkey not in( + select p_partkey from part + where part.p_type like '.*MEDIUM POLISHED.*' + ) + and ( + part.p_size = 3 + or part.p_size = 9 + or part.p_size = 14 + or part.p_size = 19 + or part.p_size = 23 + or part.p_size = 36 + or part.p_size = 45 + or part.p_size = 49 + ) + and partsupp.ps_suppkey not in ( + select + s_suppkey + from + supplier + where + supplier.s_comment like '.*Customer.*Complaints.*' + ) + group by + part.p_brand, + part.p_type, + part.p_size + order by + part.p_brand, + part.p_type, + part.p_size + ) +order by + supplier_cnt desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q17.sql b/test/src/test/resources/tpch/queries/q17.sql new file mode 100644 index 0000000000..c573ec0e2d --- /dev/null +++ b/test/src/test/resources/tpch/queries/q17.sql @@ -0,0 +1,29 @@ +insert into tmpTableB(key, p_partkey, val) values ( + select + part.p_partkey, + 0.2 * tmp + from ( + select + part.p_partkey, + avg(lineitem.l_quantity) as tmp + from + lineitem + join part on lineitem.l_partkey = part.p_partkey + group by part.p_partkey + ) +); + +select + tmp2 / 7 as avg_yearly +from ( + select + sum(lineitem.l_extendedprice) as tmp2 + from + lineitem + join part on part.p_partkey = lineitem.l_partkey + join tmpTableB on tmpTableB.p_partkey = lineitem.l_partkey + where + part.p_brand = 'Brand#23' + and part.p_container = 'MED BOX' + and lineitem.l_quantity < tmpTableB.val + ); \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q18.sql b/test/src/test/resources/tpch/queries/q18.sql new file mode 100644 index 0000000000..aaa10ae7b3 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q18.sql @@ -0,0 +1,35 @@ +select + customer.c_name, + customer.c_custkey, + orders.o_orderkey, + orders.o_orderdate, + orders.o_totalprice, + sum(lineitem.l_quantity) +from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on orders.o_orderkey = lineitem.l_orderkey +where + orders.o_orderkey in ( + select + lineitem.l_orderkey + from ( + select + l_orderkey, + sum(l_quantity) + from + lineitem + group by + l_orderkey + having + sum(lineitem.l_quantity) > 300 + ) + ) +group by + customer.c_name, + customer.c_custkey, + orders.o_orderkey, + orders.o_orderdate, + orders.o_totalprice +order by + orders.o_totalprice desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q19.sql b/test/src/test/resources/tpch/queries/q19.sql new file mode 100644 index 0000000000..0af851b6a0 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q19.sql @@ -0,0 +1,48 @@ +select + sum(tmp) as revenue +from ( + select + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp + from + lineitem + join part on part.p_partkey = lineitem.l_partkey + where ( + part.p_brand = 'Brand#12' + and ( + part.p_container = 'SM CASE' + or part.p_container = 'SM BOX' + or part.p_container = 'SM PACK' + or part.p_container = 'SM PKG' + ) + and lineitem.l_quantity >= 1 and lineitem.l_quantity <= 11 + and part.p_size >= 1 and part.p_size <= 5 + and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG') + and lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + or ( + part.p_brand = 'Brand#23' + and ( + part.p_container = 'MED PKG' + or part.p_container = 'MED BOX' + or part.p_container = 'MED BAG' + or part.p_container = 'MED PACK' + ) + and lineitem.l_quantity >= 10 and lineitem.l_quantity <= 20 + and part.p_size >= 1 and part.p_size <= 10 + and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG') + and lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + or ( + part.p_brand = 'Brand#34' + and ( + part.p_container = 'LG PACK' + or part.p_container = 'LG BOX' + or part.p_container = 'LG CASE' + or part.p_container = 'LG PKG' + ) + and lineitem.l_quantity >= 20 and lineitem.l_quantity <= 30 + and part.p_size >= 1 and part.p_size <= 15 + and (lineitem.l_shipmode = 'AIR' or lineitem.l_shipmode = 'AIR REG') + and lineitem.l_shipinstruct = 'DELIVER IN PERSON' + ) + ); \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q2.sql b/test/src/test/resources/tpch/queries/q2.sql new file mode 100644 index 0000000000..a5b56e3267 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q2.sql @@ -0,0 +1,44 @@ +insert into tmpTable(key, p_key, minCost) values ( +select + part.p_partkey as p_key, + min(partsupp.ps_supplycost) as minCost +from + partsupp + join supplier on supplier.s_suppkey = partsupp.ps_suppkey + join nation on supplier.s_nationkey = nation.n_nationkey + join region on nation.n_regionkey = region.r_regionkey + join part on part.p_partkey = partsupp.ps_partkey +where + region.r_name = 'EUROPE' + and part.p_size = 15 + and part.p_type like "^.*BRASS" +group by part.p_partkey +); + +select s_acctbal, s_name, n_name, p_partkey, p_mfgr, s_address, s_phone, s_comment from + (select + supplier.s_acctbal as s_acctbal, + supplier.s_name as s_name, + nation.n_name as n_name, + part.p_partkey as p_partkey, + part.p_mfgr as p_mfgr, + supplier.s_address as s_address, + supplier.s_phone as s_phone, + supplier.s_comment as s_comment + from + part + join partsupp on part.p_partkey = partsupp.ps_partkey + join supplier on supplier.s_suppkey = partsupp.ps_suppkey + join nation on supplier.s_nationkey = nation.n_nationkey + join region on nation.n_regionkey = region.r_regionkey + join tmpTable on tmpTable.p_key = part.p_partkey and partsupp.ps_supplycost = tmpTable.minCost + where + part.p_size = 15 + and region.r_name = 'EUROPE' + and part.p_type like "^.*BRASS" + order by + nation.n_name, + supplier.s_name, + part.p_partkey + ) +order by s_acctbal desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q20.sql b/test/src/test/resources/tpch/queries/q20.sql new file mode 100644 index 0000000000..4671b79f8a --- /dev/null +++ b/test/src/test/resources/tpch/queries/q20.sql @@ -0,0 +1,45 @@ +insert into tmpTableA(key, partkey, suppkey, val) values ( + select + partkey, + suppkey, + 0.5 * tmp from( + select + l_partkey as partkey, + l_suppkey as suppkey, + sum(l_quantity) as tmp + from + lineitem + where + lineitem.l_shipdate >= 757353600000 + and lineitem.l_shipdate < 788889600000 + group by l_partkey, l_suppkey + ) +); + +select + supplier.s_name, + supplier.s_address +from + supplier + join nation on supplier.s_nationkey = nation.n_nationkey +where + supplier.s_suppkey in ( + select + partsupp.ps_suppkey + from + partsupp + join tmpTableA on tmpTableA.suppkey = partsupp.ps_suppkey and tmpTableA.partkey = partsupp.ps_partkey + where + partsupp.ps_partkey in ( + select + p_partkey + from + part + where + part.p_name like 'forest.*' + ) + and partsupp.ps_availqty > tmpTableA.val + ) + and nation.n_name = 'CANADA' +order by + supplier.s_name; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q3.sql b/test/src/test/resources/tpch/queries/q3.sql new file mode 100644 index 0000000000..81a232a712 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q3.sql @@ -0,0 +1,31 @@ +select l_orderkey, + revenue, + o_orderdate, + o_shippriority from + (select + l_orderkey, + o_orderdate, + o_shippriority, + sum(tmp) as revenue + from( + select + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp, + lineitem.l_orderkey as l_orderkey, + orders.o_orderdate as o_orderdate, + orders.o_shippriority as o_shippriority + from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on lineitem.l_orderkey = orders.o_orderkey + where + customer.c_mktsegment = 'BUILDING' + and orders.o_orderdate < 795196800000 + and lineitem.l_shipdate > 795225600000 + ) + group by + l_orderkey, + o_orderdate, + o_shippriority + ) +order by revenue desc + limit 10; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q5.sql b/test/src/test/resources/tpch/queries/q5.sql new file mode 100644 index 0000000000..197cced143 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q5.sql @@ -0,0 +1,28 @@ +select + nation.n_name, + revenue +from ( + select + nation.n_name, + sum(tmp) as revenue + from ( + select + nation.n_name, + lineitem.l_extendedprice * (1 - lineitem.l_discount) as tmp + from + customer + join orders on customer.c_custkey = orders.o_custkey + join lineitem on lineitem.l_orderkey = orders.o_orderkey + join supplier on lineitem.l_suppkey = supplier.s_suppkey and customer.c_nationkey = supplier.s_nationkey + join nation on supplier.s_nationkey = nation.n_nationkey + join region on nation.n_regionkey = region.r_regionkey + where + region.r_name = "ASIA" + and orders.o_orderdate >= 757353600000 + and orders.o_orderdate < 788889600000 + ) + group by + nation.n_name + ) +order by + revenue desc; \ No newline at end of file diff --git a/test/src/test/resources/tpch/queries/q6.sql b/test/src/test/resources/tpch/queries/q6.sql new file mode 100644 index 0000000000..3e9c5a4b65 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q6.sql @@ -0,0 +1,12 @@ +select sum(tmp) from ( + select + l_extendedprice * l_discount as tmp + from + lineitem + where + lineitem.l_shipdate >= 757353600000 + and lineitem.l_shipdate < 788889600000 + and lineitem.l_discount >= 0.05 + and lineitem.l_discount <= 0.07 + and lineitem.l_quantity < 24 + ); diff --git a/test/src/test/resources/tpch/queries/q9.sql b/test/src/test/resources/tpch/queries/q9.sql new file mode 100644 index 0000000000..311fb02ad7 --- /dev/null +++ b/test/src/test/resources/tpch/queries/q9.sql @@ -0,0 +1,32 @@ +insert into tmpTableC(key, orderkey, year) values ( + select o_orderkey, extractYear(o_orderdate) from orders +); + +select * from ( + select + nation, + o_year, + sum(amount) as sum_profit + from ( + select + nation.n_name as nation, + tmpTableC.year as o_year, + lineitem.l_extendedprice * (1 - lineitem.l_discount) - partsupp.ps_supplycost * lineitem.l_quantity as amount + from + part + join lineitem on part.p_partkey = lineitem.l_partkey + join supplier on supplier.s_suppkey = lineitem.l_suppkey + join partsupp on partsupp.ps_suppkey = lineitem.l_suppkey and partsupp.ps_partkey = lineitem.l_partkey + join orders on orders.o_orderkey = lineitem.l_orderkey + join nation on supplier.s_nationkey = nation.n_nationkey + join tmpTableC on orders.o_orderkey = tmpTableC.orderkey + where + part.p_name like '.*green.*' + ) + group by + o_year, + nation + order by + o_year desc +) +order by nation; \ No newline at end of file diff --git a/test/src/test/resources/polybench/sf0.1/q1.csv b/test/src/test/resources/tpch/sf0.1/q1.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q1.csv rename to test/src/test/resources/tpch/sf0.1/q1.csv diff --git a/test/src/test/resources/polybench/sf0.1/q10.csv b/test/src/test/resources/tpch/sf0.1/q10.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q10.csv rename to test/src/test/resources/tpch/sf0.1/q10.csv diff --git a/test/src/test/resources/polybench/sf0.1/q13.csv b/test/src/test/resources/tpch/sf0.1/q13.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q13.csv rename to test/src/test/resources/tpch/sf0.1/q13.csv diff --git a/test/src/test/resources/polybench/sf0.1/q16.csv b/test/src/test/resources/tpch/sf0.1/q16.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q16.csv rename to test/src/test/resources/tpch/sf0.1/q16.csv diff --git a/test/src/test/resources/polybench/sf0.1/q17.csv b/test/src/test/resources/tpch/sf0.1/q17.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q17.csv rename to test/src/test/resources/tpch/sf0.1/q17.csv diff --git a/test/src/test/resources/polybench/sf0.1/q18.csv b/test/src/test/resources/tpch/sf0.1/q18.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q18.csv rename to test/src/test/resources/tpch/sf0.1/q18.csv diff --git a/test/src/test/resources/polybench/sf0.1/q19.csv b/test/src/test/resources/tpch/sf0.1/q19.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q19.csv rename to test/src/test/resources/tpch/sf0.1/q19.csv diff --git a/test/src/test/resources/polybench/sf0.1/q2.csv b/test/src/test/resources/tpch/sf0.1/q2.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q2.csv rename to test/src/test/resources/tpch/sf0.1/q2.csv diff --git a/test/src/test/resources/polybench/sf0.1/q20.csv b/test/src/test/resources/tpch/sf0.1/q20.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q20.csv rename to test/src/test/resources/tpch/sf0.1/q20.csv diff --git a/test/src/test/resources/polybench/sf0.1/q3.csv b/test/src/test/resources/tpch/sf0.1/q3.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q3.csv rename to test/src/test/resources/tpch/sf0.1/q3.csv diff --git a/test/src/test/resources/polybench/sf0.1/q5.csv b/test/src/test/resources/tpch/sf0.1/q5.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q5.csv rename to test/src/test/resources/tpch/sf0.1/q5.csv diff --git a/test/src/test/resources/polybench/sf0.1/q6.csv b/test/src/test/resources/tpch/sf0.1/q6.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q6.csv rename to test/src/test/resources/tpch/sf0.1/q6.csv diff --git a/test/src/test/resources/polybench/sf0.1/q9.csv b/test/src/test/resources/tpch/sf0.1/q9.csv similarity index 100% rename from test/src/test/resources/polybench/sf0.1/q9.csv rename to test/src/test/resources/tpch/sf0.1/q9.csv From 56d12b39d48279b440cc677dc030b16ce545f695 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 17:21:21 +0800 Subject: [PATCH 169/229] remove polystore test --- .../workflows/polystore-benchmark-test.yml | 253 ------------------ .github/workflows/standard-test-suite.yml | 10 +- .github/workflows/tpc-h.yml | 2 +- 3 files changed, 6 insertions(+), 259 deletions(-) delete mode 100644 .github/workflows/polystore-benchmark-test.yml diff --git a/.github/workflows/polystore-benchmark-test.yml b/.github/workflows/polystore-benchmark-test.yml deleted file mode 100644 index b49ab5e3a4..0000000000 --- a/.github/workflows/polystore-benchmark-test.yml +++ /dev/null @@ -1,253 +0,0 @@ -name: PolyStore Benchmark Tests - -on: - pull_request: # when a PR is opened or reopened - types: [opened, reopened] - -env: - VERSION: 0.7.0-SNAPSHOT -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - PolyStore-Benchmark-Test: - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - java: [8] - python-version: ["3.9"] - os: [ubuntu-latest, macOS-13, windows-latest] - metadata: [zookeeper] - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - - name: Environment dependence - uses: ./.github/actions/dependence - with: - python-version: ${{ matrix.python-version }} - java: ${{ matrix.java }} - - - name: Display System Info - shell: bash - run: | - echo "Operating System: $(uname -a 2>/dev/null || ver)" - echo "Architecture: $(uname -m 2>/dev/null || echo %PROCESSOR_ARCHITECTURE%)" - echo "Java Version:" - java -version - echo "Python Version:" - python --version - echo "CPU Info:" - if [ "$(uname)" = "Linux" ]; then - lscpu - elif [ "$(uname)" = "Darwin" ]; then - sysctl -n machdep.cpu.brand_string - else - wmic cpu get name - fi - echo "Memory Info:" - if [ "$(uname)" = "Linux" ]; then - free -h - elif [ "$(uname)" = "Darwin" ]; then - vm_stat - else - systeminfo | findstr /C:"Total Physical Memory" - fi - - - name: generate tpch data - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner - - - name: Run DB - uses: ./.github/actions/dbRunner - with: - DB-name: "Parquet" - - - name: Run PostgreSQL - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/postgres.sh" - - - name: Run MongoDB - shell: bash - run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/mongodb.sh" - - - name: Install IGinX with Maven - shell: bash - run: | - mvn clean package -DskipTests -P-format -q - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - - # start udf path test first to avoid being effected - - name: Start IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - - - name: set client test context - uses: ./.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./.github/actions/context - with: - DB-name: "Parquet" - shell: client-before - - - name: Start storing data - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHDataInsertionIT -DfailIfNoTests=false -P-format - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/parquet.sh" - - - name: Run session test - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format - - - name: Get old version - if: always() - shell: bash - run: | - git clone https://github.com/IGinX-THU/IGinX.git - cd IGinX - mvn clean package -DskipTests -P-format - - - name: Stop ZooKeeper - uses: ./.github/actions/zookeeperRunner - with: - if-stop: "true" - - - name: Stop IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-stop: "true" - - - name: Rerun ZooKeeper - uses: ./.github/actions/zookeeperRunner - with: - if-rerun: "true" - - - name: Run DB - shell: bash - run: | - cp -f "IGinX/conf/config.properties" "IGinX/conf/config.properties.bak" - cd IGinX - if [[ "$RUNNER_OS" == "Linux" || "$RUNNER_OS" == "Windows" ]]; then - chmod +x ".github/scripts/dataSources/parquet_linux_windows.sh" - ".github/scripts/dataSources/parquet_linux_windows.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/dataSources/parquet_macos.sh" - ".github/scripts/dataSources/parquet_macos.sh" 6667 6888 test/mn test/iginx_mn false false conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - - - name: Change IGinX config - uses: ./.github/actions/confWriter - with: - DB-name: "Parquet" - Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "IGinX" - - # start udf path test first to avoid being effected - - - name: Start IGinX - shell: bash - run: | - cd IGinX - pwd - if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' core/target/iginx-core-${VERSION}/conf/config.properties - elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' core/target/iginx-core-${VERSION}/conf/config.properties - else - echo "$RUNNER_OS is not supported" - exit 1 - fi - cd .. - rm ./core/target/iginx-core-0.7.0-SNAPSHOT/udf_funcs/python_scripts/udtf_extract_year.py - chmod +x ".github/scripts/iginx/iginx_regression_udf_path.sh" - ".github/scripts/iginx/iginx_regression_udf_path.sh" - - - name: set client test context - uses: ./IGinX/.github/actions/context - with: - work-name: restart-iginx-meta - metadata: ${{ matrix.metadata }} - - - name: set client test context - uses: ./IGinX/.github/actions/context - with: - name: Pre Test Client Export File - shell: bash - run: | - cd IGinX - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x ".github/scripts/test/cli/test_outfile.sh" - ".github/scripts/test/cli/test_outfile.sh" "Parquet - elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_windows.sh" - ".github/scripts/test/cli/test_outfile_windows.sh" "Parquet" - elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x ".github/scripts/test/cli/test_outfile_macos.sh" - ".github/scripts/test/cli/test_outfile_macos.sh" "Parquet" - fi - - - name: Start storing data - if: always() - shell: bash - run: | - pwd - ls - mkdir IGinX/tpc - mkdir "IGinX/tpc/TPC-H V3.0.1" - mkdir "IGinX/tpc/TPC-H V3.0.1/data" - cp "tpc/TPC-H V3.0.1/data/nation.csv" "IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - cd IGinX - output_file="../IGinX/tpc/TPC-H V3.0.1/data/nation.csv" - mv "../test/src/test/resources/polybench/udf/udtf_extract_year.py" "udtf_extract_year.py" - chmod 777 "udtf_extract_year.py" - COMMAND1='LOAD DATA FROM INFILE "tpc/TPC-H V3.0.1/data/nation.csv" AS CSV INTO nation(key, n_nationkey, n_name, n_regionkey, n_comment); CREATE FUNCTION UDTF "extractYear" FROM "UDFExtractYear" IN "udtf_extract_year.py";' - SCRIPT_COMMAND="bash client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e '{}'" - bash -c "chmod +x client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh" - - if [ "$RUNNER_OS" = "Linux" ]; then - bash -c "echo '$COMMAND1' | xargs -0 -t -i ${SCRIPT_COMMAND}" - elif [ "$RUNNER_OS" = "Windows" ]; then - bash -c "client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.bat -e '$COMMAND1'" - elif [ "$RUNNER_OS" = "macOS" ]; then - sh -c "echo '$COMMAND1' | xargs -0 -t -I F sh client/target/iginx-client-0.7.0-SNAPSHOT/sbin/start_cli.sh -e 'F'" - fi - - - name: Run session test - if: always() - shell: bash - run: | - mvn test -q -Dtest=TPCHRunner -DfailIfNoTests=false -P-format diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 3ce96b4682..bb89b09d67 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -19,6 +19,11 @@ jobs: uses: ./.github/workflows/case-regression.yml with: metadata-matrix: '["zookeeper"]' + tpc-h-regression-test: + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' standalone-test: uses: ./.github/workflows/standalone-test.yml with: @@ -37,8 +42,3 @@ jobs: metadata-matrix: '["zookeeper"]' assemebly-test: uses: ./.github/workflows/assembly-test.yml - tpc-h: - uses: ./.github/workflows/tpc-h.yml - with: - os-matrix: '["ubuntu-latest"]' - metadata-matrix: '["zookeeper"]' diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index cc16b9f0bc..bf0ed44fd5 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -1,4 +1,4 @@ -name: "TPC-H Test" +name: "TPC-H Regression Test" on: workflow_call: From 359515618406946aeecb18ce3a6d726126f9f496 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 17:36:37 +0800 Subject: [PATCH 170/229] format --- .github/workflows/standard-test-suite.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index bb89b09d67..5b6d0e89ff 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -20,10 +20,10 @@ jobs: with: metadata-matrix: '["zookeeper"]' tpc-h-regression-test: - uses: ./.github/workflows/tpc-h.yml - with: - os-matrix: '["ubuntu-latest"]' - metadata-matrix: '["zookeeper"]' + uses: ./.github/workflows/tpc-h.yml + with: + os-matrix: '["ubuntu-latest"]' + metadata-matrix: '["zookeeper"]' standalone-test: uses: ./.github/workflows/standalone-test.yml with: From 89a3d0582477834c9f2143361446312306944c42 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 18:27:22 +0800 Subject: [PATCH 171/229] debug --- .github/workflows/standard-test-suite.yml | 52 +++++++++++------------ .github/workflows/tpc-h.yml | 11 +++-- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 5b6d0e89ff..ccb6e6bdb7 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -11,34 +11,34 @@ concurrency: cancel-in-progress: true jobs: - unit-test: - uses: ./.github/workflows/unit-test.yml - unit-mds: - uses: ./.github/workflows/unit-mds.yml - case-regression: - uses: ./.github/workflows/case-regression.yml - with: - metadata-matrix: '["zookeeper"]' + # unit-test: + # uses: ./.github/workflows/unit-test.yml + # unit-mds: + # uses: ./.github/workflows/unit-mds.yml + # case-regression: + # uses: ./.github/workflows/case-regression.yml + # with: + # metadata-matrix: '["zookeeper"]' tpc-h-regression-test: uses: ./.github/workflows/tpc-h.yml with: os-matrix: '["ubuntu-latest"]' metadata-matrix: '["zookeeper"]' - standalone-test: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test-pushdown: - uses: ./.github/workflows/standalone-test-pushdown.yml - with: - metadata-matrix: '["zookeeper"]' - db-ce: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - remote-test: - uses: ./.github/workflows/remote-test.yml - with: - metadata-matrix: '["zookeeper"]' - assemebly-test: - uses: ./.github/workflows/assembly-test.yml +# standalone-test: +# uses: ./.github/workflows/standalone-test.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test-pushdown: +# uses: ./.github/workflows/standalone-test-pushdown.yml +# with: +# metadata-matrix: '["zookeeper"]' +# db-ce: +# uses: ./.github/workflows/DB-CE.yml +# with: +# metadata-matrix: '["zookeeper"]' +# remote-test: +# uses: ./.github/workflows/remote-test.yml +# with: +# metadata-matrix: '["zookeeper"]' +# assemebly-test: +# uses: ./.github/workflows/assembly-test.yml diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index bf0ed44fd5..28290d8513 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -96,6 +96,9 @@ jobs: mkdir "new" mkdir "new/IGinX" cp -r "IGinX" "new/IGinX" + cd "IGinX" + mkdir "new" + mv "../new" "new" mkdir "old" cd old git clone https://github.com/IGinX-THU/IGinX.git @@ -111,13 +114,13 @@ jobs: with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "../old/IGinX" + Root-Dir-Path: "old/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner with: version: ${VERSION} - Root-Dir-Path: "../old/IGinX" + Root-Dir-Path: "old/IGinX" - name: Run Regression Test on Old IGinX if: always() @@ -151,13 +154,13 @@ jobs: with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "../new/IGinX" + Root-Dir-Path: "new/IGinX" - name: Start New IGinX uses: ./.github/actions/iginxRunner with: version: ${VERSION} - Root-Dir-Path: "../new/IGinX" + Root-Dir-Path: "new/IGinX" - name: Run Regression Test on New IGinX if: always() From d34c17efc8f0376c44a72d2847688b05af031516 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 18:36:46 +0800 Subject: [PATCH 172/229] debug --- .github/actions/confWriter/action.yml | 2 ++ .github/workflows/tpc-h.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 7bcd6f0338..ecb6788e25 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -77,6 +77,8 @@ runs: - name: Change UDF conf shell: bash run: | + pwd + ls if [ "$RUNNER_OS" == "Linux" ]; then sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 28290d8513..6c67a618e7 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -103,6 +103,7 @@ jobs: cd old git clone https://github.com/IGinX-THU/IGinX.git cd IGinX + pwd - name: Install IGinX with Maven shell: bash From f8a9c9e6a508dc173773eec0f52cb03855d6e52a Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 18:45:22 +0800 Subject: [PATCH 173/229] debug --- .github/workflows/tpc-h.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 6c67a618e7..9cb4b37283 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -115,13 +115,13 @@ jobs: with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "old/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/old/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner with: version: ${VERSION} - Root-Dir-Path: "old/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/old/IGinX" - name: Run Regression Test on Old IGinX if: always() @@ -155,13 +155,13 @@ jobs: with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "new/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/new/IGinX" - name: Start New IGinX uses: ./.github/actions/iginxRunner with: version: ${VERSION} - Root-Dir-Path: "new/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/new/IGinX" - name: Run Regression Test on New IGinX if: always() From 336cbf44faa87173ac483c6cec84b1f8500a46ae Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 18:57:09 +0800 Subject: [PATCH 174/229] debug --- .github/workflows/tpc-h.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 9cb4b37283..6a4b44a147 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -108,7 +108,15 @@ jobs: - name: Install IGinX with Maven shell: bash run: | + cd "core" + pwd + ls + cd .. mvn clean package -DskipTests -P-format -q + cd "core" + pwd + ls + cd .. - name: Change IGinX config uses: ./.github/actions/confWriter From 071f8792bc5783b0e4b460e8f88b1355e1ce7be4 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 19:09:05 +0800 Subject: [PATCH 175/229] debug --- .github/actions/confWriter/action.yml | 2 -- .github/workflows/tpc-h.yml | 17 +++++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index ecb6788e25..7bcd6f0338 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -77,8 +77,6 @@ runs: - name: Change UDF conf shell: bash run: | - pwd - ls if [ "$RUNNER_OS" == "Linux" ]; then sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 6a4b44a147..7455894f7b 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -103,20 +103,12 @@ jobs: cd old git clone https://github.com/IGinX-THU/IGinX.git cd IGinX - pwd - - name: Install IGinX with Maven + - name: Install Old IGinX with Maven shell: bash run: | - cd "core" - pwd - ls - cd .. + cd "old/IGinX" mvn clean package -DskipTests -P-format -q - cd "core" - pwd - ls - cd .. - name: Change IGinX config uses: ./.github/actions/confWriter @@ -135,6 +127,8 @@ jobs: if: always() shell: bash run: | + pwd + ls mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format - name: Stop ZooKeeper @@ -157,6 +151,7 @@ jobs: shell: bash run: | cd "../../new/IGinX" + mvn clean package -DskipTests -P-format -q - name: Change IGinX config uses: ./.github/actions/confWriter @@ -175,4 +170,6 @@ jobs: if: always() shell: bash run: | + pwd + ls mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format From b02620e6b2ad9c23d4e8c73e656664f292cc87da Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 19:32:40 +0800 Subject: [PATCH 176/229] debug --- .github/workflows/standard-test-suite.yml | 12 ++++--- .github/workflows/tpc-h.yml | 40 ++++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index ccb6e6bdb7..b8b7c66a53 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -1,10 +1,12 @@ name: Standard Test Suite -on: - pull_request: # when a PR is opened or reopened - types: [opened, reopened] - branches: - - main +#on: +# pull_request: # when a PR is opened or reopened +# types: [opened, reopened] +# branches: +# - main + +on: [pull_request] concurrency: group: "${{ github.workflow }}-${{ github.ref }}" diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 7455894f7b..79b130e20d 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -89,7 +89,7 @@ jobs: with: DB-name: ${{ matrix.DB-name }} - - name: Switch to Old Version + - name: Get Old Version shell: bash run: | cd .. @@ -118,17 +118,22 @@ jobs: Root-Dir-Path: "${GITHUB_WORKSPACE}/old/IGinX" - name: Start Old IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - Root-Dir-Path: "${GITHUB_WORKSPACE}/old/IGinX" + if: always() + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 + else + echo "$RUNNER_OS is not supported in this test" + exit 1 + fi - name: Run Regression Test on Old IGinX if: always() shell: bash run: | - pwd - ls + cd "old/IGinX" mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format - name: Stop ZooKeeper @@ -147,10 +152,10 @@ jobs: with: if-rerun: "true" - - name: Switch to New Version + - name: Install New IGinX with Maven shell: bash run: | - cd "../../new/IGinX" + cd "new/IGinX" mvn clean package -DskipTests -P-format -q - name: Change IGinX config @@ -161,15 +166,20 @@ jobs: Root-Dir-Path: "${GITHUB_WORKSPACE}/new/IGinX" - name: Start New IGinX - uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - Root-Dir-Path: "${GITHUB_WORKSPACE}/new/IGinX" + if: always() + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + chmod +x "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 + else + echo "$RUNNER_OS is not supported in this test" + exit 1 + fi - name: Run Regression Test on New IGinX if: always() shell: bash run: | - pwd - ls + cd "new/IGinX" mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format From 0af9c75aed019cc8ce7e1527b5747f10153de30c Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 19:42:42 +0800 Subject: [PATCH 177/229] debug --- .github/scripts/iginx/iginx.sh | 4 ++++ .github/workflows/tpc-h.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 9091b0241c..36cb1a76c3 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,6 +20,10 @@ set -e +cd $3 + +pwd + sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" core/target/iginx-core-*/conf/config.properties diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 79b130e20d..bf6d9ad84c 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -123,7 +123,7 @@ jobs: run: | if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 + "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 old/IGinX else echo "$RUNNER_OS is not supported in this test" exit 1 @@ -171,7 +171,7 @@ jobs: run: | if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 + "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 new/IGinX else echo "$RUNNER_OS is not supported in this test" exit 1 From 64ec73651f27f6402cfcc1017392177fa6558a12 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 19:54:07 +0800 Subject: [PATCH 178/229] Update iginx.sh --- .github/scripts/iginx/iginx.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 36cb1a76c3..d5dddb474b 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,9 +20,9 @@ set -e -cd $3 +sh -c "cd $3" -pwd +sh -c "pwd" sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties From 650d95a6e2326367bcd3ec10ec45460e53bf35de Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 20:00:56 +0800 Subject: [PATCH 179/229] Update iginx.sh --- .github/scripts/iginx/iginx.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index d5dddb474b..068bc94d67 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,23 +20,19 @@ set -e -sh -c "cd $3" +sed -i "s/port=[0-9]\+/port=$1/g" $3/core/target/iginx-core-*/conf/config.properties -sh -c "pwd" +sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" $3/core/target/iginx-core-*/conf/config.properties -sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties +sed -i "s/restPort=[0-9]\+/restPort=$2/g" $3/core/target/iginx-core-*/conf/config.properties -sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" core/target/iginx-core-*/conf/config.properties +sh -c "chmod +x $3/core/target/iginx-core-*/sbin/start_iginx.sh" -sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.properties - -sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" - -sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" +sh -c "nohup $3/core/target/iginx-core-*/sbin/start_iginx.sh > $3/iginx-$1.log 2>&1 &" sh -c "sleep 3" -log_file="iginx-$1.log" +log_file="$3/iginx-$1.log" timeout=30 interval=2 From 9db3f9b6d8a2f9166c7175ad999108691eb09e12 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 20:04:12 +0800 Subject: [PATCH 180/229] Update iginx.sh --- .github/scripts/iginx/iginx.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 068bc94d67..0066d14277 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,11 +20,11 @@ set -e -sed -i "s/port=[0-9]\+/port=$1/g" $3/core/target/iginx-core-*/conf/config.properties +sed -i "s/port=[0-9]\+/port=$1/g" '$3'/core/target/iginx-core-*/conf/config.properties -sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" $3/core/target/iginx-core-*/conf/config.properties +sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" '$3'/core/target/iginx-core-*/conf/config.properties -sed -i "s/restPort=[0-9]\+/restPort=$2/g" $3/core/target/iginx-core-*/conf/config.properties +sed -i "s/restPort=[0-9]\+/restPort=$2/g" '$3'/core/target/iginx-core-*/conf/config.properties sh -c "chmod +x $3/core/target/iginx-core-*/sbin/start_iginx.sh" From a5059174e6ef72933110ccb55cb17c0e08f2da6f Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 20:08:37 +0800 Subject: [PATCH 181/229] Update iginx.sh --- .github/scripts/iginx/iginx.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 0066d14277..b825f5678e 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,19 +20,21 @@ set -e -sed -i "s/port=[0-9]\+/port=$1/g" '$3'/core/target/iginx-core-*/conf/config.properties +cd "$3" -sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" '$3'/core/target/iginx-core-*/conf/config.properties +sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties -sed -i "s/restPort=[0-9]\+/restPort=$2/g" '$3'/core/target/iginx-core-*/conf/config.properties +sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" core/target/iginx-core-*/conf/config.properties -sh -c "chmod +x $3/core/target/iginx-core-*/sbin/start_iginx.sh" +sed -i "s/restPort=[0-9]\+/restPort=$2/g" core/target/iginx-core-*/conf/config.properties -sh -c "nohup $3/core/target/iginx-core-*/sbin/start_iginx.sh > $3/iginx-$1.log 2>&1 &" +sh -c "chmod +x core/target/iginx-core-*/sbin/start_iginx.sh" + +sh -c "nohup core/target/iginx-core-*/sbin/start_iginx.sh > iginx-$1.log 2>&1 &" sh -c "sleep 3" -log_file="$3/iginx-$1.log" +log_file="iginx-$1.log" timeout=30 interval=2 From 8d290e499d69f3955ef73fc6e9025b4610e60e16 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 20:19:10 +0800 Subject: [PATCH 182/229] debug --- .github/scripts/iginx/iginx.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index b825f5678e..807f247da3 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -22,6 +22,10 @@ set -e cd "$3" +echo "$PWD" + +echo "$3" + sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" core/target/iginx-core-*/conf/config.properties From d3fdfd87df29381fe46f7cf7126e6c20e583e9ee Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 20:30:05 +0800 Subject: [PATCH 183/229] debug --- .github/scripts/iginx/iginx.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index 807f247da3..e31336c94a 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,11 +20,11 @@ set -e -cd "$3" +cd $3 -echo "$PWD" +echo $PWD -echo "$3" +echo $3 sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties From 36f5a95d16f5ffb1b99bfddef272a946db7c19d1 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 5 Jul 2024 20:46:56 +0800 Subject: [PATCH 184/229] debug --- .github/scripts/iginx/iginx.sh | 6 ------ .github/workflows/tpc-h.yml | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/scripts/iginx/iginx.sh b/.github/scripts/iginx/iginx.sh index e31336c94a..9091b0241c 100644 --- a/.github/scripts/iginx/iginx.sh +++ b/.github/scripts/iginx/iginx.sh @@ -20,12 +20,6 @@ set -e -cd $3 - -echo $PWD - -echo $3 - sed -i "s/port=[0-9]\+/port=$1/g" core/target/iginx-core-*/conf/config.properties sed -i "s/#iginx_port=[0-9]\+#/#iginx_port=$1#/g" core/target/iginx-core-*/conf/config.properties diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index bf6d9ad84c..d17f017fef 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -121,6 +121,7 @@ jobs: if: always() shell: bash run: | + cd "old/IGinX" if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 old/IGinX From 5cf08a9d636b0a9fa135bd239d998bf0edae4fb7 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sat, 6 Jul 2024 14:04:39 +0800 Subject: [PATCH 185/229] debug --- .github/actions/iginxRunner/action.yml | 33 ++++++++++------------ .github/workflows/tpc-h.yml | 38 ++++++++++---------------- 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index d8927c81ca..859354faf2 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -9,11 +9,6 @@ inputs: description: "to test UDF path detection" required: false default: "false" - Root-Dir-Path: - description: "the path of IGinX root directory" - required: false - default: "${GITHUB_WORKSPACE}" - runs: using: "composite" # Mandatory parameter steps: @@ -22,37 +17,37 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 fi - chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_udf_path.sh" - "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_udf_path.sh" ${VERSION} + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" ${VERSION} - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' name: Start IGinX shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx.sh" - "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx.sh" 6888 6666 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_windows.sh" - "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_macos.sh" - "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' name: Stop IGinX shell: bash run: | - chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_kill.sh" - "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_kill.sh" + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index d17f017fef..9abff01ef0 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -92,22 +92,8 @@ jobs: - name: Get Old Version shell: bash run: | - cd .. - mkdir "new" - mkdir "new/IGinX" - cp -r "IGinX" "new/IGinX" - cd "IGinX" - mkdir "new" - mv "../new" "new" - mkdir "old" - cd old git clone https://github.com/IGinX-THU/IGinX.git cd IGinX - - - name: Install Old IGinX with Maven - shell: bash - run: | - cd "old/IGinX" mvn clean package -DskipTests -P-format -q - name: Change IGinX config @@ -115,16 +101,16 @@ jobs: with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "${GITHUB_WORKSPACE}/old/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX if: always() shell: bash run: | - cd "old/IGinX" + cd "IGinX" if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/old/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 old/IGinX + chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 else echo "$RUNNER_OS is not supported in this test" exit 1 @@ -134,7 +120,11 @@ jobs: if: always() shell: bash run: | - cd "old/IGinX" + pwd + ls + cd IGinX + pwd + ls mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format - name: Stop ZooKeeper @@ -156,7 +146,8 @@ jobs: - name: Install New IGinX with Maven shell: bash run: | - cd "new/IGinX" + pwd + ls mvn clean package -DskipTests -P-format -q - name: Change IGinX config @@ -171,8 +162,8 @@ jobs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/new/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 new/IGinX + chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 else echo "$RUNNER_OS is not supported in this test" exit 1 @@ -182,5 +173,6 @@ jobs: if: always() shell: bash run: | - cd "new/IGinX" + pwd + ls mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format From 29d0eb4d00553ae2a0734e26cde44e4058906aeb Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sat, 6 Jul 2024 14:17:37 +0800 Subject: [PATCH 186/229] debug --- .github/workflows/tpc-h.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 9abff01ef0..ebd359186d 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -110,7 +110,7 @@ jobs: cd "IGinX" if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" else echo "$RUNNER_OS is not supported in this test" exit 1 @@ -155,7 +155,6 @@ jobs: with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "${GITHUB_WORKSPACE}/new/IGinX" - name: Start New IGinX if: always() @@ -163,7 +162,7 @@ jobs: run: | if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" else echo "$RUNNER_OS is not supported in this test" exit 1 From 3f82032caedf510af6156875b6fc45a9ecb2695f Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sat, 6 Jul 2024 14:25:04 +0800 Subject: [PATCH 187/229] debug --- .github/workflows/tpc-h.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index ebd359186d..636befd913 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -110,7 +110,7 @@ jobs: cd "IGinX" if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 else echo "$RUNNER_OS is not supported in this test" exit 1 @@ -162,7 +162,7 @@ jobs: run: | if [ "$RUNNER_OS" == "Linux" ]; then chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" + "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 else echo "$RUNNER_OS is not supported in this test" exit 1 From 885a2be3306daef2cf9588d7e007b7630792c464 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sat, 6 Jul 2024 14:58:23 +0800 Subject: [PATCH 188/229] debug --- .github/workflows/tpc-h.yml | 9 --------- .../integration/func/sql/tpch/TPCHRegressionIT.java | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 636befd913..4f2a3ee9c7 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -120,11 +120,6 @@ jobs: if: always() shell: bash run: | - pwd - ls - cd IGinX - pwd - ls mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format - name: Stop ZooKeeper @@ -146,8 +141,6 @@ jobs: - name: Install New IGinX with Maven shell: bash run: | - pwd - ls mvn clean package -DskipTests -P-format -q - name: Change IGinX config @@ -172,6 +165,4 @@ jobs: if: always() shell: bash run: | - pwd - ls mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index f818b31389..b3a9e3513d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -212,7 +212,7 @@ private void insertTable(String table, List fields, List type break; case DATE: // 日期类型需要转为时间戳 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - sb.append(dateFormat.parse(items[i]).getTime()); + sb.append(dateFormat.parse(items[i] + " 08:00:00").getTime()); // 转时区，否则结果错误 sb.append(", "); break; default: From 9f031c25bd9abf18bc9081f205d81e22b2b95954 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sat, 6 Jul 2024 15:19:05 +0800 Subject: [PATCH 189/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index b3a9e3513d..4c8fe09d1d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -211,8 +211,8 @@ private void insertTable(String table, List fields, List type sb.append("\", "); break; case DATE: // 日期类型需要转为时间戳 - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - sb.append(dateFormat.parse(items[i] + " 08:00:00").getTime()); // 转时区，否则结果错误 + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + sb.append(dateFormat.parse(items[i] + " 08:00:00").getTime()); sb.append(", "); break; default: From eb7c3566f99c352a0c8e87ebea8428d1d3e2f3ad Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sun, 7 Jul 2024 20:40:56 +0800 Subject: [PATCH 190/229] test --- .github/scripts/benchmarks/tpch.sh | 2 +- .github/workflows/tpc-h.yml | 6 ++++-- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 7 ++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index 3e20b9aa31..f02744fa58 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -11,7 +11,7 @@ else fi cd tpchdata # 目标文件夹路径 - destination_folder="../../tpc/TPC-H V3.0.1/data" + destination_folder="../tpc/TPC-H V3.0.1/data" # 确保目标文件夹存在，如果不存在则创建 mkdir -p "$destination_folder" diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 4f2a3ee9c7..36c4d1d742 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -81,8 +81,10 @@ jobs: chmod +x "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" "${GITHUB_WORKSPACE}/.github/scripts/benchmarks/tpch.sh" - - name: Run ZooKeeper - uses: ./.github/actions/zookeeperRunner + - name: Run Metadata + uses: ./.github/actions/metadataRunner + with: + metadata: ${{ matrix.metadata }} - name: Run DB uses: ./.github/actions/dbRunner diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 4c8fe09d1d..595a2ed9d6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -40,7 +40,7 @@ public class TPCHRegressionIT { // .tbl文件所在目录 private static final String dataPath = - System.getProperty("user.dir") + "/../../tpc/TPC-H V3.0.1/data"; + System.getProperty("user.dir") + "/../tpc/TPC-H V3.0.1/data"; // 最大重复测试次数 private static final int MAX_REPETITIONS_NUM = 5; @@ -211,8 +211,8 @@ private void insertTable(String table, List fields, List type sb.append("\", "); break; case DATE: // 日期类型需要转为时间戳 - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - sb.append(dateFormat.parse(items[i] + " 08:00:00").getTime()); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + sb.append(dateFormat.parse(items[i]).getTime()); sb.append(", "); break; default: @@ -230,6 +230,7 @@ private void insertTable(String table, List fields, List type sb = new StringBuilder(insertPrefix); } } + System.out.printf("INSERT %d RECORDS INTO TABLE [%s]%n", count, table); } catch (IOException | ParseException | SessionException e) { throw new RuntimeException(e); } From eaff86970b84351c4b830c86af4fa12aec4c79be Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sun, 7 Jul 2024 21:36:49 +0800 Subject: [PATCH 191/229] test --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 595a2ed9d6..d1d8db30f9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -245,6 +245,10 @@ public void clearData() throws SessionException { public void test() { System.out.println("start"); try { + String s = "SHOW COLUMNS;"; + SessionExecuteSqlResult res = conn.executeSql(s); + res.print(false, ""); + // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); // 执行垃圾回收，尽量释放内存 From fd50fab682de1492facb013a60b7d5a2db69e0ce Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sun, 7 Jul 2024 23:03:22 +0800 Subject: [PATCH 192/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index d1d8db30f9..367d500521 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -226,6 +226,7 @@ private void insertTable(String table, List fields, List type if (count % 10000 == 0) { sb.setLength(sb.length() - 2); sb.append(";"); + System.out.println(sb); conn.executeSql(sb.toString()); sb = new StringBuilder(insertPrefix); } From 3b5aab927639317232c2802cbd82d5cbcc5a2e13 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sun, 7 Jul 2024 23:28:11 +0800 Subject: [PATCH 193/229] remove q1 --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 367d500521..1a068a8472 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -212,7 +212,9 @@ private void insertTable(String table, List fields, List type break; case DATE: // 日期类型需要转为时间戳 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - sb.append(dateFormat.parse(items[i]).getTime()); + long time = dateFormat.parse(items[i]).getTime(); + sb.append(time); + System.out.println(items[i] + " -> " + time); sb.append(", "); break; default: @@ -226,7 +228,6 @@ private void insertTable(String table, List fields, List type if (count % 10000 == 0) { sb.setLength(sb.length() - 2); sb.append(";"); - System.out.println(sb); conn.executeSql(sb.toString()); sb = new StringBuilder(insertPrefix); } @@ -258,7 +259,8 @@ public void test() { long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); long startTime; // 13有问题 - List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = From 6f69ee2be9d5deddaecad98189d020505183a335 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Sun, 7 Jul 2024 23:37:22 +0800 Subject: [PATCH 194/229] Update TPCHRegressionIT.java --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 1a068a8472..d998604163 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -214,7 +214,6 @@ private void insertTable(String table, List fields, List type SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); long time = dateFormat.parse(items[i]).getTime(); sb.append(time); - System.out.println(items[i] + " -> " + time); sb.append(", "); break; default: From ac6a0e29857181e2c82fc3b390b69d96d89445a2 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Mon, 8 Jul 2024 08:56:20 +0800 Subject: [PATCH 195/229] Update TPCHRegressionIT.java --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index d998604163..35a35160a9 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -259,7 +259,7 @@ public void test() { long startTime; // 13有问题 // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); - List queryIds = Arrays.asList(2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(3, 5, 6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = From 43f93f40405e5c4d1f7740634faed14dea6e705c Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Mon, 8 Jul 2024 09:57:19 +0800 Subject: [PATCH 196/229] debug --- .../func/sql/tpch/TPCHRegressionIT.java | 20 ++++++++++++++++++- test/src/test/resources/tpch/queries/q1.sql | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 35a35160a9..b48deeb02c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -21,8 +21,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Scanner; +import java.util.TimeZone; import java.util.stream.Collectors; import org.junit.After; import org.junit.AfterClass; @@ -32,6 +34,22 @@ public class TPCHRegressionIT { + public static void main(String[] args) throws ParseException { + + String s = "1998-09-02"; + SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat1.setTimeZone(TimeZone.getTimeZone("GMT+8:00")); + long time = dateFormat1.parse(s).getTime(); + System.out.println(time); + + SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + // dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT")); + Date date = new Date(time); + System.out.println(dateFormat2.format(date)); + + + } + // host info protected static String defaultTestHost = "127.0.0.1"; protected static int defaultTestPort = 6888; @@ -259,7 +277,7 @@ public void test() { long startTime; // 13有问题 // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); - List queryIds = Arrays.asList(3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = diff --git a/test/src/test/resources/tpch/queries/q1.sql b/test/src/test/resources/tpch/queries/q1.sql index c77305f28d..e0026456c9 100644 --- a/test/src/test/resources/tpch/queries/q1.sql +++ b/test/src/test/resources/tpch/queries/q1.sql @@ -21,7 +21,7 @@ from ( from lineitem where - lineitem.l_shipdate <= 904694400000 + lineitem.l_shipdate <= 904665600000 ) group by lineitem.l_returnflag, From f368d9e8e3d725688c3b09ab40447cbb85cd6a96 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Mon, 8 Jul 2024 10:01:54 +0800 Subject: [PATCH 197/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 1 + test/src/test/resources/tpch/queries/q1.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index b48deeb02c..88a4e94b35 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -230,6 +230,7 @@ private void insertTable(String table, List fields, List type break; case DATE: // 日期类型需要转为时间戳 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); long time = dateFormat.parse(items[i]).getTime(); sb.append(time); sb.append(", "); diff --git a/test/src/test/resources/tpch/queries/q1.sql b/test/src/test/resources/tpch/queries/q1.sql index e0026456c9..c77305f28d 100644 --- a/test/src/test/resources/tpch/queries/q1.sql +++ b/test/src/test/resources/tpch/queries/q1.sql @@ -21,7 +21,7 @@ from ( from lineitem where - lineitem.l_shipdate <= 904665600000 + lineitem.l_shipdate <= 904694400000 ) group by lineitem.l_returnflag, From 52e2b34ad6169dc16648b96b5c66e8868440bc7a Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Mon, 8 Jul 2024 10:15:22 +0800 Subject: [PATCH 198/229] Update TPCHRegressionIT.java --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 88a4e94b35..57fe5303e8 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -38,12 +38,12 @@ public static void main(String[] args) throws ParseException { String s = "1998-09-02"; SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat1.setTimeZone(TimeZone.getTimeZone("GMT+8:00")); + dateFormat1.setTimeZone(TimeZone.getTimeZone("GMT")); long time = dateFormat1.parse(s).getTime(); System.out.println(time); SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); - // dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT")); + dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT")); Date date = new Date(time); System.out.println(dateFormat2.format(date)); @@ -230,7 +230,7 @@ private void insertTable(String table, List fields, List type break; case DATE: // 日期类型需要转为时间戳 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT+8:00")); long time = dateFormat.parse(items[i]).getTime(); sb.append(time); sb.append(", "); From 24aa002fc4e121893cb634a647c9434203747a28 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Mon, 8 Jul 2024 10:50:17 +0800 Subject: [PATCH 199/229] Update TPCHRegressionIT.java --- .../func/sql/tpch/TPCHRegressionIT.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 57fe5303e8..ef424e76ec 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -35,19 +35,16 @@ public class TPCHRegressionIT { public static void main(String[] args) throws ParseException { - - String s = "1998-09-02"; - SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat1.setTimeZone(TimeZone.getTimeZone("GMT")); - long time = dateFormat1.parse(s).getTime(); - System.out.println(time); - +// String s = "1998-09-02"; +// SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd"); +// dateFormat1.setTimeZone(TimeZone.getTimeZone("GMT")); +// long time = dateFormat1.parse(s).getTime(); +// System.out.println(time); + long time = 757353600000L; SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT")); Date date = new Date(time); System.out.println(dateFormat2.format(date)); - - } // host info @@ -230,7 +227,7 @@ private void insertTable(String table, List fields, List type break; case DATE: // 日期类型需要转为时间戳 SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("GMT+8:00")); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); long time = dateFormat.parse(items[i]).getTime(); sb.append(time); sb.append(", "); @@ -278,7 +275,7 @@ public void test() { long startTime; // 13有问题 // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); - List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = From 761b227fd05a4a317fa74f82c8125a635b7c1ab2 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 17:34:51 +0800 Subject: [PATCH 200/229] debug --- .../func/sql/tpch/TPCHRegressionIT.java | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index ef424e76ec..a79781b392 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Scanner; import java.util.TimeZone; @@ -34,19 +33,6 @@ public class TPCHRegressionIT { - public static void main(String[] args) throws ParseException { -// String s = "1998-09-02"; -// SimpleDateFormat dateFormat1 = new SimpleDateFormat("yyyy-MM-dd"); -// dateFormat1.setTimeZone(TimeZone.getTimeZone("GMT")); -// long time = dateFormat1.parse(s).getTime(); -// System.out.println(time); - long time = 757353600000L; - SimpleDateFormat dateFormat2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); - dateFormat2.setTimeZone(TimeZone.getTimeZone("GMT")); - Date date = new Date(time); - System.out.println(dateFormat2.format(date)); - } - // host info protected static String defaultTestHost = "127.0.0.1"; protected static int defaultTestPort = 6888; @@ -210,8 +196,8 @@ private void insertTable(String table, List fields, List type while ((line = br.readLine()) != null) { String[] items = line.split("\\|"); sb.append("("); - count++; sb.append(count); // 插入自增key列 + count++; sb.append(", "); assert fields.size() == items.length; for (int i = 0; i < items.length; i++) { @@ -247,6 +233,12 @@ private void insertTable(String table, List fields, List type sb = new StringBuilder(insertPrefix); } } + // 插入剩余数据 + if (sb.length() != insertPrefix.length()) { + sb.setLength(sb.length() - 2); + sb.append(";"); + conn.executeSql(sb.toString()); + } System.out.printf("INSERT %d RECORDS INTO TABLE [%s]%n", count, table); } catch (IOException | ParseException | SessionException e) { throw new RuntimeException(e); @@ -262,10 +254,6 @@ public void clearData() throws SessionException { public void test() { System.out.println("start"); try { - String s = "SHOW COLUMNS;"; - SessionExecuteSqlResult res = conn.executeSql(s); - res.print(false, ""); - // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); // 执行垃圾回收，尽量释放内存 @@ -274,8 +262,7 @@ public void test() { long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); long startTime; // 13有问题 - // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); - List queryIds = Arrays.asList(6, 9, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = From d48bce2cc8a230964f46b22be09fc4e475e8fc33 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 17:39:53 +0800 Subject: [PATCH 201/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index a79781b392..615097a6f2 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -262,7 +262,8 @@ public void test() { long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); long startTime; // 13有问题 - List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = From e14c487d70d9cc577fb1cc6ad1d0c3a41729ff26 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 19:18:33 +0800 Subject: [PATCH 202/229] debug --- .github/actions/iginxRunner/action.yml | 32 ++++--- .github/scripts/benchmarks/tpch.sh | 18 ++++ .github/workflows/tpc-h.yml | 29 ++---- .../func/sql/tpch/TPCHRegressionIT.java | 91 ++++++++++++++----- .../resources/tpch/udf/udtf_extract_year.py | 46 ++++++++++ thu_cloud_download.py | 18 ++++ 6 files changed, 178 insertions(+), 56 deletions(-) create mode 100644 test/src/test/resources/tpch/udf/udtf_extract_year.py diff --git a/.github/actions/iginxRunner/action.yml b/.github/actions/iginxRunner/action.yml index 859354faf2..2edacc23cc 100644 --- a/.github/actions/iginxRunner/action.yml +++ b/.github/actions/iginxRunner/action.yml @@ -9,6 +9,10 @@ inputs: description: "to test UDF path detection" required: false default: "false" + Root-Dir-Path: + description: "the path of IGinX root directory" + required: false + default: "${GITHUB_WORKSPACE}" runs: using: "composite" # Mandatory parameter steps: @@ -17,37 +21,37 @@ runs: shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then - sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties - sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties + sed -i 's/pythonCMD=python3/pythonCMD=python/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "macOS" ]; then - sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${GITHUB_WORKSPACE}/core/target/iginx-core-${VERSION}/conf/config.properties + sudo sed -i '' 's/needInitBasicUDFFunctions=false/needInitBasicUDFFunctions=true/' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties else echo "$RUNNER_OS is not supported" exit 1 fi - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_udf_path.sh" ${VERSION} + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_udf_path.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_udf_path.sh" ${VERSION} - if: inputs.if-test-udf=='false' && inputs.if-stop=='false' name: Start IGinX shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx.sh" 6888 6666 elif [ "$RUNNER_OS" == "Windows" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_windows.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_windows.sh" 6888 6666 elif [ "$RUNNER_OS" == "macOS" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_macos.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_macos.sh" 6888 6666 fi - if: inputs.if-test-udf=='false' && inputs.if-stop=='true' name: Stop IGinX shell: bash run: | - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx_kill.sh" + chmod +x "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_kill.sh" + "${{ inputs.Root-Dir-Path }}/.github/scripts/iginx/iginx_kill.sh" diff --git a/.github/scripts/benchmarks/tpch.sh b/.github/scripts/benchmarks/tpch.sh index f02744fa58..f5aeecad85 100644 --- a/.github/scripts/benchmarks/tpch.sh +++ b/.github/scripts/benchmarks/tpch.sh @@ -1,4 +1,22 @@ #!/bin/bash +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + if [ "$RUNNER_OS" = "Windows" ]; then python thu_cloud_download.py \ diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 36c4d1d742..f71cf83726 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -106,17 +106,10 @@ jobs: Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX - if: always() - shell: bash - run: | - cd "IGinX" - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/IGinX/.github/scripts/iginx/iginx.sh" 6888 6666 - else - echo "$RUNNER_OS is not supported in this test" - exit 1 - fi + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" - name: Run Regression Test on Old IGinX if: always() @@ -152,16 +145,10 @@ jobs: Set-Filter-Fragment-OFF: "true" - name: Start New IGinX - if: always() - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - chmod +x "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" - "${GITHUB_WORKSPACE}/.github/scripts/iginx/iginx.sh" 6888 6666 - else - echo "$RUNNER_OS is not supported in this test" - exit 1 - fi + uses: ./.github/actions/iginxRunner + with: + version: ${VERSION} + if-test-udf: "true" - name: Run Regression Test on New IGinX if: always() diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 615097a6f2..09b186746d 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -1,15 +1,34 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package cn.edu.tsinghua.iginx.integration.func.sql.tpch; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.DATE; import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.NUM; import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.STR; +import static org.junit.Assert.fail; import cn.edu.tsinghua.iginx.exception.SessionException; -import cn.edu.tsinghua.iginx.integration.tool.MultiConnection; import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import java.io.BufferedReader; +import java.io.File; import java.io.FileReader; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -30,9 +49,13 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TPCHRegressionIT { + private static final Logger LOGGER = LoggerFactory.getLogger(TPCHRegressionIT.class); + // host info protected static String defaultTestHost = "127.0.0.1"; protected static int defaultTestPort = 6888; @@ -40,16 +63,19 @@ public class TPCHRegressionIT { protected static String defaultTestPass = "root"; // .tbl文件所在目录 - private static final String dataPath = + private static final String dataDir = System.getProperty("user.dir") + "/../tpc/TPC-H V3.0.1/data"; + // udf文件所在目录 + private static final String udfDir = "src/test/resources/tpch/udf/"; + // 最大重复测试次数 private static final int MAX_REPETITIONS_NUM = 5; // 回归阈值 private static final double REGRESSION_THRESHOLD_PERCENTAGE = 0.1; - protected static MultiConnection conn; + protected static Session session; enum FieldType { NUM, @@ -61,20 +87,18 @@ public TPCHRegressionIT() {} @BeforeClass public static void setUp() throws SessionException { - conn = - new MultiConnection( - new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass)); - conn.openSession(); + session = new Session(defaultTestHost, defaultTestPort, defaultTestUser, defaultTestPass); + session.openSession(); } @AfterClass public static void tearDown() throws SessionException { - clearAllData(conn); - conn.closeSession(); + clearAllData(session); + session.closeSession(); } @Before - public void insertData() { + public void prepare() { List tableList = Arrays.asList( "region", "nation", "supplier", "part", "partsupp", "customer", "orders", "lineitem"); @@ -171,9 +195,17 @@ public void insertData() { Arrays.asList( NUM, NUM, NUM, NUM, NUM, NUM, NUM, NUM, STR, STR, DATE, DATE, DATE, STR, STR, STR)); + // 插入数据 for (int i = 0; i < 8; i++) { insertTable(tableList.get(i), fieldsList.get(i), typesList.get(i)); } + + List> UDFInfos = new ArrayList<>(); + UDFInfos.add(Arrays.asList("UDTF", "extractYear", "UDFExtractYear", "udtf_extract_year.py")); + // 注册UDF函数 + for (List UDFInfo : UDFInfos) { + registerUDF(UDFInfo); + } } private void insertTable(String table, List fields, List types) { @@ -190,7 +222,7 @@ private void insertTable(String table, List fields, List type long count = 0; try (BufferedReader br = - new BufferedReader(new FileReader(String.format("%s/%s.tbl", dataPath, table)))) { + new BufferedReader(new FileReader(String.format("%s/%s.tbl", dataDir, table)))) { StringBuilder sb = new StringBuilder(insertPrefix); String line; while ((line = br.readLine()) != null) { @@ -229,7 +261,7 @@ private void insertTable(String table, List fields, List type if (count % 10000 == 0) { sb.setLength(sb.length() - 2); sb.append(";"); - conn.executeSql(sb.toString()); + session.executeSql(sb.toString()); sb = new StringBuilder(insertPrefix); } } @@ -237,22 +269,40 @@ private void insertTable(String table, List fields, List type if (sb.length() != insertPrefix.length()) { sb.setLength(sb.length() - 2); sb.append(";"); - conn.executeSql(sb.toString()); + session.executeSql(sb.toString()); } - System.out.printf("INSERT %d RECORDS INTO TABLE [%s]%n", count, table); + LOGGER.info("Insert {} records into table [{}].", count, table); } catch (IOException | ParseException | SessionException e) { - throw new RuntimeException(e); + LOGGER.error("Insert into table {} fail. Caused by:", table, e); + fail(); + } + } + + private void registerUDF(List UDFInfo) { + String SINGLE_UDF_REGISTER_SQL = "CREATE FUNCTION %s \"%s\" FROM \"%s\" IN \"%s\";"; + File udfFile = new File(udfDir + UDFInfo.get(3)); + String register = + String.format( + SINGLE_UDF_REGISTER_SQL, + UDFInfo.get(0), + UDFInfo.get(1), + UDFInfo.get(2), + udfFile.getAbsolutePath()); + try { + session.executeRegisterTask(register, false); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", register, e); + fail(); } } @After public void clearData() throws SessionException { - conn.executeSql("CLEAR DATA;"); + session.executeSql("CLEAR DATA;"); } @Test public void test() { - System.out.println("start"); try { // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); @@ -262,8 +312,7 @@ public void test() { long usedMemoryBefore = runtime.totalMemory() - runtime.freeMemory(); long startTime; // 13有问题 - // List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); - List queryIds = Arrays.asList(1, 2, 3, 5, 6, 10, 16, 17, 18, 19, 20); + List queryIds = Arrays.asList(1, 2, 3, 5, 6, 9, 10, 16, 17, 18, 19, 20); for (int queryId : queryIds) { // read from sql file String sqlString = @@ -279,7 +328,7 @@ public void test() { continue; } sql += ";"; - result = conn.executeSql(sql); + result = session.executeSql(sql); result.print(false, ""); } // 再次执行垃圾回收 @@ -397,7 +446,7 @@ public void test() { private static long executeSQL(String sql) throws SessionException { long startTime = System.currentTimeMillis(); - SessionExecuteSqlResult result = conn.executeSql(sql); + SessionExecuteSqlResult result = session.executeSql(sql); return System.currentTimeMillis() - startTime; } diff --git a/test/src/test/resources/tpch/udf/udtf_extract_year.py b/test/src/test/resources/tpch/udf/udtf_extract_year.py new file mode 100644 index 0000000000..fc4fcf4f5c --- /dev/null +++ b/test/src/test/resources/tpch/udf/udtf_extract_year.py @@ -0,0 +1,46 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from datetime import datetime, timezone +import pytz +class UDFExtractYear: + def __init__(self): + pass + + def extractYear(self, num): + # Unix timestamp is in milliseconds + timestamp_in_seconds = num / 1000 + # TODO 直接将timestamp增加8小时 + tz = pytz.timezone('Asia/Shanghai') + dt = datetime.fromtimestamp(timestamp_in_seconds, tz=tz) + return float(dt.year) + def transform(self, data, args, kvargs): + res = self.buildHeader(data) + dateRow = [] + for num in data[2][1:]: + dateRow.append(self.extractYear(num)) + res.append(dateRow) + return res + + def buildHeader(self, data): + colNames = [] + colTypes = [] + for name in data[0][1:]: + colNames.append("extractYear(" + name + ")") + colTypes.append("DOUBLE") + return [colNames, colTypes] diff --git a/thu_cloud_download.py b/thu_cloud_download.py index e15a7d8427..474e9b9dd4 100644 --- a/thu_cloud_download.py +++ b/thu_cloud_download.py @@ -1,3 +1,21 @@ +# +# IGinX - the polystore system with high performance +# Copyright (C) Tsinghua University +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + import os import re import logging From 91cb0869facca60022197acfd36fcd761f299aee Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 19:25:34 +0800 Subject: [PATCH 203/229] debug --- .github/workflows/tpc-h.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index f71cf83726..af4bfda4dd 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -94,6 +94,7 @@ jobs: - name: Get Old Version shell: bash run: | + mvn clean package -DskipTests -P-format -q git clone https://github.com/IGinX-THU/IGinX.git cd IGinX mvn clean package -DskipTests -P-format -q From 54d5132f8ba92af13bc4f7076dee4bb25b5e2db6 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 19:35:55 +0800 Subject: [PATCH 204/229] debug --- .github/workflows/tpc-h.yml | 6 ++++++ .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index af4bfda4dd..8dbe773f3b 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -118,6 +118,12 @@ jobs: run: | mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format + - name: Show IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log + - name: Stop ZooKeeper uses: ./.github/actions/zookeeperRunner with: diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 09b186746d..6a348af743 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -304,6 +304,9 @@ public void clearData() throws SessionException { @Test public void test() { try { + String s = "SHOW FUNCTIONS;"; + SessionExecuteSqlResult res = session.executeSql(s); + res.print(false, ""); // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); // 执行垃圾回收，尽量释放内存 From 78f59377ee1b601aa959e5d35de6df7d984760ab Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 20:07:33 +0800 Subject: [PATCH 205/229] debug --- .github/workflows/tpc-h.yml | 2 +- .../func/sql/tpch/TPCHRegressionIT.java | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 8dbe773f3b..3501460591 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -122,7 +122,7 @@ jobs: if: always() shell: bash run: | - cat iginx-*.log + cat iginx-*.log - name: Stop ZooKeeper uses: ./.github/actions/zookeeperRunner diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 6a348af743..370f7c06d3 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -304,7 +304,7 @@ public void clearData() throws SessionException { @Test public void test() { try { - String s = "SHOW FUNCTIONS;"; + String s = "select o_orderkey, extractYear(o_orderdate) from orders;"; SessionExecuteSqlResult res = session.executeSql(s); res.print(false, ""); // 获取当前JVM的Runtime实例 @@ -331,7 +331,12 @@ public void test() { continue; } sql += ";"; - result = session.executeSql(sql); + try { + result = session.executeSql(sql); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", sql, e); + fail(); + } result.print(false, ""); } // 再次执行垃圾回收 @@ -442,14 +447,20 @@ public void test() { } } - } catch (SessionException | IOException e) { - throw new RuntimeException(e); + } catch (IOException e) { + LOGGER.error("Test fail. Caused by:", e); + fail(); } } - private static long executeSQL(String sql) throws SessionException { + private static long executeSQL(String sql) { long startTime = System.currentTimeMillis(); - SessionExecuteSqlResult result = session.executeSql(sql); + try { + SessionExecuteSqlResult result = session.executeSql(sql); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", sql, e); + fail(); + } return System.currentTimeMillis() - startTime; } @@ -487,7 +498,8 @@ private static List> csvReader(String filePath) { data.add(row); } } catch (IOException e) { - throw new RuntimeException(e); + LOGGER.error("Read file {} fail. Caused by:", filePath, e); + fail(); } System.out.println(data); return data; From 1e7400b108f1dae5c802539f419dcda8842f989e Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 20:12:08 +0800 Subject: [PATCH 206/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 370f7c06d3..c38447fb67 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -305,7 +305,13 @@ public void clearData() throws SessionException { public void test() { try { String s = "select o_orderkey, extractYear(o_orderdate) from orders;"; - SessionExecuteSqlResult res = session.executeSql(s); + SessionExecuteSqlResult res = null; + try { + res = session.executeSql(s); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); + fail(); + } res.print(false, ""); // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); From e498d563613cf0a6b44f911bec12305f13817f90 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 20:23:39 +0800 Subject: [PATCH 207/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index c38447fb67..559165329b 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -304,7 +304,7 @@ public void clearData() throws SessionException { @Test public void test() { try { - String s = "select o_orderkey, extractYear(o_orderdate) from orders;"; + String s = "select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 30;"; SessionExecuteSqlResult res = null; try { res = session.executeSql(s); From 3e1e4d3996f46493f44bfe280751a57d1ed8929f Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 20:46:11 +0800 Subject: [PATCH 208/229] debug --- .../integration/func/sql/tpch/TPCHRegressionIT.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 559165329b..bd9e9bf7a6 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -70,7 +70,7 @@ public class TPCHRegressionIT { private static final String udfDir = "src/test/resources/tpch/udf/"; // 最大重复测试次数 - private static final int MAX_REPETITIONS_NUM = 5; + private static final int MAX_REPETITIONS_NUM = 10; // 回归阈值 private static final double REGRESSION_THRESHOLD_PERCENTAGE = 0.1; @@ -289,6 +289,7 @@ private void registerUDF(List UDFInfo) { UDFInfo.get(2), udfFile.getAbsolutePath()); try { + LOGGER.info("Execute register UDF statement: {}", register); session.executeRegisterTask(register, false); } catch (SessionException e) { LOGGER.error("Statement: \"{}\" execute fail. Caused by:", register, e); @@ -304,7 +305,8 @@ public void clearData() throws SessionException { @Test public void test() { try { - String s = "select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 30;"; + String s = + "select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 30;"; SessionExecuteSqlResult res = null; try { res = session.executeSql(s); @@ -441,14 +443,15 @@ public void test() { } } - // 重复5次后耗时仍超过阈值 + // 重复10次后耗时仍超过阈值 if (regressionDetected) { System.out.printf( "performance degradation of query %d exceeds %f%n", queryIds.get(i), REGRESSION_THRESHOLD_PERCENTAGE); System.out.printf("old timing: %.3fms%n", oldTimeCost); System.out.printf("new timing: %.3fms%n", newTimeCost); - throw new RuntimeException("performance degradation exceeds the threshold"); + LOGGER.error("TPC-H query {} regression test fail.", queryIds.get(i)); + fail(); } } } From 0ab63b094c39084c9fc3ce6d899bd4a2eda3a531 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 21:34:19 +0800 Subject: [PATCH 209/229] debug --- .../func/sql/tpch/TPCHRegressionIT.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index bd9e9bf7a6..099d1f2471 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -398,6 +398,14 @@ public void test() { } } + String sql = "show columns;"; + try { + res = session.executeSql(sql); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); + fail(); + } + String fileName = "src/test/resources/tpch/oldTimeCosts.txt"; if (!Files.exists(Paths.get(fileName))) { // 文件不存在，即此次跑的是主分支代码，需要将查询时间写入文件 List timeCosts = new ArrayList<>(); @@ -409,10 +417,17 @@ public void test() { readSqlFileAsString("src/test/resources/tpch/queries/q" + queryId + ".sql"); // 执行查询语句, split by ; 最后一句为执行结果 String[] sqls = sqlString.split(";"); - long timeCost = executeSQL(sqls[sqls.length - 2] + ";"); - timeCosts.add(timeCost); + + // 主分支上重复查询两次取平均值 + long sum = 0; + for (int i = 0; i < 2; i++) { + long timeCost = executeSQL(sqls[sqls.length - 2] + ";"); + sum += timeCost; + } + sum /= 2; + timeCosts.add(sum); System.out.printf( - "end tpc-h query %d in main branch, time cost: %dms%n", queryId, timeCost); + "end tpc-h query %d in main branch, time cost: %dms%n", queryId, sum); } writeToFile(timeCosts, fileName); } else { // 文件存在，即此次跑的是新分支代码，需要读取文件进行比较 From f41e49365b901608b74b73a302363ed71bf7853e Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 22:00:58 +0800 Subject: [PATCH 210/229] debug --- .../iginx/integration/func/sql/tpch/TPCHRegressionIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index 099d1f2471..a23d4bfb26 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -306,7 +306,7 @@ public void clearData() throws SessionException { public void test() { try { String s = - "select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 30;"; + "explain select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 10;"; SessionExecuteSqlResult res = null; try { res = session.executeSql(s); @@ -401,6 +401,7 @@ public void test() { String sql = "show columns;"; try { res = session.executeSql(sql); + res.print(false, ""); } catch (SessionException e) { LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); fail(); @@ -426,8 +427,7 @@ public void test() { } sum /= 2; timeCosts.add(sum); - System.out.printf( - "end tpc-h query %d in main branch, time cost: %dms%n", queryId, sum); + System.out.printf("end tpc-h query %d in main branch, time cost: %dms%n", queryId, sum); } writeToFile(timeCosts, fileName); } else { // 文件存在，即此次跑的是新分支代码，需要读取文件进行比较 From 39061be4d83cd3443c2202939cb44ab507676296 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Tue, 9 Jul 2024 22:25:42 +0800 Subject: [PATCH 211/229] debug --- .../integration/func/sql/tpch/TPCHRegressionIT.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java index a23d4bfb26..4619ad0fac 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java @@ -73,7 +73,7 @@ public class TPCHRegressionIT { private static final int MAX_REPETITIONS_NUM = 10; // 回归阈值 - private static final double REGRESSION_THRESHOLD_PERCENTAGE = 0.1; + private static final double REGRESSION_THRESHOLD_PERCENTAGE = 0.2; protected static Session session; @@ -398,15 +398,6 @@ public void test() { } } - String sql = "show columns;"; - try { - res = session.executeSql(sql); - res.print(false, ""); - } catch (SessionException e) { - LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); - fail(); - } - String fileName = "src/test/resources/tpch/oldTimeCosts.txt"; if (!Files.exists(Paths.get(fileName))) { // 文件不存在，即此次跑的是主分支代码，需要将查询时间写入文件 List timeCosts = new ArrayList<>(); From 92dc533195e8d77805afae2a02f706e66ce9962c Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 11 Jul 2024 22:10:42 +0800 Subject: [PATCH 212/229] debug --- .../naive/NaiveOperatorMemoryExecutor.java | 17 ++++------ .../optimizer/rules/ColumnPruningRule.java | 17 ++++++++++ .../iginx/integration/tool/ConfLoader.java | 8 +++++ .../{func/sql => }/tpch/TPCHRegressionIT.java | 34 ++++++++----------- test/src/test/resources/testConfig.properties | 4 +++ 5 files changed, 51 insertions(+), 29 deletions(-) rename test/src/test/java/cn/edu/tsinghua/iginx/integration/{func/sql => }/tpch/TPCHRegressionIT.java (94%) diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java index 68e80661c7..fca427c618 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/engine/physical/memory/execute/naive/NaiveOperatorMemoryExecutor.java @@ -192,7 +192,7 @@ private Table transformToTable(RowStream stream) throws PhysicalException { return new Table(header, rows); } - private RowStream executeProject(Project project, Table table) throws PhysicalException { + private RowStream executeProject(Project project, Table table) { List patterns = project.getPatterns(); Header header = table.getHeader(); List targetFields = new ArrayList<>(); @@ -244,7 +244,7 @@ private RowStream executeSort(Sort sort, Table table) throws PhysicalException { return table; } - private RowStream executeLimit(Limit limit, Table table) throws PhysicalException { + private RowStream executeLimit(Limit limit, Table table) { int rowSize = table.getRowSize(); Header header = table.getHeader(); List rows = new ArrayList<>(); @@ -370,8 +370,7 @@ private RowStream executeDownsample(Downsample downsample, Table table) throws P return RowUtils.joinMultipleTablesByKey(tableList); } - private RowStream executeRowTransform(RowTransform rowTransform, Table table) - throws PhysicalException { + private RowStream executeRowTransform(RowTransform rowTransform, Table table) { List> list = new ArrayList<>(); rowTransform .getFunctionCallList() @@ -473,7 +472,7 @@ private RowStream executeMappingTransform(MappingTransform mappingTransform, Tab return RowUtils.calMappingTransform(table, functionCallList); } - private RowStream executeRename(Rename rename, Table table) throws PhysicalException { + private RowStream executeRename(Rename rename, Table table) { Header header = table.getHeader(); Map aliasMap = rename.getAliasMap(); @@ -495,8 +494,7 @@ private RowStream executeRename(Rename rename, Table table) throws PhysicalExcep return new Table(newHeader, rows); } - private RowStream executeAddSchemaPrefix(AddSchemaPrefix addSchemaPrefix, Table table) - throws PhysicalException { + private RowStream executeAddSchemaPrefix(AddSchemaPrefix addSchemaPrefix, Table table) { Header header = table.getHeader(); String schemaPrefix = addSchemaPrefix.getSchemaPrefix(); @@ -635,7 +633,7 @@ private RowStream executeJoin(Join join, Table tableA, Table tableB) throws Phys // 检查时间戳 if (!headerA.hasKey() || !headerB.hasKey()) { throw new InvalidOperatorParameterException( - "row streams for join operator by time should have timestamp."); + "row streams for join operator by key should have key."); } List newFields = new ArrayList<>(); newFields.addAll(headerA.getFields()); @@ -731,8 +729,7 @@ private RowStream executeJoin(Join join, Table tableA, Table tableB) throws Phys } } - private RowStream executeCrossJoin(CrossJoin crossJoin, Table tableA, Table tableB) - throws PhysicalException { + private RowStream executeCrossJoin(CrossJoin crossJoin, Table tableA, Table tableB) { Header newHeader = HeaderUtils.constructNewHead( tableA.getHeader(), tableB.getHeader(), crossJoin.getPrefixA(), crossJoin.getPrefixB()); diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java index 4eb237fb74..e84bb78259 100644 --- a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java @@ -330,6 +330,23 @@ private void collectColumns( leftColumns.addAll(leftOrder); rightColumns.addAll(rightOrder); } + } else if (operator.getType().equals(OperatorType.Join)) { + List leftPatterns = + OperatorUtils.getPatternFromOperatorChildren( + ((OperatorSource) ((BinaryOperator) operator).getSourceA()).getOperator(), + new ArrayList<>()); + List rightPatterns = + OperatorUtils.getPatternFromOperatorChildren( + ((OperatorSource) ((BinaryOperator) operator).getSourceB()).getOperator(), + new ArrayList<>()); + for (String column : columns) { + if (leftPatterns.contains(column)) { + leftColumns.add(column); + } + if (rightPatterns.contains(column)) { + rightColumns.add(column); + } + } } else { leftColumns.addAll(columns); rightColumns.addAll(columns); diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java index c0f6fcff94..3c56aa2224 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tool/ConfLoader.java @@ -126,6 +126,14 @@ public String getDBCETestWay() { return getTestProperty(DBCE_TEST_WAY, DEFAULT_DBCE_TEST_WAY); } + public int getMaxRepetitionsNum() { + return Integer.parseInt(properties.getProperty("max_repetitions_num")); + } + + public double getRegressionThreshold() { + return Double.parseDouble(properties.getProperty("regression_threshold")); + } + public ConfLoader(String confPath) { this.confPath = confPath; try { diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java similarity index 94% rename from test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java rename to test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java index 4619ad0fac..8b3bde7220 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/sql/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java @@ -16,15 +16,17 @@ * along with this program. If not, see . */ -package cn.edu.tsinghua.iginx.integration.func.sql.tpch; +package cn.edu.tsinghua.iginx.integration.tpch; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; -import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.DATE; -import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.NUM; -import static cn.edu.tsinghua.iginx.integration.func.sql.tpch.TPCHRegressionIT.FieldType.STR; +import static cn.edu.tsinghua.iginx.integration.tpch.TPCHRegressionIT.FieldType.DATE; +import static cn.edu.tsinghua.iginx.integration.tpch.TPCHRegressionIT.FieldType.NUM; +import static cn.edu.tsinghua.iginx.integration.tpch.TPCHRegressionIT.FieldType.STR; import static org.junit.Assert.fail; import cn.edu.tsinghua.iginx.exception.SessionException; +import cn.edu.tsinghua.iginx.integration.controller.Controller; +import cn.edu.tsinghua.iginx.integration.tool.ConfLoader; import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.session.SessionExecuteSqlResult; import java.io.BufferedReader; @@ -70,10 +72,10 @@ public class TPCHRegressionIT { private static final String udfDir = "src/test/resources/tpch/udf/"; // 最大重复测试次数 - private static final int MAX_REPETITIONS_NUM = 10; + private static int MAX_REPETITIONS_NUM; // 回归阈值 - private static final double REGRESSION_THRESHOLD_PERCENTAGE = 0.2; + private static double REGRESSION_THRESHOLD; protected static Session session; @@ -83,7 +85,11 @@ enum FieldType { DATE } - public TPCHRegressionIT() {} + public TPCHRegressionIT() { + ConfLoader conf = new ConfLoader(Controller.CONFIG_FILE); + MAX_REPETITIONS_NUM = conf.getMaxRepetitionsNum(); + REGRESSION_THRESHOLD = conf.getRegressionThreshold(); + } @BeforeClass public static void setUp() throws SessionException { @@ -305,16 +311,6 @@ public void clearData() throws SessionException { @Test public void test() { try { - String s = - "explain select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 10;"; - SessionExecuteSqlResult res = null; - try { - res = session.executeSql(s); - } catch (SessionException e) { - LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); - fail(); - } - res.print(false, ""); // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); // 执行垃圾回收，尽量释放内存 @@ -423,7 +419,7 @@ public void test() { writeToFile(timeCosts, fileName); } else { // 文件存在，即此次跑的是新分支代码，需要读取文件进行比较 List oldTimeCosts = readFromFile(fileName); - double multiplyPercentage = 1 + REGRESSION_THRESHOLD_PERCENTAGE; + double multiplyPercentage = 1 + REGRESSION_THRESHOLD; for (int i = 0; i < queryIds.size(); i++) { double oldTimeCost = (double) oldTimeCosts.get(i); double newTimeCost = 0; @@ -453,7 +449,7 @@ public void test() { if (regressionDetected) { System.out.printf( "performance degradation of query %d exceeds %f%n", - queryIds.get(i), REGRESSION_THRESHOLD_PERCENTAGE); + queryIds.get(i), REGRESSION_THRESHOLD); System.out.printf("old timing: %.3fms%n", oldTimeCost); System.out.printf("new timing: %.3fms%n", newTimeCost); LOGGER.error("TPC-H query {} regression test fail.", queryIds.get(i)); diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index b6ac7558ae..568dfcb4a5 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -71,3 +71,7 @@ stand_alone_DB=IoTDB12 # Local test DB-CE is_scaling=false DBCE_test_way=oriHasDataExpHasData + +# TPC-H test +max_repetitions_num=10 +regression_threshold=0.3 From 7b5846cb9c8b0083faa98b7173fa1f5f03e19dde Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 11 Jul 2024 22:33:30 +0800 Subject: [PATCH 213/229] debug --- .github/workflows/tpc-h.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 3501460591..6a867ea0dd 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -118,7 +118,7 @@ jobs: run: | mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format - - name: Show IGinX log + - name: Show Old IGinX log if: always() shell: bash run: | @@ -162,3 +162,9 @@ jobs: shell: bash run: | mvn test -q -Dtest=TPCHRegressionIT -DfailIfNoTests=false -P-format + + - name: Show New IGinX log + if: always() + shell: bash + run: | + cat iginx-*.log From d76726828b51334c1d1651b577fdbaedcbfb6678 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 11 Jul 2024 23:19:12 +0800 Subject: [PATCH 214/229] add error log print && open other tests --- .github/workflows/standard-test-suite.yml | 64 +++++++++---------- .github/workflows/tpc-h.yml | 2 +- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 7 +- 3 files changed, 39 insertions(+), 34 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index b8b7c66a53..0e08e4655d 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -1,46 +1,46 @@ name: Standard Test Suite -#on: -# pull_request: # when a PR is opened or reopened -# types: [opened, reopened] -# branches: -# - main +on: + pull_request: # when a PR is opened or reopened + types: [opened, reopened] + branches: + - main -on: [pull_request] +#on: [pull_request] concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: true jobs: - # unit-test: - # uses: ./.github/workflows/unit-test.yml - # unit-mds: - # uses: ./.github/workflows/unit-mds.yml - # case-regression: - # uses: ./.github/workflows/case-regression.yml - # with: - # metadata-matrix: '["zookeeper"]' + unit-test: + uses: ./.github/workflows/unit-test.yml + unit-mds: + uses: ./.github/workflows/unit-mds.yml + case-regression: + uses: ./.github/workflows/case-regression.yml + with: + metadata-matrix: '["zookeeper"]' tpc-h-regression-test: uses: ./.github/workflows/tpc-h.yml with: os-matrix: '["ubuntu-latest"]' metadata-matrix: '["zookeeper"]' -# standalone-test: -# uses: ./.github/workflows/standalone-test.yml -# with: -# metadata-matrix: '["zookeeper"]' -# standalone-test-pushdown: -# uses: ./.github/workflows/standalone-test-pushdown.yml -# with: -# metadata-matrix: '["zookeeper"]' -# db-ce: -# uses: ./.github/workflows/DB-CE.yml -# with: -# metadata-matrix: '["zookeeper"]' -# remote-test: -# uses: ./.github/workflows/remote-test.yml -# with: -# metadata-matrix: '["zookeeper"]' -# assemebly-test: -# uses: ./.github/workflows/assembly-test.yml + standalone-test: + uses: ./.github/workflows/standalone-test.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test-pushdown: + uses: ./.github/workflows/standalone-test-pushdown.yml + with: + metadata-matrix: '["zookeeper"]' + db-ce: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + remote-test: + uses: ./.github/workflows/remote-test.yml + with: + metadata-matrix: '["zookeeper"]' + assemebly-test: + uses: ./.github/workflows/assembly-test.yml diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 6a867ea0dd..e6ba2d8b8f 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -167,4 +167,4 @@ jobs: if: always() shell: bash run: | - cat iginx-*.log + cat iginx-*.log diff --git a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 62e1a35b44..218a021b66 100644 --- a/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSources/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -176,6 +176,7 @@ record = dataSet.next(); columnsInterval = new ColumnsInterval(dataPrefix); } } catch (IoTDBConnectionException | StatementExecutionException e) { + LOGGER.error("get time series failure: ", e); throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } @@ -186,7 +187,7 @@ record = dataSet.next(); } @Override - public void release() throws PhysicalException { + public void release() { sessionPool.close(); } @@ -244,6 +245,7 @@ private void getColumns2StorageUnit(List columns, Map co } dataSet.close(); } catch (IoTDBConnectionException | StatementExecutionException e) { + LOGGER.error("get time series failure: ", e); throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } } @@ -315,6 +317,7 @@ private TaskExecuteResult executeProjectWithFilter( sessionPool.executeQueryStatement(statement), true, project, filter)); return new TaskExecuteResult(rowStream); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { + LOGGER.error("execute project task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( new IoTDBTaskExecuteFailureException("execute project task in iotdb12 failure", e)); } @@ -371,6 +374,7 @@ private TaskExecuteResult executeProjectDummyWithFilter(Project project, Filter sessionPool.executeQueryStatement(statement), false, project, filter)); return new TaskExecuteResult(rowStream); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { + LOGGER.error("execute project task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( new IoTDBTaskExecuteFailureException("execute project task in iotdb12 failure", e)); } @@ -396,6 +400,7 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { break; } if (e != null) { + LOGGER.error("execute insert task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( null, new IoTDBException("execute insert task in iotdb12 failure", e)); } From a82d77a246c7b5041b8e168ef8097156e7f0dc9e Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 11 Jul 2024 23:23:01 +0800 Subject: [PATCH 215/229] debug --- .github/workflows/standard-test-suite.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 0e08e4655d..39103d1557 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -1,12 +1,12 @@ name: Standard Test Suite -on: - pull_request: # when a PR is opened or reopened - types: [opened, reopened] - branches: - - main +#on: +# pull_request: # when a PR is opened or reopened +# types: [opened, reopened] +# branches: +# - main -#on: [pull_request] +on: [pull_request] concurrency: group: "${{ github.workflow }}-${{ github.ref }}" From 606b9192dd493111706e31fb2c2b442f60ec1f63 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 11 Jul 2024 23:24:26 +0800 Subject: [PATCH 216/229] test --- .github/workflows/standard-test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 39103d1557..59459becc6 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -1,6 +1,6 @@ name: Standard Test Suite -#on: +#on:1 # pull_request: # when a PR is opened or reopened # types: [opened, reopened] # branches: From 8a2db5fd00364e02707c1b766557700232c1ca86 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Thu, 11 Jul 2024 23:56:02 +0800 Subject: [PATCH 217/229] debug --- .github/workflows/standard-test-suite.yml | 52 +++++++++---------- .../integration/tpch/TPCHRegressionIT.java | 10 ++++ 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 59459becc6..7e268585b0 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -13,34 +13,34 @@ concurrency: cancel-in-progress: true jobs: - unit-test: - uses: ./.github/workflows/unit-test.yml - unit-mds: - uses: ./.github/workflows/unit-mds.yml - case-regression: - uses: ./.github/workflows/case-regression.yml - with: - metadata-matrix: '["zookeeper"]' +# unit-test: +# uses: ./.github/workflows/unit-test.yml +# unit-mds: +# uses: ./.github/workflows/unit-mds.yml +# case-regression: +# uses: ./.github/workflows/case-regression.yml +# with: +# metadata-matrix: '["zookeeper"]' tpc-h-regression-test: uses: ./.github/workflows/tpc-h.yml with: os-matrix: '["ubuntu-latest"]' metadata-matrix: '["zookeeper"]' - standalone-test: - uses: ./.github/workflows/standalone-test.yml - with: - metadata-matrix: '["zookeeper"]' - standalone-test-pushdown: - uses: ./.github/workflows/standalone-test-pushdown.yml - with: - metadata-matrix: '["zookeeper"]' - db-ce: - uses: ./.github/workflows/DB-CE.yml - with: - metadata-matrix: '["zookeeper"]' - remote-test: - uses: ./.github/workflows/remote-test.yml - with: - metadata-matrix: '["zookeeper"]' - assemebly-test: - uses: ./.github/workflows/assembly-test.yml +# standalone-test: +# uses: ./.github/workflows/standalone-test.yml +# with: +# metadata-matrix: '["zookeeper"]' +# standalone-test-pushdown: +# uses: ./.github/workflows/standalone-test-pushdown.yml +# with: +# metadata-matrix: '["zookeeper"]' +# db-ce: +# uses: ./.github/workflows/DB-CE.yml +# with: +# metadata-matrix: '["zookeeper"]' +# remote-test: +# uses: ./.github/workflows/remote-test.yml +# with: +# metadata-matrix: '["zookeeper"]' +# assemebly-test: +# uses: ./.github/workflows/assembly-test.yml diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java index 8b3bde7220..9ed8e7376c 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java @@ -311,6 +311,16 @@ public void clearData() throws SessionException { @Test public void test() { try { + String s = + "explain select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 10;"; + SessionExecuteSqlResult res = null; + try { + res = session.executeSql(s); + } catch (SessionException e) { + LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); + fail(); + } + res.print(false, ""); // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); // 执行垃圾回收，尽量释放内存 From 10ac5a77b5c9646e08372b2120fd7daa56602ae9 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 00:20:35 +0800 Subject: [PATCH 218/229] debug --- .github/workflows/standard-test-suite.yml | 16 ++++++++-------- .github/workflows/tpc-h.yml | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index 7e268585b0..a471709f55 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -13,14 +13,14 @@ concurrency: cancel-in-progress: true jobs: -# unit-test: -# uses: ./.github/workflows/unit-test.yml -# unit-mds: -# uses: ./.github/workflows/unit-mds.yml -# case-regression: -# uses: ./.github/workflows/case-regression.yml -# with: -# metadata-matrix: '["zookeeper"]' + # unit-test: + # uses: ./.github/workflows/unit-test.yml + # unit-mds: + # uses: ./.github/workflows/unit-mds.yml + # case-regression: + # uses: ./.github/workflows/case-regression.yml + # with: + # metadata-matrix: '["zookeeper"]' tpc-h-regression-test: uses: ./.github/workflows/tpc-h.yml with: diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index e6ba2d8b8f..8a81e69bd5 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -95,16 +95,16 @@ jobs: shell: bash run: | mvn clean package -DskipTests -P-format -q - git clone https://github.com/IGinX-THU/IGinX.git - cd IGinX - mvn clean package -DskipTests -P-format -q + # git clone https://github.com/IGinX-THU/IGinX.git + # cd IGinX + # mvn clean package -DskipTests -P-format -q - name: Change IGinX config uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} - Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" + # Set-Filter-Fragment-OFF: "true" + # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner From 7a01af1d74317c416c2be199de0406a8f74b89b7 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 00:55:34 +0800 Subject: [PATCH 219/229] debug --- .github/workflows/tpc-h.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 8a81e69bd5..62493da463 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -94,7 +94,7 @@ jobs: - name: Get Old Version shell: bash run: | - mvn clean package -DskipTests -P-format -q + mvn clean install -DskipTests -P-format -q # git clone https://github.com/IGinX-THU/IGinX.git # cd IGinX # mvn clean package -DskipTests -P-format -q @@ -143,7 +143,7 @@ jobs: - name: Install New IGinX with Maven shell: bash run: | - mvn clean package -DskipTests -P-format -q + mvn clean install -DskipTests -P-format -q - name: Change IGinX config uses: ./.github/actions/confWriter From 27b803d63ac12c74d0b79c3d920ee6f81c1efa95 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 01:00:12 +0800 Subject: [PATCH 220/229] debug --- .github/workflows/tpc-h.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 62493da463..f174e320d6 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -103,7 +103,7 @@ jobs: uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} - # Set-Filter-Fragment-OFF: "true" + Set-Filter-Fragment-OFF: "true" # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX From 9a2504b2ecee035b2eb179fe8a3315774c18a440 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 01:09:16 +0800 Subject: [PATCH 221/229] debug --- .github/workflows/tpc-h.yml | 13 +++++++------ .../cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index f174e320d6..bcfe5ff022 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -94,17 +94,17 @@ jobs: - name: Get Old Version shell: bash run: | - mvn clean install -DskipTests -P-format -q - # git clone https://github.com/IGinX-THU/IGinX.git - # cd IGinX - # mvn clean package -DskipTests -P-format -q + mvn clean package -DskipTests -P-format -q + git clone https://github.com/IGinX-THU/IGinX.git + cd IGinX + mvn clean package -DskipTests -P-format -q - name: Change IGinX config uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" - # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner @@ -143,13 +143,14 @@ jobs: - name: Install New IGinX with Maven shell: bash run: | - mvn clean install -DskipTests -P-format -q + mvn clean package -DskipTests -P-format -q - name: Change IGinX config uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" + Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start New IGinX uses: ./.github/actions/iginxRunner diff --git a/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index 218a021b66..fbfdbb5b8d 100644 --- a/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -128,7 +128,7 @@ private boolean testConnection() { session.open(false); session.close(); } catch (IoTDBConnectionException e) { - LOGGER.error("test connection error: ", e); + LOGGER.debug("test connection error: ", e); return false; } return true; @@ -176,7 +176,7 @@ record = dataSet.next(); columnsInterval = new ColumnsInterval(dataPrefix); } } catch (IoTDBConnectionException | StatementExecutionException e) { - LOGGER.error("get time series failure: ", e); + LOGGER.debug("get time series failure: ", e); throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } @@ -245,7 +245,7 @@ private void getColumns2StorageUnit(List columns, Map co } dataSet.close(); } catch (IoTDBConnectionException | StatementExecutionException e) { - LOGGER.error("get time series failure: ", e); + LOGGER.debug("get time series failure: ", e); throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } } @@ -317,7 +317,7 @@ private TaskExecuteResult executeProjectWithFilter( sessionPool.executeQueryStatement(statement), true, project, filter)); return new TaskExecuteResult(rowStream); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { - LOGGER.error("execute project task in iotdb12 failure, caused by: ", e); + LOGGER.debug("execute project task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( new IoTDBTaskExecuteFailureException("execute project task in iotdb12 failure", e)); } @@ -374,7 +374,7 @@ private TaskExecuteResult executeProjectDummyWithFilter(Project project, Filter sessionPool.executeQueryStatement(statement), false, project, filter)); return new TaskExecuteResult(rowStream); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { - LOGGER.error("execute project task in iotdb12 failure, caused by: ", e); + LOGGER.debug("execute project task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( new IoTDBTaskExecuteFailureException("execute project task in iotdb12 failure", e)); } @@ -400,7 +400,7 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { break; } if (e != null) { - LOGGER.error("execute insert task in iotdb12 failure, caused by: ", e); + LOGGER.debug("execute insert task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( null, new IoTDBException("execute insert task in iotdb12 failure", e)); } From fd064d708b4a5d0ee2ea3d1a23aa6dcb56dde847 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 10:01:21 +0800 Subject: [PATCH 222/229] open pushdown --- .github/workflows/tpc-h.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index bcfe5ff022..d6511ac5e0 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -95,22 +95,25 @@ jobs: shell: bash run: | mvn clean package -DskipTests -P-format -q - git clone https://github.com/IGinX-THU/IGinX.git - cd IGinX - mvn clean package -DskipTests -P-format -q + # git clone https://github.com/IGinX-THU/IGinX.git + # cd IGinX + # mvn clean package -DskipTests -P-format -q - name: Change IGinX config uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} + Push-Down: "true" Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" + Metadata: ${{ matrix.metadata }} + # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner with: version: ${VERSION} if-test-udf: "true" + # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Run Regression Test on Old IGinX if: always() @@ -149,8 +152,9 @@ jobs: uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} + Push-Down: "true" Set-Filter-Fragment-OFF: "true" - Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" + Metadata: ${{ matrix.metadata }} - name: Start New IGinX uses: ./.github/actions/iginxRunner From 933a552dbdb28ccf87049d6cb328f99469928db9 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 10:21:13 +0800 Subject: [PATCH 223/229] close push down --- .github/workflows/tpc-h.yml | 2 -- .../iginx/logical/optimizer/rules/ColumnPruningRule.java | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index d6511ac5e0..305651a52e 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -103,7 +103,6 @@ jobs: uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} - Push-Down: "true" Set-Filter-Fragment-OFF: "true" Metadata: ${{ matrix.metadata }} # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" @@ -152,7 +151,6 @@ jobs: uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} - Push-Down: "true" Set-Filter-Fragment-OFF: "true" Metadata: ${{ matrix.metadata }} diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java index e84bb78259..126818c50a 100644 --- a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java @@ -76,6 +76,7 @@ public void onMatch(RuleCall call) { 大致优化思路是，自顶向下地遍历树，对于每个节点，我们都会收集它的列，然后递归地处理它的子节点。 对于不同的节点，我们会进行根据列收集进行裁剪或将列加入收集中。 */ + LOGGER.debug("ColumnPruningRule on Match!"); collectColumns(call.getMatchedRoot(), new HashSet<>(), null, new ArrayList<>()); } @@ -347,6 +348,9 @@ private void collectColumns( rightColumns.add(column); } } + LOGGER.debug("columns: {}", columns); + LOGGER.debug("leftColumns: {}", leftColumns); + LOGGER.debug("rightColumns: {}", rightColumns); } else { leftColumns.addAll(columns); rightColumns.addAll(columns); From 26784ffc761ccd98bee9fc2aa1a6b38f98874210 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 10:32:45 +0800 Subject: [PATCH 224/229] debug --- .github/workflows/tpc-h.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 305651a52e..764eec53d0 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -95,23 +95,23 @@ jobs: shell: bash run: | mvn clean package -DskipTests -P-format -q - # git clone https://github.com/IGinX-THU/IGinX.git - # cd IGinX - # mvn clean package -DskipTests -P-format -q + git clone https://github.com/IGinX-THU/IGinX.git + cd IGinX + mvn clean package -DskipTests -P-format -q - - name: Change IGinX config + - name: Change Old IGinX config uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" Metadata: ${{ matrix.metadata }} - # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" + Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" + # with: + # version: ${VERSION} + # if-test-udf: "true" # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Run Regression Test on Old IGinX @@ -147,7 +147,7 @@ jobs: run: | mvn clean package -DskipTests -P-format -q - - name: Change IGinX config + - name: Change New IGinX config uses: ./.github/actions/confWriter with: DB-name: ${{ matrix.DB-name }} @@ -156,9 +156,6 @@ jobs: - name: Start New IGinX uses: ./.github/actions/iginxRunner - with: - version: ${VERSION} - if-test-udf: "true" - name: Run Regression Test on New IGinX if: always() From 3615c37af42b0a73f234b6016d79bc16a2da8704 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 10:54:48 +0800 Subject: [PATCH 225/229] debug --- .github/actions/confWriter/action.yml | 9 +++++---- .github/workflows/tpc-h.yml | 4 ---- .../iginx/logical/optimizer/rules/ColumnPruningRule.java | 4 ---- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 7bcd6f0338..1e3e876b0c 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -144,6 +144,7 @@ runs: name: Set FragmentPruningByFilterRule OFF shell: bash run: | + echo "$PWD" if [ "$RUNNER_OS" == "Linux" ]; then sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then @@ -205,7 +206,7 @@ runs: exit 1 fi - - name: Copy Optimizer Jar - shell: bash - run: | - cp -f "${{ inputs.Root-Dir-Path }}/optimizer/src/main/resources/iginx-optimizer-${VERSION}.jar" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/lib/" +# - name: Copy Optimizer Jar +# shell: bash +# run: | +# cp -f "${{ inputs.Root-Dir-Path }}/optimizer/src/main/resources/iginx-optimizer-${VERSION}.jar" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/lib/" diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 764eec53d0..3467bb8ce5 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -109,10 +109,6 @@ jobs: - name: Start Old IGinX uses: ./.github/actions/iginxRunner - # with: - # version: ${VERSION} - # if-test-udf: "true" - # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Run Regression Test on Old IGinX if: always() diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java index 126818c50a..e84bb78259 100644 --- a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java @@ -76,7 +76,6 @@ public void onMatch(RuleCall call) { 大致优化思路是，自顶向下地遍历树，对于每个节点，我们都会收集它的列，然后递归地处理它的子节点。 对于不同的节点，我们会进行根据列收集进行裁剪或将列加入收集中。 */ - LOGGER.debug("ColumnPruningRule on Match!"); collectColumns(call.getMatchedRoot(), new HashSet<>(), null, new ArrayList<>()); } @@ -348,9 +347,6 @@ private void collectColumns( rightColumns.add(column); } } - LOGGER.debug("columns: {}", columns); - LOGGER.debug("leftColumns: {}", leftColumns); - LOGGER.debug("rightColumns: {}", rightColumns); } else { leftColumns.addAll(columns); rightColumns.addAll(columns); From e84727598dcd93e00ff556f58c4a765845e760af Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 11:38:21 +0800 Subject: [PATCH 226/229] debug --- .github/actions/confWriter/action.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/confWriter/action.yml b/.github/actions/confWriter/action.yml index 1e3e876b0c..7bcd6f0338 100644 --- a/.github/actions/confWriter/action.yml +++ b/.github/actions/confWriter/action.yml @@ -144,7 +144,6 @@ runs: name: Set FragmentPruningByFilterRule OFF shell: bash run: | - echo "$PWD" if [ "$RUNNER_OS" == "Linux" ]; then sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties elif [ "$RUNNER_OS" == "Windows" ]; then @@ -206,7 +205,7 @@ runs: exit 1 fi -# - name: Copy Optimizer Jar -# shell: bash -# run: | -# cp -f "${{ inputs.Root-Dir-Path }}/optimizer/src/main/resources/iginx-optimizer-${VERSION}.jar" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/lib/" + - name: Copy Optimizer Jar + shell: bash + run: | + cp -f "${{ inputs.Root-Dir-Path }}/optimizer/src/main/resources/iginx-optimizer-${VERSION}.jar" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/lib/" From 6aebda2678744a6fcc7265cbfc1de06931ce0017 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 11:53:25 +0800 Subject: [PATCH 227/229] debug --- .github/workflows/tpc-h.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tpc-h.yml b/.github/workflows/tpc-h.yml index 3467bb8ce5..85ebd99fba 100644 --- a/.github/workflows/tpc-h.yml +++ b/.github/workflows/tpc-h.yml @@ -95,9 +95,9 @@ jobs: shell: bash run: | mvn clean package -DskipTests -P-format -q - git clone https://github.com/IGinX-THU/IGinX.git - cd IGinX - mvn clean package -DskipTests -P-format -q + # git clone https://github.com/IGinX-THU/IGinX.git + # cd IGinX + # mvn clean package -DskipTests -P-format -q - name: Change Old IGinX config uses: ./.github/actions/confWriter @@ -105,10 +105,12 @@ jobs: DB-name: ${{ matrix.DB-name }} Set-Filter-Fragment-OFF: "true" Metadata: ${{ matrix.metadata }} - Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" + # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Start Old IGinX uses: ./.github/actions/iginxRunner + # with: + # Root-Dir-Path: "${GITHUB_WORKSPACE}/IGinX" - name: Run Regression Test on Old IGinX if: always() From 722a7642e8972f8c4e25132ad381c6ff592918fd Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 12:23:20 +0800 Subject: [PATCH 228/229] remove unsed debug info --- .github/workflows/standard-test-suite.yml | 64 +++++++++---------- .../tsinghua/iginx/iotdb/IoTDBStorage.java | 7 +- .../integration/tpch/TPCHRegressionIT.java | 10 --- 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/.github/workflows/standard-test-suite.yml b/.github/workflows/standard-test-suite.yml index a471709f55..67e41e0f75 100644 --- a/.github/workflows/standard-test-suite.yml +++ b/.github/workflows/standard-test-suite.yml @@ -1,46 +1,44 @@ name: Standard Test Suite -#on:1 -# pull_request: # when a PR is opened or reopened -# types: [opened, reopened] -# branches: -# - main - -on: [pull_request] +on: + pull_request: # when a PR is opened or reopened + types: [opened, reopened] + branches: + - main concurrency: group: "${{ github.workflow }}-${{ github.ref }}" cancel-in-progress: true jobs: - # unit-test: - # uses: ./.github/workflows/unit-test.yml - # unit-mds: - # uses: ./.github/workflows/unit-mds.yml - # case-regression: - # uses: ./.github/workflows/case-regression.yml - # with: - # metadata-matrix: '["zookeeper"]' + unit-test: + uses: ./.github/workflows/unit-test.yml + unit-mds: + uses: ./.github/workflows/unit-mds.yml + case-regression: + uses: ./.github/workflows/case-regression.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test: + uses: ./.github/workflows/standalone-test.yml + with: + metadata-matrix: '["zookeeper"]' + standalone-test-pushdown: + uses: ./.github/workflows/standalone-test-pushdown.yml + with: + metadata-matrix: '["zookeeper"]' + db-ce: + uses: ./.github/workflows/DB-CE.yml + with: + metadata-matrix: '["zookeeper"]' + remote-test: + uses: ./.github/workflows/remote-test.yml + with: + metadata-matrix: '["zookeeper"]' + assemebly-test: + uses: ./.github/workflows/assembly-test.yml tpc-h-regression-test: uses: ./.github/workflows/tpc-h.yml with: os-matrix: '["ubuntu-latest"]' metadata-matrix: '["zookeeper"]' -# standalone-test: -# uses: ./.github/workflows/standalone-test.yml -# with: -# metadata-matrix: '["zookeeper"]' -# standalone-test-pushdown: -# uses: ./.github/workflows/standalone-test-pushdown.yml -# with: -# metadata-matrix: '["zookeeper"]' -# db-ce: -# uses: ./.github/workflows/DB-CE.yml -# with: -# metadata-matrix: '["zookeeper"]' -# remote-test: -# uses: ./.github/workflows/remote-test.yml -# with: -# metadata-matrix: '["zookeeper"]' -# assemebly-test: -# uses: ./.github/workflows/assembly-test.yml diff --git a/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java index fbfdbb5b8d..972985280c 100644 --- a/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java +++ b/dataSource/iotdb12/src/main/java/cn/edu/tsinghua/iginx/iotdb/IoTDBStorage.java @@ -128,7 +128,7 @@ private boolean testConnection() { session.open(false); session.close(); } catch (IoTDBConnectionException e) { - LOGGER.debug("test connection error: ", e); + LOGGER.error("test connection error: ", e); return false; } return true; @@ -176,7 +176,6 @@ record = dataSet.next(); columnsInterval = new ColumnsInterval(dataPrefix); } } catch (IoTDBConnectionException | StatementExecutionException e) { - LOGGER.debug("get time series failure: ", e); throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } @@ -245,7 +244,6 @@ private void getColumns2StorageUnit(List columns, Map co } dataSet.close(); } catch (IoTDBConnectionException | StatementExecutionException e) { - LOGGER.debug("get time series failure: ", e); throw new IoTDBTaskExecuteFailureException("get time series failure: ", e); } } @@ -317,7 +315,6 @@ private TaskExecuteResult executeProjectWithFilter( sessionPool.executeQueryStatement(statement), true, project, filter)); return new TaskExecuteResult(rowStream); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { - LOGGER.debug("execute project task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( new IoTDBTaskExecuteFailureException("execute project task in iotdb12 failure", e)); } @@ -374,7 +371,6 @@ private TaskExecuteResult executeProjectDummyWithFilter(Project project, Filter sessionPool.executeQueryStatement(statement), false, project, filter)); return new TaskExecuteResult(rowStream); } catch (IoTDBConnectionException | StatementExecutionException | PhysicalException e) { - LOGGER.debug("execute project task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( new IoTDBTaskExecuteFailureException("execute project task in iotdb12 failure", e)); } @@ -400,7 +396,6 @@ public TaskExecuteResult executeInsert(Insert insert, DataArea dataArea) { break; } if (e != null) { - LOGGER.debug("execute insert task in iotdb12 failure, caused by: ", e); return new TaskExecuteResult( null, new IoTDBException("execute insert task in iotdb12 failure", e)); } diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java index 9ed8e7376c..8b3bde7220 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/tpch/TPCHRegressionIT.java @@ -311,16 +311,6 @@ public void clearData() throws SessionException { @Test public void test() { try { - String s = - "explain select o_orderkey, extractYear(o_orderdate) from orders order by o_orderkey limit 10;"; - SessionExecuteSqlResult res = null; - try { - res = session.executeSql(s); - } catch (SessionException e) { - LOGGER.error("Statement: \"{}\" execute fail. Caused by:", s, e); - fail(); - } - res.print(false, ""); // 获取当前JVM的Runtime实例 Runtime runtime = Runtime.getRuntime(); // 执行垃圾回收，尽量释放内存 From 99e3c0769b4b37c72bf999dbc5df3f2121ba6a55 Mon Sep 17 00:00:00 2001 From: jzl18thu Date: Fri, 12 Jul 2024 14:32:03 +0800 Subject: [PATCH 229/229] fix test --- .../logical/optimizer/rules/ColumnPruningRule.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java index 808dfc31ae..5583f00048 100644 --- a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/ColumnPruningRule.java @@ -334,11 +334,17 @@ private void collectColumns( ((OperatorSource) ((BinaryOperator) operator).getSourceB()).getOperator(), new ArrayList<>()); for (String column : columns) { - if (leftPatterns.contains(column)) { - leftColumns.add(column); + for (String leftPattern : leftPatterns) { + if (OperatorUtils.covers(leftPattern, column)) { + leftColumns.add(column); + break; + } } - if (rightPatterns.contains(column)) { - rightColumns.add(column); + for (String rightPattern : rightPatterns) { + if (OperatorUtils.covers(rightPattern, column)) { + rightColumns.add(column); + break; + } } } } else {