Merge branch 'master' into johnsca/fix/support-proxy-for-jwk-validation

cube-js · Jan 13, 2025 · f538e5e · f538e5e
2 parents 83ad8f0 + 80591ea
commit f538e5e
Show file tree

Hide file tree

Showing 421 changed files with 26,978 additions and 6,750 deletions.
diff --git a/.github/actions/integration/dremio.sh b/.github/actions/integration/dremio.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Dremio [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:dremio
+
+echo "::endgroup::"
diff --git a/.github/actions/integration/firebolt.sh b/.github/actions/integration/firebolt.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+echo "::group::Firebolt [cloud]"
+yarn lerna run --concurrency 1 --stream --no-prefix integration:firebolt
+
+echo "::endgroup::"
diff --git a/.github/actions/integration/vertica.sh b/.github/actions/integration/vertica.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+export TEST_VERTICA_VERSION=12.0.4-0
+
+echo "::group::Vertica ${TEST_VERTICA_VERSION}"
+docker pull vertica/vertica-ce:${TEST_VERTICA_VERSION}
+yarn lerna run --concurrency 1 --stream --no-prefix integration:vertica
+echo "::endgroup::"
diff --git a/.github/actions/smoke.sh b/.github/actions/smoke.sh
@@ -57,6 +57,10 @@ echo "::group::MongoBI"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mongobi
 echo "::endgroup::"
 
+echo "::group::Vertica"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:vertica
+echo "::endgroup::"
+
 echo "::group::RBAC"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:rbac
 echo "::endgroup::"
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,5 +1,5 @@
 **Check List**
-- [ ] Tests has been run in packages where changes made if available
+- [ ] Tests have been run in packages where changes made if available
 - [ ] Linter has been run for changed code
 - [ ] Tests for the changes have been added if not covered yet
 - [ ] Docs have been added / updated if required

diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml
@@ -24,6 +24,7 @@ on:
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
+      - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
@@ -49,6 +50,7 @@ on:
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
+      - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -152,7 +152,7 @@ jobs:
     strategy:
       matrix:
         node-version: [20.x]
-        os-version: ["macos-12"]
+        os-version: ["macos-13"]
         target: ["x86_64-apple-darwin", "aarch64-apple-darwin"]
         python-version: ["3.9", "3.10", "3.11", "3.12", "fallback"]
         include:
@@ -677,7 +677,7 @@ jobs:
             tar_executable: tar
             # Please use minimal possible version of macOS, because it produces constraint on libstdc++
           - target: x86_64-apple-darwin
-            os: macos-12
+            os: macos-13
             executable_name: cubestored
             # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.
             strip: false

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -42,6 +42,7 @@ jobs:
       matrix:
         # Current docker version + next LTS
         node-version: [20.x, 22.x]
+        python-version: [3.11]
       fail-fast: false
 
     steps:
@@ -65,6 +66,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -92,6 +97,12 @@ jobs:
         run: yarn tsc
       - name: Build client
         run: yarn build
+      - name: Build cubejs-backend-native (with Python)
+        run: yarn run native:build-release-python
+        working-directory: ./packages/cubejs-backend-native
+        env:
+          PYO3_PYTHON: python${{ matrix.python-version }}
+
       - name: Lerna test
         run: yarn lerna run --concurrency 1 --stream --no-prefix unit
 #      - uses: codecov/codecov-action@v1
@@ -313,13 +324,19 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: (needs['latest-tag-sha'].outputs.sha != github.sha)
+    env:
+      CLOUD_DATABASES: >
+        firebolt
+        dremio
+      # Athena (just to check for secrets availability)
+      DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }}
 
     strategy:
       matrix:
         node-version: [20.x]
         db: [
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
-            'mysql-aurora-serverless', 'crate', 'mongobi'
+            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio', 'vertica'
         ]
       fail-fast: false
 
@@ -370,6 +387,21 @@ jobs:
           retry_wait_seconds: 15
           timeout_minutes: 30
           command: ./.github/actions/integration/${{ matrix.db }}.sh
+        # It's enough to test for any one secret because they are set all at once or not set all
+        if: |
+          (contains(env.CLOUD_DATABASES, matrix.db) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') ||
+          (!contains(env.CLOUD_DATABASES, matrix.db))
+        env:
+          # Firebolt Integration
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }}
+          DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }}
+          # Dremio Integration
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }}
+          DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}
 
   integration-smoke:
     needs: [ latest-tag-sha, build-cubestore ]
@@ -380,6 +412,7 @@ jobs:
     strategy:
       matrix:
         node-version: [ 20.x ]
+        python-version: [ 3.11 ]
       fail-fast: false
 
     steps:
@@ -405,6 +438,10 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Get yarn cache directory path
         id: yarn-cache-dir-path
         run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -432,6 +469,11 @@ jobs:
         uses: GoodManWEN/oracle-client-action@main
       - name: Build client
         run: yarn build
+      - name: Build cubejs-backend-native (with Python)
+        run: yarn run native:build-release-python
+        working-directory: ./packages/cubejs-backend-native
+        env:
+          PYO3_PYTHON: python${{ matrix.python-version }}
       - name: Lerna tsc
         run: yarn tsc
       - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact

diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml
@@ -196,7 +196,7 @@ jobs:
             # cubestored.exe: CantPackException: superfluous data between sections
             compress: false
             # Please use minimal possible version of macOS, because it produces constraint on libstdc++
-          - os: macos-12
+          - os: macos-13
             target: x86_64-apple-darwin
             executable_name: cubestored
             # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.

diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml
@@ -127,7 +127,7 @@ jobs:
             # cubestored.exe: CantPackException: superfluous data between sections
             compress: false
             # Please use minimal possible version of macOS, because it produces constraint on libstdc++
-          - os: macos-12
+          - os: macos-13
             target: x86_64-apple-darwin
             executable_name: cubestored
             # upx has been disabled because it is crashing for macOS Ventura or above! It was disabled on 2024-09-05.

diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -0,0 +1 @@
+nodeLinker: node-modules