feat(vertica-driver): Introduce VerticaDriver (#9081)

* feat(vertica-driver): VerticaDriver

* fixed linter error

* fix dependencies versions

* driver & smoke tests

* vertica driver added to integration tests

* add vertica to smoke test

* update packages

* vertica driver published

* vertica driver

* update README.md

* docker installation instruction

* up version

* update README and package refs

* add cube build pipeline

* remove

* Added Query Implementation (#2)

* Added custom dialect for CubeDriver

* Sql fixes (#5)

* Additional sql fixes (#6)

* draft

* update test

* fix build

* update vertica driver dependencies

* vertica documentation

* add vertica to data sources

* fix lint errors

* update jest

* include vertica tests

* update wait strategy and test shared package

* change version of vertica driver

* some ci-related fixes

* fix version, names, refs to align with the Cube repo

* update yarn.lock

* fix old refs

---------

Co-authored-by: Eduard Karacharov <eduard.karacharov@gmail.com>
Co-authored-by: timbrownls20 <timbrownls26@hotmail.com>
Co-authored-by: Roger <qi.chen@knowitall.net.au>
Co-authored-by: David O'Bryen <dj.obryen@gmail.com>

.github/actions/integration/vertica.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eo pipefail
+
+# Debug log for test containers
+export DEBUG=testcontainers
+
+export TEST_VERTICA_VERSION=12.0.4-0
+
+echo "::group::Vertica ${TEST_VERTICA_VERSION}"
+docker pull vertica/vertica-ce:${TEST_VERTICA_VERSION}
+yarn lerna run --concurrency 1 --stream --no-prefix integration:vertica
+echo "::endgroup::"

.github/actions/smoke.sh

@@ -57,6 +57,10 @@ echo "::group::MongoBI"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:mongobi
 echo "::endgroup::"
 
+echo "::group::Vertica"
+yarn lerna run --concurrency 1 --stream --no-prefix smoke:vertica
+echo "::endgroup::"
+
 echo "::group::RBAC"
 yarn lerna run --concurrency 1 --stream --no-prefix smoke:rbac
 echo "::endgroup::"

.github/workflows/drivers-tests.yml

@@ -24,6 +24,7 @@ on:
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
+      - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'
@@ -49,6 +50,7 @@ on:
       - 'packages/cubejs-mysql-driver/**'
       - 'packages/cubejs-postgres-driver/**'
       - 'packages/cubejs-snowflake-driver/**'
+      - 'packages/cubejs-vertica-driver/**'
 
       # To test SQL API Push down
       - 'packages/cubejs-backend-native/**'

.github/workflows/push.yml

@@ -336,7 +336,7 @@ jobs:
         node-version: [20.x]
         db: [
             'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
-            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio'
+            'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio', 'vertica'
         ]
       fail-fast: false
 

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

docs/pages/product/configuration/data-sources.mdx

@@ -59,6 +59,11 @@ sources][ref-config-multi-data-src] out of the box.
     imageUrl="https://static.cube.dev/icons/firebolt.svg"
     title="Firebolt"
   />
+   <GridItem
+    url="data-sources/vertica"
+    imageUrl="https://static.cube.dev/icons/vertica.svg"
+    title="Vertica"
+  />
 </Grid>
 
 ## Query engines

docs/pages/product/configuration/data-sources/vertica.mdx

@@ -0,0 +1,46 @@
+# Vertica
+
+## Prerequisites
+
+- The hostname for the [Vertica][vertica] database server
+- The username/password for the [Vertica][vertica] database server
+- The name of the database to use within the [Vertica][vertica] database server
+
+## Setup
+
+### Manual
+
+Add the following to a `.env` file in your Cube project:
+
+```dotenv
+CUBEJS_DB_TYPE=vertica
+CUBEJS_DB_HOST=my.vertica.host
+CUBEJS_DB_USER=vertica_user
+CUBEJS_DB_PASS=**********
+CUBEJS_DB_SCHEMA=my_vertica_schema
+```
+
+## Environment Variables
+
+| Environment Variable       | Description                                                                         | Possible Values                               | Required |
+| -------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------- | :------: |
+| `CUBEJS_DB_HOST`           | The host URL for a database                                                         | A valid database host URL                     |    ✅    |
+| `CUBEJS_DB_PORT`           | The port for the database connection                                                | A valid port number                           |    ❌    |
+| `CUBEJS_DB_USER`           | The username used to connect to the database                                        | A valid database username                     |    ✅    |
+| `CUBEJS_DB_PASS`           | The password used to connect to the database                                        | A valid database password                     |    ✅    |
+| `CUBEJS_DB_SCHEMA`         | The schema within the database to connect to                                        | A valid schema name within a Presto database  |    ✅    |
+| `CUBEJS_DB_SSL`            | If `true`, enables SSL encryption for database connections from Cube                | `true`, `false`                               |    ❌    |
+| `CUBEJS_CONCURRENCY`       | The number of concurrent connections each queue has to the database. Default is `2` | A valid number                                |    ❌    |
+| `CUBEJS_DB_MAX_POOL`       | The maximum number of concurrent database connections to pool. Default is `8`       | A valid number                                |    ❌    |
+
+
+## SSL
+
+To enable SSL-encrypted connections between Cube and Verica, set the
+`CUBEJS_DB_SSL` environment variable to `true`. For more information on how to
+configure custom certificates, please check out [Enable SSL Connections to the
+Database][ref-recipe-enable-ssl].
+
+[vertica]: https://www.vertica.com/documentation/vertica/all/
+[ref-recipe-enable-ssl]:/guides/recipes/data-sources/using-ssl-connections-to-data-source
+

packages/cubejs-docker/dev.Dockerfile

@@ -73,6 +73,7 @@ COPY packages/cubejs-sqlite-driver/package.json packages/cubejs-sqlite-driver/pa
 COPY packages/cubejs-ksql-driver/package.json packages/cubejs-ksql-driver/package.json
 COPY packages/cubejs-dbt-schema-extension/package.json packages/cubejs-dbt-schema-extension/package.json
 COPY packages/cubejs-jdbc-driver/package.json packages/cubejs-jdbc-driver/package.json
+COPY packages/cubejs-vertica-driver/package.json packages/cubejs-vertica-driver/package.json
 # Skip
 # COPY packages/cubejs-testing/package.json packages/cubejs-testing/package.json
 # COPY packages/cubejs-docker/package.json packages/cubejs-docker/package.json
@@ -150,6 +151,7 @@ COPY packages/cubejs-ksql-driver/ packages/cubejs-ksql-driver/
 COPY packages/cubejs-dbt-schema-extension/ packages/cubejs-dbt-schema-extension/
 COPY packages/cubejs-jdbc-driver/ packages/cubejs-jdbc-driver/
 COPY packages/cubejs-databricks-jdbc-driver/ packages/cubejs-databricks-jdbc-driver/
+COPY packages/cubejs-vertica-driver/ packages/cubejs-vertica-driver/
 # Skip
 # COPY packages/cubejs-testing/ packages/cubejs-testing/
 # COPY packages/cubejs-docker/ packages/cubejs-docker/

packages/cubejs-docker/package.json

@@ -36,6 +36,7 @@
     "@cubejs-backend/snowflake-driver": "1.1.10",
     "@cubejs-backend/sqlite-driver": "1.1.10",
     "@cubejs-backend/trino-driver": "1.1.11",
+    "@cubejs-backend/vertica-driver": "1.1.11",
     "cubejs-cli": "1.1.11",
     "typescript": "~5.2.2"
   },

packages/cubejs-server-core/src/core/DriverDependencies.js

@@ -26,6 +26,7 @@ module.exports = {
   ksql: '@cubejs-backend/ksql-driver',
   questdb: '@cubejs-backend/questdb-driver',
   materialize: '@cubejs-backend/materialize-driver',
+  vertica: '@cubejs-backend/vertica-driver',
   pinot: '@cubejs-backend/pinot-driver',
   // List for JDBC drivers
   'databricks-jdbc': '@cubejs-backend/databricks-jdbc-driver',

packages/cubejs-server-core/src/core/server.ts

@@ -126,7 +126,8 @@ export class CubejsServerCore {
 
   protected readonly orchestratorStorage: OrchestratorStorage = new OrchestratorStorage();
 
-  protected repositoryFactory: ((_context: RequestContext) => SchemaFileRepository) | (() => FileRepository);
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  protected repositoryFactory: ((context: RequestContext) => SchemaFileRepository) | (() => FileRepository);
 
   protected contextToDbType: DbTypeAsyncFn;
 
@@ -162,7 +163,8 @@ export class CubejsServerCore {
 
   protected apiGatewayInstance: ApiGateway | null = null;
 
-  public readonly event: (_name: string, _props?: object) => Promise<void>;
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public readonly event: (name: string, props?: object) => Promise<void>;
 
   public projectFingerprint: string | null = null;
 

packages/cubejs-testing-shared/src/db-container-runners/index.ts

@@ -5,6 +5,7 @@ export * from './clickhouse';
 export * from './questdb';
 export * from './materialize';
 export * from './crate';
+export * from './vertica';
 export * from './prestodb';
 export * from './mssql';
 export * from './trino';

packages/cubejs-testing-shared/src/db-container-runners/oracle.ts

@@ -1,5 +1,4 @@
 import { GenericContainer, Wait } from 'testcontainers';
-import { isCI } from '@cubejs-backend/shared';
 
 import { DbRunnerAbstract, DBRunnerContainerOptions } from './db-runner.abstract';
 

packages/cubejs-testing-shared/src/db-container-runners/vertica.ts

@@ -0,0 +1,18 @@
+import { GenericContainer, Wait } from 'testcontainers';
+import { DbRunnerAbstract } from './db-runner.abstract';
+
+export class VerticaDBRunner extends DbRunnerAbstract {
+  public static startContainer() {
+    const version = process.env.TEST_VERTICA_VERSION || '12.0.4-0';
+
+    const container = new GenericContainer(`vertica/vertica-ce:${version}`)
+      .withEnvironment({ TZ: 'Antarctica/Troll', VERTICA_DB_NAME: 'test', VMART_ETL_SCRIPT: '', VMART_ETL_SQL: '' })
+      .withExposedPorts(5433)
+      .withStartupTimeout(60 * 1000)
+      .withWaitStrategy(
+        Wait.forLogMessage('Vertica is now running')
+      );
+
+    return container.start();
+  }
+}

packages/cubejs-testing/birdbox-fixtures/driver-test-data/CAST.js

@@ -23,6 +23,10 @@ export const DB_CAST = {
     SELECT_PREFIX: 'with tmp_tbl as (\n',
     SELECT_SUFFIX: ')\nselect * from tmp_tbl',
   },
+  vertica: {
+    SELECT_PREFIX: '',
+    SELECT_SUFFIX: '',
+  },
 };
 export const DATE_CAST = {
   athena: {
@@ -49,4 +53,8 @@ export const DATE_CAST = {
     DATE_PREFIX: 'to_date(',
     DATE_SUFFIX: ', \'YYYY-MM-DD\')',
   },
+  vertica: {
+    DATE_PREFIX: 'to_date(',
+    DATE_SUFFIX: ', \'YYYY-MM-DD\')',
+  },
 };

packages/cubejs-testing/birdbox-fixtures/vertica.yml

@@ -0,0 +1,44 @@
+version: "2.2"
+
+services:
+  cube:
+    container_name: birdbox-cube
+    image: ${BIRDBOX_CUBEJS_REGISTRY_PATH}cubejs/cube:${BIRDBOX_CUBEJS_VERSION:-latest}
+    environment:
+      CUBEJS_DB_TYPE: vertica
+
+      CUBEJS_DB_HOST: db
+      CUBEJS_DB_PORT: 5433
+      CUBEJS_DB_NAME: vmart
+      CUBEJS_DB_USER: dbadmin
+      CUBEJS_DB_PASS:
+
+      CUBEJS_DEV_MODE: "true"
+      CUBEJS_WEB_SOCKETS: "true"
+      CUBEJS_API_SECRET: mysupersecret
+    volumes:
+      - ./vertica/schema:/cube/conf/schema
+    ports:
+      - "4000"
+    depends_on:
+      db:
+        condition: service_healthy
+    links:
+      - db
+    restart: always
+
+  # Database instances, for troubleshooting.
+  # NOT required.
+  # Tests provide db instances via testcontainers, in both LOCAL and DOCKER modes.
+
+  db:
+    container_name: birdbox-db
+    image: vertica/vertica-ce:12.0.4-0
+    ports:
+      - "5433"
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "/opt/vertica/bin/vsql -c 'select 1;' || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 5

packages/cubejs-testing/birdbox-fixtures/vertica/schema/Orders.js

@@ -0,0 +1,34 @@
+cube(`Orders`, {
+  sql: `
+  with orders as (
+    select 1 as id, 100 as amount, 'new' status
+    UNION ALL
+    select 2 as id, 200 as amount, 'new' status
+    UNION ALL
+    select 3 as id, 300 as amount, 'processed' status
+    UNION ALL
+    select 4 as id, 500 as amount, 'processed' status
+    UNION ALL
+    select 5 as id, 600 as amount, 'shipped' status
+  )
+  select * from orders
+  `,
+  measures: {
+    count: {
+      type: `count`,
+    },
+    totalAmount: {
+      sql: `amount`,
+      type: `sum`,
+    },
+    toRemove: {
+      type: `count`,
+    },
+  },
+  dimensions: {
+    status: {
+      sql: `status`,
+      type: `string`,
+    },
+  },
+});

packages/cubejs-testing/package.json

@@ -46,6 +46,8 @@
     "driver:questdb:snap": "jest --verbose --updateSnapshot -i dist/test/driver-questdb.test.js",
     "driver:databricks": "jest --verbose -i dist/test/driver-databricks.test.js",
     "driver:databricks:snap": "jest --verbose --updateSnapshot -i dist/test/driver-databricks.test.js",
+    "driver:vertica": "jest --verbose -i dist/test/driver-vertica.test.js",
+    "driver:vertica:snap": "jest --verbose --updateSnapshot -i dist/test/driver-vertica.test.js",
     "integration:cubestore": "jest --verbose --updateSnapshot -i dist/test/driver-cubestore.test.js",
     "rest:postgres": "yarn tsc && clear && jest --verbose -i dist/test/rest-postgres.test.js",
     "smoke:athena": "jest --verbose -i dist/test/smoke-athena.test.js",
@@ -73,6 +75,8 @@
     "smoke:postgres": "jest --verbose -i dist/test/smoke-postgres.test.js",
     "smoke:redshift": "jest --verbose -i dist/test/smoke-redshift.test.js",
     "smoke:redshift:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-redshift.test.js",
+    "smoke:vertica": "jest --verbose -i dist/test/smoke-vertica.test.js",
+    "smoke:vertica:snapshot": "jest --verbose --updateSnapshot -i dist/test/smoke-vertica.test.js",
     "smoke:rbac": "TZ=UTC jest --verbose -i dist/test/smoke-rbac.test.js",
     "smoke:cubesql": "TZ=UTC jest --verbose --forceExit -i dist/test/smoke-cubesql.test.js",
     "smoke:cubesql:snapshot": "TZ=UTC jest --verbose --forceExit --updateSnapshot -i dist/test/smoke-cubesql.test.js",

packages/cubejs-testing/src/REQUIRED_ENV_VARS.ts

@@ -53,6 +53,7 @@ export const REQUIRED_ENV_VARS: {[key: string]: string[]} = {
     'CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET',
     'CUBEJS_DB_EXPORT_BUCKET_AWS_REGION',
   ],
+  vertica: [],
   prestodb: [],
   trino: [],
   mssql: [],

packages/cubejs-testing/src/birdbox.ts

@@ -48,7 +48,7 @@ interface Args {
   log: Log,
 }
 
-export type DriverType = 'postgresql' | 'postgres' | 'multidb' | 'materialize' | 'crate' | 'bigquery' | 'athena' | 'postgresql-cubestore' | 'firebolt' | 'questdb' | 'redshift' | 'databricks-jdbc' | 'prestodb' | 'mssql' | 'trino' | 'oracle' | 'duckdb' | 'snowflake';
+export type DriverType = 'postgresql' | 'postgres' | 'multidb' | 'materialize' | 'crate' | 'bigquery' | 'athena' | 'postgresql-cubestore' | 'firebolt' | 'questdb' | 'redshift' | 'databricks-jdbc' | 'prestodb' | 'mssql' | 'trino' | 'oracle' | 'duckdb' | 'snowflake' | 'vertica';
 
 export type Schemas = string[];
 
@@ -103,6 +103,7 @@ const driverNameToFolderNameMapper: Record<DriverType, string> = {
   questdb: 'postgresql',
   redshift: 'postgresql',
   'databricks-jdbc': 'databricks-jdbc',
+  vertica: 'vertica',
   prestodb: 'postgresql',
   mssql: 'mssql',
   trino: 'postgresql',