feat(inception): initial commit

.github/workflows/nodejs.yml

@@ -0,0 +1,62 @@
+name: Node build, test and publish
+
+on:
+  pull_request:
+    types: [assigned, opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cleanup stale actions
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Use Node.js 18
+      uses: actions/setup-node@v3
+      with:
+        node-version: 20
+        cache: yarn
+    - name: npm install, lint, build, and test
+      run: |
+        yarn install --immutable
+        yarn lint
+        yarn build
+        yarn test
+      env:
+        CI: true
+
+  publish-npm:
+    needs: build
+    if: github.ref == 'refs/heads/main'
+    permissions:
+      contents: write
+      issues: write
+      id-token: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+          cache: yarn
+      - run: yarn install --immutable
+      - run: yarn build
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.SESAMECARE_OSS_NPM_TOKEN }}
+        run: |
+          yarn dlx semantic-release

.gitignore

@@ -0,0 +1,68 @@
+# Custom
+src/generated
+__tests__/generated
+.transpiled
+.nyc_output
+.eslintcache
+
+# TypeScript incremental compilation cache
+*.tsbuildinfo
+
+# Stock
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+*.orig
+
+work
+/build
+pids
+logs
+results
+coverage
+lib-cov
+html-report
+xunit.xml
+node_modules
+npm-debug.log
+
+.project
+.idea
+.settings
+.iml
+*.sublime-workspace
+*.sublime-project
+
+.DS_Store*
+ehthumbs.db
+Icon?
+Thumbs.db
+.AppleDouble
+.LSOverride
+.Spotlight-V100
+.Trashes
+
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+.node_repl_history
+
+# TypeScript incremental compilation cache
+*.tsbuildinfo
+# Added by coconfig
+.eslintignore
+.npmignore
+tsconfig.json
+tsconfig.build.json
+.prettierrc.js
+.eslintrc.js
+.commitlintrc.json
+vitest.config.ts

.vscode/settings.json

@@ -0,0 +1,14 @@
+{
+  "files.exclude": {
+    "**/.git": true,
+    "**/.svn": true,
+    "**/.hg": true,
+    "**/CVS": true,
+    "**/.DS_Store": true,
+    "**/Thumbs.db": true,
+    "**/.retool_types/**": true,
+    "**/*tsconfig.json": true,
+    ".cache": true,
+    "retool.config.json": true
+  }
+}

.yarnrc.yml

@@ -0,0 +1,7 @@
+nodeLinker: node-modules
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
+    spec: "@yarnpkg/plugin-interactive-tools"
+
+yarnPath: .yarn/releases/yarn-3.6.0.cjs

Makefile

@@ -0,0 +1,30 @@
+.PHONY: all clean
+
+# Postgres database things
+export PGUSER ?= postgres
+export PGPASSWORD ?= postgres
+export PGHOST ?= localhost
+
+src_files := $(shell find src -name '*.ts')
+build_files := $(patsubst src/%.ts,$(build_dir)/%.js,$(src_files))
+
+all: dbi ts
+
+# Typescript items
+ts: $(word 1, $(build_files))
+
+$(word 1, $(build_files)): $(src_files)
+	./node_modules/.bin/tsc -p tsconfig.build.json
+
+dbi: __tests__/generated/database.ts
+
+clean:
+	rm -rf __tests__/generated
+	rm -rf build/
+
+__tests__/generated/database.ts:
+	echo "Generating database types"
+	DATABASE_URL=postgres://$(PGUSER):$(PGPASSWORD)@$(PGHOST)/chtest yarn kysely-codegen \
+	        --dialect postgres --schema public \
+	        --out-file __tests__/generated/database.ts
+	yarn lint --fix __tests__/generated/database.ts

README.md

@@ -0,0 +1,45 @@
+# to-clickhouse
+
+@sesamecare-oss/to-clickhouse is a set of utilities meant to make it easier to synchronize data between a relational store such as Postgres and a Clickhouse database.
+
+## "Fact" tables
+
+Low row count tables such as "address types" can be synchronized using a simple copy strategy. Given a Clickhouse table like so:
+
+```sql
+CREATE TABLE identity__address_types (
+    address_type_id Int32,
+    name String,
+    created_at DateTime64
+) ENGINE = ReplacingMergeTree(created_at)
+ORDER BY address_type_id;
+```
+
+...this module can copy that to Clickhouse from Postgres like so:
+
+```typescript
+await synchronizeTable({
+      getRows(bookmark, limit) {
+        return db
+          .selectFrom('address_types')
+          .selectAll()
+          .where((eb) => bookmark?.rowId ? eb('address_type_id', '>', Number(bookmark.rowId)) : eb.val(true))
+          .orderBy('address_type_id')
+          .limit(limit)
+          .stream();
+      },
+      getBookmark(row) {
+        return {
+          rowId: String(row.address_type_id),
+          rowTimestamp: row.created_at as Date,
+        };
+      },
+      insert(stream) {
+        return ch.insert({
+          table: 'identity__address_types',
+          values: stream,
+          format: 'JSONEachRow',
+        });
+      },
+    }, {})
+```

__tests__/db/clickhouse.sql

@@ -0,0 +1,6 @@
+CREATE TABLE identity__address_types (
+    address_type_id Int32,
+    name String,
+    created_at DateTime64
+) ENGINE = ReplacingMergeTree(created_at)
+ORDER BY address_type_id;

__tests__/db/pg.sql

@@ -0,0 +1,76 @@
+CREATE TYPE favorite_color_enum AS ENUM (
+  'red',
+  'blue',
+  'other'
+);
+
+CREATE
+OR REPLACE FUNCTION set_updated_at () RETURNS TRIGGER AS $$
+BEGIN
+NEW.updated_at = now();
+RETURN NEW;
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE TABLE individuals (
+    individual_id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    individual_uuid uuid NOT NULL DEFAULT gen_random_uuid(),
+    created_at timestamp without time zone NOT NULL DEFAULT timezone('utc'::text, now()),
+    favorite_color favorite_color_enum,
+    updated_at timestamp with time zone NOT NULL DEFAULT now()
+);
+
+-- Indices -------------------------------------------------------
+
+CREATE INDEX idx_individuals_updated_at ON individuals(updated_at timestamptz_ops);
+
+-- Triggers -------------------------------------------------------
+
+CREATE TRIGGER individuals_setupdate
+  BEFORE UPDATE ON public.individuals
+  FOR EACH ROW
+  EXECUTE FUNCTION public.set_updated_at();
+
+CREATE TABLE address_types (
+    address_type_id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    name text NOT NULL UNIQUE UNIQUE,
+    created_at timestamp without time zone NOT NULL DEFAULT timezone('utc'::text, now())
+);
+
+-- DDL generated by Postico 2.0.5
+-- Not all database features are supported. Do not use for backup.
+
+-- Table Definition ----------------------------------------------
+
+CREATE TABLE individual_addresses (
+    individual_address_id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+    individual_id bigint NOT NULL REFERENCES individuals(individual_id),
+    address_uuid uuid NOT NULL,
+    address_type_id integer REFERENCES address_types(address_type_id),
+    created_at timestamp without time zone NOT NULL DEFAULT timezone('utc'::text, now()),
+    deleted_at timestamp without time zone,
+    updated_at timestamp with time zone NOT NULL DEFAULT now()
+);
+
+-- Indices -------------------------------------------------------
+
+CREATE UNIQUE INDEX address_by_individual ON individual_addresses(individual_id int8_ops,address_type_id int4_ops) WHERE deleted_at IS NULL;
+CREATE INDEX idx_addresses_updated_at ON individual_addresses(updated_at timestamptz_ops);
+
+-- Triggers -------------------------------------------------------
+
+CREATE TRIGGER addresses_setupdate
+  BEFORE UPDATE ON public.individual_addresses
+  FOR EACH ROW
+  EXECUTE FUNCTION public.set_updated_at();
+
+INSERT INTO address_types (name) VALUES ('home');
+INSERT INTO address_types (name) VALUES ('office');
+
+INSERT INTO individuals (favorite_color) VALUES ('red');
+INSERT INTO individuals (favorite_color) VALUES ('blue');
+INSERT INTO individuals (favorite_color) VALUES ('other');
+
+INSERT INTO individual_addresses (individual_id, address_uuid, address_type_id) VALUES (1, gen_random_uuid(), 1);
+INSERT INTO individual_addresses (individual_id, address_uuid, address_type_id) VALUES (1, gen_random_uuid(), 2);
+INSERT INTO individual_addresses (individual_id, address_uuid, address_type_id) VALUES (3, gen_random_uuid(), 1);