Create driver and migrations for spanner (psql) (#306)

* Create driver and migrations for spanner (psql) - [x] create seperate migration directories for spanner SQL files - [x] create `pgx` driver to use spanner-pg-adapter - [x] create additional config directory to configure `pgx` driver - [x] create docker compose file for building dev containers with spanener and spanner-pg-adapter Signed-off-by: Bailin He <bahe@equinix.com> * renamed enums Signed-off-by: Bailin He <bahe@equinix.com> * use spanner `.env` file Signed-off-by: Bailin He <bahe@equinix.com> * update helm chart Signed-off-by: Bailin He <bahe@equinix.com> * fix typo `db.engine` Signed-off-by: Bailin He <bahe@equinix.com> --------- Signed-off-by: Bailin He <bahe@equinix.com>
infratographer · Dec 18, 2024 · 1ffb114 · 1ffb114
1 parent f706f45
commit 1ffb114
Show file tree

Hide file tree

Showing 34 changed files with 616 additions and 43 deletions.
diff --git a/.devcontainer/.env b/.devcontainer/.env
@@ -19,8 +19,8 @@ IDENTITYAPI_TRACING_PROVIDER=jaeger
 IDENTITYAPI_TRACING_JAEGER_ENDPOINT=http://localhost:14268/api/traces
 IDENTITYAPI_CRDB_URI="postgresql://root@crdb:26257/identityapi_dev?sslmode=disable"
 
+PERMISSIONSAPI_DB_ENGINE=cockroach
 PERMISSIONSAPI_CRDB_URI="postgresql://root@crdb:26257/permissionsapi?sslmode=disable"
-
 PERMISSIONSAPI_TRACING_ENABLED=true
 PERMISSIONSAPI_TRACING_PROVIDER=otlpgrpc
 PERMISSIONSAPI_TRACING_OTLP_ENDPOINT=jaeger:4317

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -2,6 +2,9 @@
 {
 	"name": "permissions-api",
 	"dockerComposeFile":"docker-compose.yml",
+	// use the following instead if you want to use spanner and its emulator
+	// instead of crdb
+	// "dockerComposeFile":"spanner.docker-compose.yml",
 	"service": "app",
 	"workspaceFolder": "/workspace",
 	"shutdownAction": "stopCompose",

diff --git a/.devcontainer/spanner.docker-compose.yml b/.devcontainer/spanner.docker-compose.yml
@@ -0,0 +1,172 @@
+networks:
+  infradev:
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        VARIANT: 1.23-bullseye
+        NODE_VERSION: "none"
+    command: sleep infinity
+# uncomment along with bind volume to use SSH for dev container access
+#    ports:
+#      - "127.0.0.1:2224:2222"
+    env_file:
+      - spanner.env
+    volumes:
+      - ./nsc:/nsc
+      - ./nats:/nats
+      - ..:/workspace:cached
+#      - type: bind
+#        source: ~/.ssh/authorized_keys
+#        target: /home/vscode/.ssh/authorized_keys
+#        read_only: true
+    networks:
+      - infradev
+    # Use "forwardPorts" in **devcontainer.json** to forward a port locally.
+    # (Adding the "ports" property to this file will not forward from a Codespace.)
+
+  # DB
+  spanner:
+    image: gcr.io/cloud-spanner-emulator/emulator
+    networks:
+      - infradev
+    ports:
+      - "9010:9010"
+      - "9020:9020"
+
+  spanner-pg:
+    image: gcr.io/cloud-spanner-pg-adapter/pgadapter
+    command:
+      - "-p test-project"
+      - "-i test-instance"
+      - "-r autoConfigEmulator=true"
+      - "-e spanner:9010"
+      - "-c \"\""
+      - "-x"
+      - -ddl=AutocommitExplicitTransaction
+    ports:
+      - "5432:5432"
+    depends_on:
+      - spanner
+    networks:
+      - infradev
+
+  create-databases-pg:
+    image: postgres:13.4
+    restart: on-failure:5
+    command: |
+      psql -h spanner-pg -c 'CREATE DATABASE permissionsapi;'
+    depends_on:
+      - spanner-pg
+    networks:
+      - infradev
+
+  create-databases-spanner:
+    image: alpine/curl
+    restart: on-failure:5
+    command: |
+      curl -X POST \
+        http://spanner:9020/v1/projects/test-project/instances/test-instance/databases \
+        -H 'Content-Type: application/json' \
+        -d '{
+          "createStatement": "CREATE DATABASE spicedb",
+          "databaseDialect": "GOOGLE_STANDARD_SQL"
+        }'
+    depends_on:
+      create-databases-pg:
+        condition: service_completed_successfully
+    networks:
+      - infradev
+
+  create-goose-table:
+    image: postgres:13.4
+    depends_on:
+      create-databases-pg:
+        condition: service_completed_successfully
+    restart: on-failure:5
+    networks:
+      - infradev
+    command: |
+      psql -h spanner-pg -d projects/test-project/instances/test-instance/databases/permissionsapi \
+        -c \
+        'CREATE TABLE IF NOT EXISTS public.goose_db_version (
+          id BIGINT NOT NULL DEFAULT (extract(epoch from CURRENT_TIMESTAMP)*1000)::bigint,
+          version_id BIGINT NOT NULL,
+          is_applied BOOLEAN NOT NULL,
+          tstamp TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
+          PRIMARY KEY (id)
+        );
+
+        INSERT INTO goose_db_version (version_id, is_applied) VALUES (0, true);'
+
+  migrate_spicedb:
+    image: authzed/spicedb:v1.38.1
+    command: datastore migrate head
+    restart: on-failure:5
+    env_file:
+      - spanner.env
+    depends_on:
+      create-databases-pg:
+        condition: service_completed_successfully
+    networks:
+      - infradev
+
+  spicedb:
+    image: authzed/spicedb:v1.38.1
+    command: serve
+    restart: unless-stopped
+    env_file:
+      - spanner.env
+    depends_on:
+      migrate_spicedb:
+        condition: service_completed_successfully
+    ports:
+      - 50051:50051
+    networks:
+      - infradev
+
+  nats-init:
+    image: natsio/nats-box
+    environment:
+      - NSC_HOME=/nsc
+    volumes:
+      - ./nsc:/nsc
+      - ./nats:/nats
+      - ./scripts:/scripts
+    command:
+      - /scripts/nats_init.sh
+
+  nats:
+    image: 'nats:alpine'
+    depends_on:
+      - nats-init
+    command:
+      - -c
+      - '/etc/nats/nats-server.conf'
+      - -D
+    volumes:
+      - ./nats/:/etc/nats
+    restart: unless-stopped
+    networks:
+      - infradev
+
+  jaeger:
+    image: jaegertracing/all-in-one:1.60.0
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - 16688:16686
+    networks:
+      - infradev
+
+  mock-oauth2-server:
+    image: ghcr.io/navikt/mock-oauth2-server:2.1.10
+    networks:
+      - infradev
+    environment:
+      - PORT=8081
+    ports:
+      - 8081:8081
diff --git a/.devcontainer/spanner.env b/.devcontainer/spanner.env
@@ -0,0 +1,38 @@
+# spicedb container config
+SPICEDB_GRPC_PRESHARED_KEY=infradev
+SPICEDB_DATASTORE_ENGINE=spanner
+SPICEDB_DATASTORE_CONN_URI='projects/test-project/instances/test-instance/databases/spicedb'
+SPICEDB_DATASTORE_SPANNER_EMULATOR_HOST=spanner:9010
+SPICEDB_LOG_LEVEL=info
+SPICEDB_LOG_FORMAT=console
+SPICEDB_OTEL_PROVIDER=jaeger
+SPICEDB_OTEL_INSECURE=true
+SPICEDB_OTEL_ENDPOINT=http://app:14268/api/traces
+SPICEDB_TESTING_ONLY_SCHEMA_ADDITIVE_WRITES=true
+
+# zed CLI tool config
+ZED_ENDPOINT=spicedb:50051
+ZED_INSECURE=true
+ZED_TOKEN=infradev
+
+PERMISSIONSAPI_DB_DRIVER=postgres
+PERMISSIONSAPI_SPANNER_URI="postgresql://spanner-pg:5432/permissionsapi?sslmode=disable"
+PERMISSIONSAPI_TRACING_ENABLED=true
+PERMISSIONSAPI_TRACING_PROVIDER=otlpgrpc
+PERMISSIONSAPI_TRACING_OTLP_ENDPOINT=jaeger:4317
+PERMISSIONSAPI_TRACING_OTLP_INSECURE=true
+PERMISSIONSAPI_SPICEDB_ENDPOINT=spicedb:50051
+PERMISSIONSAPI_SPICEDB_KEY=infradev
+PERMISSIONSAPI_SPICEDB_INSECURE=true
+
+PERMISSIONSAPI_PUBSUB_NAME=permissionsapi
+PERMISSIONSAPI_PUBSUB_CREDENTIALS="/tmp/user.creds"
+PERMISSIONSAPI_PUBSUB_SERVER="nats://nats:4222"
+PERMISSIONSAPI_PUBSUB_STREAM="permissionsapi"
+PERMISSIONSAPI_PUBSUB_PREFIX="com.infratographer.events"
+
+NATS_URL="nats://nats:4222"
+NATS_CREDS="/tmp/user.creds"
+
+NKEYS_PATH="/workspace/.devcontainer/nsc/nkeys"
+NSC_HOME="/workspace/.devcontainer/nsc/nats"
diff --git a/README.md b/README.md
@@ -127,6 +127,12 @@ To get started, you can use either [VS Code][vs-code] or the official [CLI][cli]
 [vs-code]: https://code.visualstudio.com/docs/devcontainers/containers
 [cli]: https://github.com/devcontainers/cli
 
+### Spanner Emulator
+
+To develop on Spanner DB emulator:
+
+1. edit `.devcontainer/devcontainer.json` and use `spanner.docker-compose.yml`
+
 ### Manually setting up SSH agent forwarding
 
 The provided dev container listens for SSH connections on port 2222 and bind mounts `~/.ssh/authorized_keys` from the host to facilitate SSH. In order to perform Git operations (i.e., committing code in the container), you will need to enable SSH agent forwarding from your machine to the dev container. While VS Code handles this automatically, for other editors you will need to set this up manually.

diff --git a/chart/permissions-api/templates/config-server.yaml b/chart/permissions-api/templates/config-server.yaml
@@ -10,4 +10,4 @@ metadata:
     service: server
 data:
   config.yaml: |
-    {{- pick .Values.config "server" "oidc" "crdb" "spicedb" "tracing" "events" | toYaml | nindent 4 }}
+    {{- pick .Values.config "server" "oidc" "db" "psql" "crdb" "spicedb" "tracing" "events" | toYaml | nindent 4 }}
diff --git a/chart/permissions-api/templates/config-worker.yaml b/chart/permissions-api/templates/config-worker.yaml
@@ -10,4 +10,4 @@ metadata:
     service: worker
 data:
   config.yaml: |
-    {{- pick .Values.config "server" "events" "oidc" "crdb" "spicedb" "tracing" | toYaml | nindent 4 }}
+    {{- pick .Values.config "server" "events" "oidc" "db" "psql" "crdb" "spicedb" "tracing" | toYaml | nindent 4 }}
diff --git a/chart/permissions-api/templates/deployment-server.yaml b/chart/permissions-api/templates/deployment-server.yaml
@@ -44,7 +44,7 @@ spec:
       securityContext:
         {{- toYaml .Values.deployment.podSecurityContext | nindent 8 }}
       {{- end }}
-      {{- if eq .Values.config.crdb.migrateHook "init" }}
+      {{- if eq .Values.config.db.migrateHook "init" }}
       initContainers:
         - name: {{ include "common.names.name" . }}-migrate-database-init
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -54,6 +54,16 @@ spec:
             - up
             - --config
             - /config/config.yaml
+          {{- if eq .Values.config.db.engine "postgres"}}
+          {{- with .Values.config.psql.uriSecretName }}
+          env:
+            - name: PERMISSIONSAPI_PSQL_URI
+              valueFrom:
+                secretKeyRef:
+                  name: {{ . }}
+                  key: uri
+          {{- end }}
+          {{- else }}
           {{- with .Values.config.crdb.uriSecretName }}
           env:
             - name: PERMISSIONSAPI_CRDB_URI
@@ -62,6 +72,7 @@ spec:
                   name: {{ . }}
                   key: uri
           {{- end }}
+          {{- end }}
           {{- with .Values.deployment.resources }}
           resources:
             {{- toYaml . | nindent 12 }}

diff --git a/chart/permissions-api/templates/deployment-worker.yaml b/chart/permissions-api/templates/deployment-worker.yaml
@@ -55,6 +55,13 @@ spec:
           env:
             - name: PERMISSIONSAPI_SERVER_LISTEN
               value: ":{{ include "permapi.listenPort" . }}"
+          {{- with .Values.config.psql.uriSecretName }}
+            - name: PERMISSIONSAPI_PSQL_URI
+              valueFrom:
+                secretKeyRef:
+                  name: {{ . }}
+                  key: uri
+          {{- end }}
           {{- with .Values.config.crdb.uriSecretName }}
             - name: PERMISSIONSAPI_CRDB_URI
               valueFrom:

diff --git a/chart/permissions-api/templates/job-migrate-database.yaml b/chart/permissions-api/templates/job-migrate-database.yaml
@@ -1,9 +1,9 @@
-{{- if has .Values.config.crdb.migrateHook (list "pre-sync" "manual") }}
+{{- if has .Values.config.db.migrateHook (list "pre-sync" "manual") }}
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  {{- if eq .Values.config.crdb.migrateHook "manual" }}
+  {{- if eq .Values.config.db.migrateHook "manual" }}
   name: {{ include "common.names.name" . }}-migrate-database
   {{- else }}
   generateName: migrate-database-
@@ -41,6 +41,16 @@ spec:
             - up
             - --config
             - /config/config.yaml
+          {{- if eq .Values.config.db.engine "postgres"}}
+          {{- with .Values.config.psql.uriSecretName }}
+          env:
+            - name: PERMISSIONSAPI_PSQL_URI
+              valueFrom:
+                secretKeyRef:
+                  name: {{ . }}
+                  key: uri
+          {{- end }}
+          {{- else }}
           {{- with .Values.config.crdb.uriSecretName }}
           env:
             - name: PERMISSIONSAPI_CRDB_URI
@@ -49,6 +59,7 @@ spec:
                   name: {{ . }}
                   key: uri
           {{- end }}
+          {{- end }}
           {{- with .Values.deployment.resources }}
           resources:
             {{- toYaml . | nindent 12 }}

diff --git a/chart/permissions-api/values.yaml b/chart/permissions-api/values.yaml
@@ -45,12 +45,42 @@ config:
     pskSecretName: ""
     # policyConfigMapName is the name of the Config Map containing the policy file configuration
     policyConfigMapName: ""
-  crdb:
+  db:
+    # db engine to use for the permissions-api, cockroach or postgres, defaults to be cockroach
+    engine: "cockroach"
     # migrateHook sets when to run database migrations. one of: pre-sync, init, manual
     #  - pre-sync: hook runs as a job before any other changes are synced.
     #  - init: is run as an init container to the server deployment and may run multiple times if replica count is high.
     #  - manual: a migrate-database job will be available to triggered manually
     migrateHook: "init"
+  psql:
+    # name is the database name
+    name: ""
+    # host is the database host
+    host: ""
+    # user is the auth username to the database
+    user: ""
+    # password is the auth password to the database
+    password: ""
+    # params is the connection parameters to the database
+    params: ""
+    # uri is the raw uri connection string
+    uri: ""
+    # uriSecretName if set retrieves the `uri` from the provided secret name
+    uriSecretName: ""
+    # caSecretName if defined mounts database certificates from the provided secret
+    # secrets are mounted at `caMountPath`
+    caSecretName: ""
+    # caMountPath is the path the caSecretName is mounted at
+    caMountPath: /etc/ssl/crdb/
+    connections:
+      # max_open is the maximum number of open connections to the database
+      max_open: 0
+      # max_idle is the maximum number of connections in the idle connection
+      max_idle: 0
+      # max_lifetime is the maximum amount of time a connection may be idle
+      max_lifetime: 0 
+  crdb:
     # name is the database name
     name: ""
     # host is the database host