chore: Handle migrations with least permissions necessary

.github/workflows/release.yaml

@@ -49,7 +49,7 @@ jobs:
       - name: Build and Publish API
         run: |
           docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push packages/api
-          docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ steps.get_version.outputs.version-without-v }} --push packages/api/supabase
+          docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" .
 
           zarf package create packages/api --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
           zarf package create packages/api --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
@@ -63,7 +63,7 @@ jobs:
       - name: Build and Publish UI
         run: |
           docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${{ steps.get_version.outputs.version-without-v }} --push src/leapfrogai_ui
-          docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${{ steps.get_version.outputs.version-without-v }} --push src/leapfrogai_ui/supabase
+          docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" .
 
           zarf package create packages/ui --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
           zarf package create packages/ui --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
@@ -76,6 +76,8 @@ jobs:
 
       - name: Build and Publish Supabase
         run: |
+          docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" .
+
           zarf package create packages/supabase --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm
           zarf package create packages/supabase --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm
 

packages/api/supabase/Dockerfile → Dockerfile.migrations

@@ -1,6 +1,9 @@
 FROM --platform=$TARGETPLATFORM cgr.dev/chainguard/bash:latest
 ARG TARGETPLATFORM
 ARG version="1.169.8"
+ARG MIGRATIONS_DIR
+
+RUN test -n "$MIGRATIONS_DIR" || (echo "MIGRATIONS_PATH is required" && false)
 
 # Download the supabase cli
 RUN mkdir -p /usr/local/bin
@@ -13,4 +16,4 @@ RUN ARCH=$(echo $TARGETPLATFORM | cut -d '/' -f2) \
 USER 65532:65532
 
 # Download the migration scripts
-COPY --chown=65532:65532 migrations/*.sql supabase/migrations/
+COPY --chown=65532:65532 ${MIGRATIONS_DIR}/*.sql supabase/migrations/

Makefile

@@ -37,9 +37,14 @@ sdk-wheel: ## build wheels for the leapfrogai_sdk package as a dependency for ot
 	-rm ${SDK_DEST}/*.whl
 	python -m pip wheel src/leapfrogai_sdk -w ${SDK_DEST}
 
-build-supabase:
+build-supabase: local-registry
+	## Build the migration container for this version of the supabase package
+	docker build -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" .
+	docker tag ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION}
+	docker push localhost:5000/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION}
+
 	## Build the Zarf package
-	uds zarf package create packages/supabase -o packages/supabase --set IMAGE_VERSION=${LOCAL_VERSION} --confirm
+	uds zarf package create packages/supabase -o packages/supabase --registry-override=ghcr.io=localhost:5000 --set IMAGE_VERSION=${LOCAL_VERSION} --confirm
 
 setup-api-deps: sdk-wheel ## Download the wheels for the leapfrogai_api dependencies
 	-rm packages/api/build/*.whl
@@ -51,7 +56,7 @@ build-api: local-registry setup-api-deps ## Build the leapfrogai_api container a
 	docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION}
 
 	## Build the migration container for this version of the API
-	docker build -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} packages/api/supabase
+	docker build -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" .
 	docker tag ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION}
 
 	## Push the images to the local registry (Zarf is super slow if the image is only in the local daemon)
@@ -68,7 +73,7 @@ build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package
 	docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION}
 
 	## Build the migration container for the version of the UI
-	docker build -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} src/leapfrogai_ui/supabase
+	docker build -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" .
 	docker tag ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION}
 
 	## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon)

packages/api/chart/templates/migration-job.yaml

@@ -26,8 +26,8 @@ spec:
         args:
           - -c
           - >-
-            supabase migration fetch --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --debug || true &&
-            supabase db push --db-url="postgresql://supabase_admin:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --include-all --debug
+            supabase migration fetch --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --debug || true &&
+            supabase db push --db-url="postgresql://postgres:$POSTGRES_PASSWORD@$MIGRATION_SERVICE_NAME.$MIGRATION_NAMESPACE.svc.cluster.local:$MIGRATION_SERVICE_PORT/postgres" --include-all --debug
         securityContext:
           runAsUser: {{ .Values.image.securityContext.runAsUser }}
           runAsGroup: {{ .Values.image.securityContext.runAsGroup }}

packages/api/supabase/migrations/20240419164109_v0.8.0_openai_types.sql

@@ -0,0 +1,187 @@
+-- Create a table to store OpenAI Assistant Objects
+create table
+  assistant_objects (
+    id uuid primary key DEFAULT uuid_generate_v4(),
+    created_at bigint default extract(epoch from now()) not null,
+    description varchar(512),
+    instructions text,
+    metadata jsonb,
+    model varchar(255) not null,
+    name varchar(255),
+    object text check (object in ('assistant')),
+    tools jsonb,
+    response_format jsonb,
+    temperature float,
+    tool_resources jsonb,
+    top_p float
+  );
+
+-- Create a table to store the OpenAI Thread Objects
+create table
+  thread_objects (
+    id uuid primary key DEFAULT uuid_generate_v4(),
+    user_id uuid references auth.users not null,
+    object text check (object in ('thread')),
+    created_at bigint default extract(epoch FROM NOW()) NOT NULL,
+    tool_resources jsonb,
+    metadata jsonb
+  );
+
+-- Create a table to store the OpenAI Message Objects
+create table
+  message_objects (
+    id uuid primary key DEFAULT uuid_generate_v4(),
+    user_id uuid references auth.users not null,
+    object text check (object in ('thread.message')),
+    created_at bigint default extract(epoch FROM NOW()) not null,
+    thread_id uuid references thread_objects (id) on delete cascade not null,
+    status text,
+    incomplete_details jsonb,
+    completed_at bigint,
+    incomplete_at bigint,
+    role text,
+    content jsonb,
+    assistant_id uuid, -- No foreign key constraint, can be null and doesn't have to refer to an assistant that exists
+    run_id uuid, -- No foreign key constraint, can be null and doesn't have to refer to a thread that exists
+    attachments jsonb,
+    metadata jsonb
+  );
+
+-- RLS policies
+alter table thread_objects enable row level security;
+alter table message_objects enable row level security;
+
+-- Policies for thread_objects
+create policy "Individuals can view their own thread_objects." on thread_objects for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create thread_objects." on thread_objects for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own thread_objects." on thread_objects for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own thread_objects." on thread_objects for
+    delete using (auth.uid() = user_id);
+
+-- Policies for message_objects
+create policy "Individuals can view their own message_objects." on message_objects for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create message_objects." on message_objects for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own message_objects." on message_objects for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own message_objects." on message_objects for
+    delete using (auth.uid() = user_id);
+
+-- Indexes for foreign keys for message_objects
+CREATE INDEX message_objects_user_id ON message_objects (user_id);
+CREATE INDEX message_objects_thread_id ON message_objects (thread_id);
+CREATE INDEX message_objects_created_at ON thread_objects (created_at);
+
+-- Indexes for common filtering and sorting for thread_objects
+CREATE INDEX thread_objects_id ON thread_objects (id);
+CREATE INDEX thread_objects_user_id ON thread_objects (user_id);
+CREATE INDEX thread_objects_created_at ON thread_objects (created_at);
+
+-- Add user_id column to assistant_objects and file_objects tables
+alter table assistant_objects
+    add column user_id uuid references auth.users not null;
+alter table file_objects
+        add column user_id uuid references auth.users not null;
+-- Set buckets to private
+update storage.buckets
+    set public = false
+    where id = 'file_bucket';
+
+-- RLS policies
+alter table assistant_objects enable row level security;
+alter table file_objects enable row level security;
+
+-- Policies for assistant_objects
+create policy "Individuals can view their own assistant_objects. " on assistant_objects for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create assistant_objects." on assistant_objects for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own assistant_objects." on assistant_objects for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own assistant_objects." on assistant_objects for
+    delete using (auth.uid() = user_id);
+
+-- Policies for file_objects
+create policy "Individuals can view their own file_objects." on file_objects for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create file_objects." on file_objects for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own file_objects." on file_objects for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own file_objects." on file_objects for
+    delete using (auth.uid() = user_id);
+
+-- Policies for file_bucket
+create policy "Any authenticated individual can add files to file_bucket."
+on storage.objects for
+    insert to authenticated with check (bucket_id = 'file_bucket');
+create policy "Individuals can view their own files in the file_bucket."
+on storage.objects for
+    select using (bucket_id = 'file_bucket' AND auth.uid() = owner);
+create policy "Individuals can delete their own files."
+on storage.objects for
+    delete using (bucket_id = 'file_bucket' AND auth.uid() = owner);
+create policy "Individuals can update their own files in file_bucket."
+on storage.objects for
+    update using (auth.uid() = owner) with check (bucket_id = 'file_bucket');
+
+ALTER TABLE file_objects ALTER COLUMN created_at SET DEFAULT extract(epoch from now());
+ALTER TABLE file_objects ALTER COLUMN created_at SET NOT NULL;
+ALTER TABLE file_objects ADD CHECK (object in ('file'));
+
+-- Create a table to store the OpenAI Run Objects
+create table
+  run_objects (
+    id uuid primary key default uuid_generate_v4(),
+    user_id uuid references auth.users not null,
+    object text check (object in ('thread.run')),
+    created_at bigint default extract(epoch FROM NOW()) not null,
+    thread_id uuid references thread_objects (id) on delete cascade not null,
+    assistant_id uuid references assistant_objects (id) on delete cascade not null,
+    status text,
+    required_action jsonb,
+    last_error jsonb,
+    expires_at bigint,
+    started_at bigint,
+    cancelled_at bigint,
+    failed_at bigint,
+    completed_at bigint,
+    model text,
+    instructions text,
+    tools jsonb,
+    metadata jsonb,
+    parallel_tool_calls boolean,
+    stream boolean,
+    file_ids uuid[],
+    incomplete_details jsonb,
+    usage jsonb,
+    temperature float,
+    top_p float,
+    max_prompt_tokens int,
+    max_completion_tokens int,
+    truncation_strategy jsonb,
+    tool_choice jsonb,
+    response_format jsonb
+  );
+
+-- RLS policies
+alter table run_objects enable row level security;
+
+-- Policies for run_objects
+create policy "Individuals can view their own run_objects." on run_objects for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create run_objects." on run_objects for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own run_objects." on run_objects for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own run_objects." on run_objects for
+    delete using (auth.uid() = user_id);
+
+-- Indexes for common filtering and sorting for run_objects
+CREATE INDEX run_objects_id ON run_objects (id);
+CREATE INDEX run_objects_user_id ON run_objects (user_id);
+CREATE INDEX run_objects_created_at ON run_objects (created_at);

packages/api/supabase/migrations/20240502193159_init-vector-stores.sql → packages/api/supabase/migrations/20240502193159_v0.8.0_vector_stores.sql

@@ -130,3 +130,38 @@ begin
   limit match_limit;
 end;
 $$;
+
+-- RLS policies
+alter table vector_store enable row level security;
+alter table vector_store_file enable row level security;
+alter table vector_content enable row level security;
+
+-- Policies for vector_store
+create policy "Individuals can view their own vector_store." on vector_store for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create vector_store." on vector_store for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own vector_store." on vector_store for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own vector_store." on vector_store for
+    delete using (auth.uid() = user_id);
+
+-- Policies for vector_store_file
+create policy "Individuals can view their own vector_store_file." on vector_store_file for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create vector_store_file." on vector_store_file for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own vector_store_file." on vector_store_file for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own vector_store_file." on vector_store_file for
+    delete using (auth.uid() = user_id);
+
+-- Policies for vector_content
+create policy "Individuals can view their own vector_content." on vector_content for
+    select using (auth.uid() = user_id);
+create policy "Individuals can create vector_content." on vector_content for
+    insert with check (auth.uid() = user_id);
+create policy "Individuals can update their own vector_content." on vector_content for
+    update using (auth.uid() = user_id);
+create policy "Individuals can delete their own vector_content." on vector_content for
+    delete using (auth.uid() = user_id);