From 93ec9e264a1dbdff61233289418612f558508135 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" Date: Mon, 4 Mar 2024 02:50:58 -0800 Subject: [PATCH] add new migrations system (#10312) --- .changeset/good-maps-deny.md | 5 + .../db/src/core/cli/commands/gen/index.ts | 56 ----- .../db/src/core/cli/commands/push/index.ts | 175 +++----------- .../db/src/core/cli/commands/verify/index.ts | 58 ++--- packages/db/src/core/cli/index.ts | 13 +- packages/db/src/core/cli/migration-queries.ts | 217 +++++------------- packages/db/src/core/cli/migrations.ts | 151 ------------ packages/db/src/core/errors.ts | 20 +- packages/db/src/core/integration/typegen.ts | 7 +- packages/db/src/core/types.ts | 5 +- packages/db/src/core/utils.ts | 4 - packages/db/src/runtime/queries.ts | 4 + packages/db/test/fixtures/basics/db/config.ts | 1 + packages/db/test/fixtures/basics/db/theme.ts | 2 +- packages/db/test/unit/column-queries.test.js | 73 +++--- 15 files changed, 188 insertions(+), 603 deletions(-) create mode 100644 .changeset/good-maps-deny.md delete mode 100644 packages/db/src/core/cli/commands/gen/index.ts delete mode 100644 packages/db/src/core/cli/migrations.ts diff --git a/.changeset/good-maps-deny.md b/.changeset/good-maps-deny.md new file mode 100644 index 000000000000..5d67d835bb08 --- /dev/null +++ b/.changeset/good-maps-deny.md @@ -0,0 +1,5 @@ +--- +"@astrojs/db": minor +--- + +Revamp migrations system diff --git a/packages/db/src/core/cli/commands/gen/index.ts b/packages/db/src/core/cli/commands/gen/index.ts deleted file mode 100644 index be157d4a5545..000000000000 --- a/packages/db/src/core/cli/commands/gen/index.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { writeFile } from 'node:fs/promises'; -import { relative } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import type { AstroConfig } from 'astro'; -import { bgRed, bold, red, reset } from 'kleur/colors'; -import type { Arguments } from 'yargs-parser'; -import type { DBConfig } from '../../../types.js'; -import { getMigrationsDirectoryUrl } from '../../../utils.js'; -import { getMigrationQueries } from '../../migration-queries.js'; -import { - MIGRATIONS_CREATED, - MIGRATIONS_UP_TO_DATE, - getMigrationStatus, - initializeMigrationsDirectory, -} from '../../migrations.js'; - -export async function cmd({ - astroConfig, - dbConfig, -}: { - astroConfig: AstroConfig; - dbConfig: DBConfig; - flags: Arguments; -}) { - const migration = await getMigrationStatus({ dbConfig, root: astroConfig.root }); - const migrationsDir = getMigrationsDirectoryUrl(astroConfig.root); - - if (migration.state === 'no-migrations-found') { - await initializeMigrationsDirectory(migration.currentSnapshot, migrationsDir); - console.log(MIGRATIONS_CREATED); - return; - } else if (migration.state === 'up-to-date') { - console.log(MIGRATIONS_UP_TO_DATE); - return; - } - - const { oldSnapshot, newSnapshot, newFilename, diff } = migration; - const { queries: migrationQueries, confirmations } = await getMigrationQueries({ - oldSnapshot, - newSnapshot, - }); - // Warn the user about any changes that lead to data-loss. - // When the user runs `db push`, they will be prompted to confirm these changes. - confirmations.map((message) => console.log(bgRed(' !!! ') + ' ' + red(message))); - const content = { - diff, - db: migrationQueries, - // TODO(fks): Encode the relevant data, instead of the raw message. - // This will give `db push` more control over the formatting of the message. - confirm: confirmations.map((c) => reset(c)), - }; - const fileUrl = new URL(newFilename, migrationsDir); - const relativePath = relative(fileURLToPath(astroConfig.root), fileURLToPath(fileUrl)); - await writeFile(fileUrl, JSON.stringify(content, undefined, 2)); - console.log(bold(relativePath) + ' created!'); -} diff --git a/packages/db/src/core/cli/commands/push/index.ts b/packages/db/src/core/cli/commands/push/index.ts index 81d232cb2904..2dd183458025 100644 --- a/packages/db/src/core/cli/commands/push/index.ts +++ b/packages/db/src/core/cli/commands/push/index.ts @@ -1,26 +1,16 @@ import type { AstroConfig } from 'astro'; -import { red } from 'kleur/colors'; -import prompts from 'prompts'; import type { Arguments } from 'yargs-parser'; -import { MISSING_SESSION_ID_ERROR } from '../../../errors.js'; import { getManagedAppTokenOrExit } from '../../../tokens.js'; import { type DBConfig, type DBSnapshot } from '../../../types.js'; -import { getMigrationsDirectoryUrl, getRemoteDatabaseUrl } from '../../../utils.js'; -import { getMigrationQueries } from '../../migration-queries.js'; +import { getRemoteDatabaseUrl } from '../../../utils.js'; import { - INITIAL_SNAPSHOT, - MIGRATIONS_NOT_INITIALIZED, - MIGRATIONS_UP_TO_DATE, - MIGRATION_NEEDED, + getMigrationQueries, + createCurrentSnapshot, createEmptySnapshot, - getMigrationStatus, - getMigrations, - loadInitialSnapshot, - loadMigration, -} from '../../migrations.js'; + getProductionCurrentSnapshot, +} from '../../migration-queries.js'; export async function cmd({ - astroConfig, dbConfig, flags, }: { @@ -29,49 +19,31 @@ export async function cmd({ flags: Arguments; }) { const isDryRun = flags.dryRun; + const isForceReset = flags.forceReset; const appToken = await getManagedAppTokenOrExit(flags.token); - const migration = await getMigrationStatus({ dbConfig, root: astroConfig.root }); - if (migration.state === 'no-migrations-found') { - console.log(MIGRATIONS_NOT_INITIALIZED); - process.exit(1); - } else if (migration.state === 'ahead') { - console.log(MIGRATION_NEEDED); - process.exit(1); - } - const migrationsDir = getMigrationsDirectoryUrl(astroConfig.root); + const productionSnapshot = await getProductionCurrentSnapshot({ appToken: appToken.token }); + const currentSnapshot = createCurrentSnapshot(dbConfig); + const isFromScratch = isForceReset || JSON.stringify(productionSnapshot) === '{}'; + const { queries: migrationQueries } = await getMigrationQueries({ + oldSnapshot: isFromScratch ? createEmptySnapshot() : productionSnapshot, + newSnapshot: currentSnapshot, + }); - // get all migrations from the filesystem - const allLocalMigrations = await getMigrations(migrationsDir); - let missingMigrations: string[] = []; - try { - const { data } = await prepareMigrateQuery({ - migrations: allLocalMigrations, - appToken: appToken.token, - }); - missingMigrations = data; - } catch (error) { - if (error instanceof Error) { - if (error.message.startsWith('{')) { - const { error: { code } = { code: '' } } = JSON.parse(error.message); - if (code === 'TOKEN_UNAUTHORIZED') { - console.error(MISSING_SESSION_ID_ERROR); - } - } - } - console.error(error); - process.exit(1); + // // push the database schema + if (migrationQueries.length === 0) { + console.log('Database schema is up to date.'); + } else { + console.log(`Database schema is out of date.`); } - // push the database schema - if (missingMigrations.length === 0) { - console.log(MIGRATIONS_UP_TO_DATE); + if (isDryRun) { + console.log('Statements:', JSON.stringify(migrationQueries, undefined, 2)); } else { - console.log(`Pushing ${missingMigrations.length} migrations...`); + console.log(`Pushing database schema updates...`); await pushSchema({ - migrations: missingMigrations, - migrationsDir, + statements: migrationQueries, appToken: appToken.token, isDryRun, - currentSnapshot: migration.currentSnapshot, + currentSnapshot: currentSnapshot, }); } // cleanup and exit @@ -80,92 +52,26 @@ export async function cmd({ } async function pushSchema({ - migrations, - migrationsDir, + statements, appToken, isDryRun, currentSnapshot, }: { - migrations: string[]; - migrationsDir: URL; + statements: string[]; appToken: string; isDryRun: boolean; currentSnapshot: DBSnapshot; }) { - // load all missing migrations - const initialSnapshot = migrations.find((m) => m === INITIAL_SNAPSHOT); - const filteredMigrations = migrations.filter((m) => m !== INITIAL_SNAPSHOT); - const missingMigrationContents = await Promise.all( - filteredMigrations.map((m) => loadMigration(m, migrationsDir)) - ); - // create a migration for the initial snapshot, if needed - const initialMigrationBatch = initialSnapshot - ? ( - await getMigrationQueries({ - oldSnapshot: createEmptySnapshot(), - newSnapshot: await loadInitialSnapshot(migrationsDir), - }) - ).queries - : []; - - // combine all missing migrations into a single batch - const confirmations = missingMigrationContents.reduce((acc, curr) => { - return [...acc, ...(curr.confirm || [])]; - }, [] as string[]); - if (confirmations.length > 0) { - const response = await prompts([ - ...confirmations.map((message, index) => ({ - type: 'confirm' as const, - name: String(index), - message: red('Warning: ') + message + '\nContinue?', - initial: true, - })), - ]); - if ( - Object.values(response).length === 0 || - Object.values(response).some((value) => value === false) - ) { - process.exit(1); - } - } - - // combine all missing migrations into a single batch - const queries = missingMigrationContents.reduce((acc, curr) => { - return [...acc, ...curr.db]; - }, initialMigrationBatch); - // apply the batch to the DB - await runMigrateQuery({ queries, migrations, snapshot: currentSnapshot, appToken, isDryRun }); -} - -async function runMigrateQuery({ - queries: baseQueries, - migrations, - snapshot, - appToken, - isDryRun, -}: { - queries: string[]; - migrations: string[]; - snapshot: DBSnapshot; - appToken: string; - isDryRun?: boolean; -}) { - const queries = ['pragma defer_foreign_keys=true;', ...baseQueries]; - const requestBody = { - snapshot, - migrations, - sql: queries, + snapshot: currentSnapshot, + sql: statements, experimentalVersion: 1, }; - if (isDryRun) { console.info('[DRY RUN] Batch query:', JSON.stringify(requestBody, null, 2)); return new Response(null, { status: 200 }); } - - const url = new URL('/migrations/run', getRemoteDatabaseUrl()); - + const url = new URL('/db/push', getRemoteDatabaseUrl()); return await fetch(url, { method: 'POST', headers: new Headers({ @@ -174,28 +80,3 @@ async function runMigrateQuery({ body: JSON.stringify(requestBody), }); } - -async function prepareMigrateQuery({ - migrations, - appToken, -}: { - migrations: string[]; - appToken: string; -}) { - const url = new URL('/migrations/prepare', getRemoteDatabaseUrl()); - const requestBody = { - migrations, - experimentalVersion: 1, - }; - const result = await fetch(url, { - method: 'POST', - headers: new Headers({ - Authorization: `Bearer ${appToken}`, - }), - body: JSON.stringify(requestBody), - }); - if (result.status >= 400) { - throw new Error(await result.text()); - } - return await result.json(); -} diff --git a/packages/db/src/core/cli/commands/verify/index.ts b/packages/db/src/core/cli/commands/verify/index.ts index 3b95835f7a64..1e486a860964 100644 --- a/packages/db/src/core/cli/commands/verify/index.ts +++ b/packages/db/src/core/cli/commands/verify/index.ts @@ -1,16 +1,13 @@ import type { AstroConfig } from 'astro'; import type { Arguments } from 'yargs-parser'; import type { DBConfig } from '../../../types.js'; -import { getMigrationQueries } from '../../migration-queries.js'; -import { - MIGRATIONS_NOT_INITIALIZED, - MIGRATIONS_UP_TO_DATE, - MIGRATION_NEEDED, - getMigrationStatus, -} from '../../migrations.js'; +import { getMigrationQueries, + createCurrentSnapshot, + createEmptySnapshot, + getProductionCurrentSnapshot, } from '../../migration-queries.js'; +import { getManagedAppTokenOrExit } from '../../../tokens.js'; export async function cmd({ - astroConfig, dbConfig, flags, }: { @@ -18,35 +15,20 @@ export async function cmd({ dbConfig: DBConfig; flags: Arguments; }) { - const status = await getMigrationStatus({ dbConfig, root: astroConfig.root }); - const { state } = status; - if (flags.json) { - if (state === 'ahead') { - const { queries: migrationQueries } = await getMigrationQueries({ - oldSnapshot: status.oldSnapshot, - newSnapshot: status.newSnapshot, - }); - const newFileContent = { - diff: status.diff, - db: migrationQueries, - }; - status.newFileContent = JSON.stringify(newFileContent, null, 2); - } - console.log(JSON.stringify(status)); - process.exit(state === 'up-to-date' ? 0 : 1); - } - switch (state) { - case 'no-migrations-found': { - console.log(MIGRATIONS_NOT_INITIALIZED); - process.exit(1); - } - case 'ahead': { - console.log(MIGRATION_NEEDED); - process.exit(1); - } - case 'up-to-date': { - console.log(MIGRATIONS_UP_TO_DATE); - return; - } + const appToken = await getManagedAppTokenOrExit(flags.token); + const productionSnapshot = await getProductionCurrentSnapshot({ appToken: appToken.token }); + const currentSnapshot = createCurrentSnapshot(dbConfig); + const { queries: migrationQueries } = await getMigrationQueries({ + oldSnapshot: JSON.stringify(productionSnapshot) !== '{}' ? productionSnapshot : createEmptySnapshot(), + newSnapshot: currentSnapshot, + }); + + if (migrationQueries.length === 0) { + console.log(`Database schema is up to date.`); + } else { + console.log(`Database schema is out of date.`); + console.log(`Run 'astro db push' to push up your latest changes.`); } + + await appToken.destroy(); } diff --git a/packages/db/src/core/cli/index.ts b/packages/db/src/core/cli/index.ts index 9ffaee7b3059..85dbdd38ae4f 100644 --- a/packages/db/src/core/cli/index.ts +++ b/packages/db/src/core/cli/index.ts @@ -23,10 +23,13 @@ export async function cli({ const { cmd } = await import('./commands/shell/index.js'); return await cmd({ astroConfig, dbConfig, flags }); } - case 'gen': + case 'gen': { + console.log('"astro db gen" is no longer needed! Visit the docs for more information.'); + return; + } case 'sync': { - const { cmd } = await import('./commands/gen/index.js'); - return await cmd({ astroConfig, dbConfig, flags }); + console.log('"astro db sync" is no longer needed! Visit the docs for more information.'); + return; } case 'push': { const { cmd } = await import('./commands/push/index.js'); @@ -76,7 +79,7 @@ astro logout End your authenticated session with Astro Studio astro link Link this directory to an Astro Studio project astro db gen Creates snapshot based on your schema -astro db push Pushes migrations to Astro Studio -astro db verify Verifies migrations have been pushed and errors if not`; +astro db push Pushes schema updates to Astro Studio +astro db verify Tests schema updates /w Astro Studio (good for CI)`; } } diff --git a/packages/db/src/core/cli/migration-queries.ts b/packages/db/src/core/cli/migration-queries.ts index 4265f36e424a..e803c622590f 100644 --- a/packages/db/src/core/cli/migration-queries.ts +++ b/packages/db/src/core/cli/migration-queries.ts @@ -2,11 +2,11 @@ import deepDiff from 'deep-diff'; import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core'; import * as color from 'kleur/colors'; import { customAlphabet } from 'nanoid'; -import prompts from 'prompts'; import { hasPrimaryKey } from '../../runtime/index.js'; import { getCreateIndexQueries, getCreateTableQuery, + getDropTableIfExistsQuery, getModifiers, getReferencesConfig, hasDefault, @@ -18,6 +18,7 @@ import { type ColumnType, type DBColumn, type DBColumns, + type DBConfig, type DBSnapshot, type DBTable, type DBTables, @@ -28,49 +29,39 @@ import { type TextColumn, columnSchema, } from '../types.js'; +import { getRemoteDatabaseUrl } from '../utils.js'; +import { RENAME_COLUMN_ERROR, RENAME_TABLE_ERROR } from '../errors.js'; const sqlite = new SQLiteAsyncDialect(); const genTempTableName = customAlphabet('abcdefghijklmnopqrstuvwxyz', 10); -/** Dependency injected for unit testing */ -type AmbiguityResponses = { - collectionRenames: Record; - columnRenames: { - [collectionName: string]: Record; - }; -}; - export async function getMigrationQueries({ oldSnapshot, newSnapshot, - ambiguityResponses, }: { oldSnapshot: DBSnapshot; newSnapshot: DBSnapshot; - ambiguityResponses?: AmbiguityResponses; }): Promise<{ queries: string[]; confirmations: string[] }> { const queries: string[] = []; const confirmations: string[] = []; - let added = getAddedCollections(oldSnapshot, newSnapshot); - let dropped = getDroppedCollections(oldSnapshot, newSnapshot); - if (!isEmpty(added) && !isEmpty(dropped)) { - const resolved = await resolveCollectionRenames(added, dropped, ambiguityResponses); - added = resolved.added; - dropped = resolved.dropped; - for (const { from, to } of resolved.renamed) { - const renameQuery = `ALTER TABLE ${sqlite.escapeName(from)} RENAME TO ${sqlite.escapeName( - to - )}`; - queries.push(renameQuery); - } + const addedCollections = getAddedCollections(oldSnapshot, newSnapshot); + const droppedTables = getDroppedCollections(oldSnapshot, newSnapshot); + const notDeprecatedDroppedTables = Object.fromEntries( + Object.entries(droppedTables).filter(([, table]) => !table.deprecated) + ); + if (!isEmpty(addedCollections) && !isEmpty(notDeprecatedDroppedTables)) { + throw new Error( + RENAME_TABLE_ERROR(Object.keys(addedCollections)[0], Object.keys(notDeprecatedDroppedTables)[0]) + ); } - for (const [collectionName, collection] of Object.entries(added)) { + for (const [collectionName, collection] of Object.entries(addedCollections)) { + queries.push(getDropTableIfExistsQuery(collectionName)); queries.push(getCreateTableQuery(collectionName, collection)); queries.push(...getCreateIndexQueries(collectionName, collection)); } - for (const [collectionName] of Object.entries(dropped)) { + for (const [collectionName] of Object.entries(droppedTables)) { const dropQuery = `DROP TABLE ${sqlite.escapeName(collectionName)}`; queries.push(dropQuery); } @@ -78,6 +69,19 @@ export async function getMigrationQueries({ for (const [collectionName, newCollection] of Object.entries(newSnapshot.schema)) { const oldCollection = oldSnapshot.schema[collectionName]; if (!oldCollection) continue; + const addedColumns = getAdded(oldCollection.columns, newCollection.columns); + const droppedColumns = getDropped(oldCollection.columns, newCollection.columns); + const notDeprecatedDroppedColumns = Object.fromEntries( + Object.entries(droppedColumns).filter(([key, col]) => !col.schema.deprecated) + ); + if (!isEmpty(addedColumns) && !isEmpty(notDeprecatedDroppedColumns)) { + throw new Error( + RENAME_COLUMN_ERROR( + `${collectionName}.${Object.keys(addedColumns)[0]}`, + `${collectionName}.${Object.keys(notDeprecatedDroppedColumns)[0]}` + ) + ); + } const result = await getCollectionChangeQueries({ collectionName, oldCollection, @@ -93,18 +97,16 @@ export async function getCollectionChangeQueries({ collectionName, oldCollection, newCollection, - ambiguityResponses, }: { collectionName: string; oldCollection: DBTable; newCollection: DBTable; - ambiguityResponses?: AmbiguityResponses; }): Promise<{ queries: string[]; confirmations: string[] }> { const queries: string[] = []; const confirmations: string[] = []; const updated = getUpdatedColumns(oldCollection.columns, newCollection.columns); - let added = getAdded(oldCollection.columns, newCollection.columns); - let dropped = getDropped(oldCollection.columns, newCollection.columns); + const added = getAdded(oldCollection.columns, newCollection.columns); + const dropped = getDropped(oldCollection.columns, newCollection.columns); /** Any foreign key changes require a full table recreate */ const hasForeignKeyChanges = Boolean( deepDiff(oldCollection.foreignKeys, newCollection.foreignKeys) @@ -120,12 +122,7 @@ export async function getCollectionChangeQueries({ confirmations, }; } - if (!hasForeignKeyChanges && !isEmpty(added) && !isEmpty(dropped)) { - const resolved = await resolveColumnRenames(collectionName, added, dropped, ambiguityResponses); - added = resolved.added; - dropped = resolved.dropped; - queries.push(...getColumnRenameQueries(collectionName, resolved.renamed)); - } + if ( !hasForeignKeyChanges && isEmpty(updated) && @@ -207,116 +204,6 @@ function getChangeIndexQueries({ return queries; } -type Renamed = Array<{ from: string; to: string }>; - -async function resolveColumnRenames( - collectionName: string, - mightAdd: DBColumns, - mightDrop: DBColumns, - ambiguityResponses?: AmbiguityResponses -): Promise<{ added: DBColumns; dropped: DBColumns; renamed: Renamed }> { - const added: DBColumns = {}; - const dropped: DBColumns = {}; - const renamed: Renamed = []; - - for (const [columnName, column] of Object.entries(mightAdd)) { - let oldColumnName = ambiguityResponses - ? ambiguityResponses.columnRenames[collectionName]?.[columnName] ?? '__NEW__' - : undefined; - if (!oldColumnName) { - const res = await prompts( - { - type: 'select', - name: 'columnName', - message: - 'New column ' + - color.blue(color.bold(`${collectionName}.${columnName}`)) + - ' detected. Was this renamed from an existing column?', - choices: [ - { title: 'New column (not renamed from existing)', value: '__NEW__' }, - ...Object.keys(mightDrop) - .filter((key) => !(key in renamed)) - .map((key) => ({ title: key, value: key })), - ], - }, - { - onCancel: () => { - process.exit(1); - }, - } - ); - oldColumnName = res.columnName as string; - } - - if (oldColumnName === '__NEW__') { - added[columnName] = column; - } else { - renamed.push({ from: oldColumnName, to: columnName }); - } - } - for (const [droppedColumnName, droppedColumn] of Object.entries(mightDrop)) { - if (!renamed.find((r) => r.from === droppedColumnName)) { - dropped[droppedColumnName] = droppedColumn; - } - } - - return { added, dropped, renamed }; -} - -async function resolveCollectionRenames( - mightAdd: DBTables, - mightDrop: DBTables, - ambiguityResponses?: AmbiguityResponses -): Promise<{ added: DBTables; dropped: DBTables; renamed: Renamed }> { - const added: DBTables = {}; - const dropped: DBTables = {}; - const renamed: Renamed = []; - - for (const [collectionName, collection] of Object.entries(mightAdd)) { - let oldCollectionName = ambiguityResponses - ? ambiguityResponses.collectionRenames[collectionName] ?? '__NEW__' - : undefined; - if (!oldCollectionName) { - const res = await prompts( - { - type: 'select', - name: 'collectionName', - message: - 'New collection ' + - color.blue(color.bold(collectionName)) + - ' detected. Was this renamed from an existing collection?', - choices: [ - { title: 'New collection (not renamed from existing)', value: '__NEW__' }, - ...Object.keys(mightDrop) - .filter((key) => !(key in renamed)) - .map((key) => ({ title: key, value: key })), - ], - }, - { - onCancel: () => { - process.exit(1); - }, - } - ); - oldCollectionName = res.collectionName as string; - } - - if (oldCollectionName === '__NEW__') { - added[collectionName] = collection; - } else { - renamed.push({ from: oldCollectionName, to: collectionName }); - } - } - - for (const [droppedCollectionName, droppedCollection] of Object.entries(mightDrop)) { - if (!renamed.find((r) => r.from === droppedCollectionName)) { - dropped[droppedCollectionName] = droppedCollection; - } - } - - return { added, dropped, renamed }; -} - function getAddedCollections(oldCollections: DBSnapshot, newCollections: DBSnapshot): DBTables { const added: DBTables = {}; for (const [key, newCollection] of Object.entries(newCollections.schema)) { @@ -333,20 +220,6 @@ function getDroppedCollections(oldCollections: DBSnapshot, newCollections: DBSna return dropped; } -function getColumnRenameQueries(unescapedCollectionName: string, renamed: Renamed): string[] { - const queries: string[] = []; - const collectionName = sqlite.escapeName(unescapedCollectionName); - - for (const { from, to } of renamed) { - const q = `ALTER TABLE ${collectionName} RENAME COLUMN ${sqlite.escapeName( - from - )} TO ${sqlite.escapeName(to)}`; - queries.push(q); - } - - return queries; -} - /** * Get ALTER TABLE queries to update the table schema. Assumes all added and dropped columns pass * `canUseAlterTableAddColumn` and `canAlterTableDropColumn` checks! @@ -552,3 +425,29 @@ type DBColumnWithDefault = function hasRuntimeDefault(column: DBColumn): column is DBColumnWithDefault { return !!(column.schema.default && isSerializedSQL(column.schema.default)); } + +export async function getProductionCurrentSnapshot({ + appToken, +}: { + appToken: string; +}): Promise { + const url = new URL('/db/schema', getRemoteDatabaseUrl()); + + const response = await fetch(url, { + method: 'POST', + headers: new Headers({ + Authorization: `Bearer ${appToken}`, + }), + }); + const result = await response.json(); + return result.data; +} + +export function createCurrentSnapshot({ tables = {} }: DBConfig): DBSnapshot { + const schema = JSON.parse(JSON.stringify(tables)); + return { experimentalVersion: 1, schema }; +} + +export function createEmptySnapshot(): DBSnapshot { + return { experimentalVersion: 1, schema: {} }; +} diff --git a/packages/db/src/core/cli/migrations.ts b/packages/db/src/core/cli/migrations.ts deleted file mode 100644 index 514d4e798a11..000000000000 --- a/packages/db/src/core/cli/migrations.ts +++ /dev/null @@ -1,151 +0,0 @@ -import deepDiff from 'deep-diff'; -import { mkdir, readFile, readdir, writeFile } from 'fs/promises'; -import { cyan, green, yellow } from 'kleur/colors'; -import { type DBConfig, type DBSnapshot } from '../types.js'; -import { getMigrationsDirectoryUrl } from '../utils.js'; -const { applyChange, diff: generateDiff } = deepDiff; - -export type MigrationStatus = - | { - state: 'no-migrations-found'; - currentSnapshot: DBSnapshot; - } - | { - state: 'ahead'; - oldSnapshot: DBSnapshot; - newSnapshot: DBSnapshot; - diff: deepDiff.Diff[]; - newFilename: string; - summary: string; - newFileContent?: string; - } - | { - state: 'up-to-date'; - currentSnapshot: DBSnapshot; - }; - -export const INITIAL_SNAPSHOT = '0000_snapshot.json'; - -export async function getMigrationStatus({ - dbConfig, - root, -}: { - dbConfig: DBConfig; - root: URL; -}): Promise { - const currentSnapshot = createCurrentSnapshot(dbConfig); - const dir = getMigrationsDirectoryUrl(root); - const allMigrationFiles = await getMigrations(dir); - - if (allMigrationFiles.length === 0) { - return { - state: 'no-migrations-found', - currentSnapshot, - }; - } - - const previousSnapshot = await initializeFromMigrations(allMigrationFiles, dir); - const diff = generateDiff(previousSnapshot, currentSnapshot); - - if (diff) { - const n = getNewMigrationNumber(allMigrationFiles); - const newFilename = `${String(n + 1).padStart(4, '0')}_migration.json`; - return { - state: 'ahead', - oldSnapshot: previousSnapshot, - newSnapshot: currentSnapshot, - diff, - newFilename, - summary: generateDiffSummary(diff), - }; - } - - return { - state: 'up-to-date', - currentSnapshot, - }; -} - -export const MIGRATIONS_CREATED = `${green( - '■ Migrations initialized!' -)}\n\n To execute your migrations, run\n ${cyan('astro db push')}`; -export const MIGRATIONS_UP_TO_DATE = `${green( - '■ No migrations needed!' -)}\n\n Your database is up to date.\n`; -export const MIGRATIONS_NOT_INITIALIZED = `${yellow( - '▶ No migrations found!' -)}\n\n To scaffold your migrations folder, run\n ${cyan('astro db sync')}\n`; -export const MIGRATION_NEEDED = `${yellow( - '▶ Changes detected!' -)}\n\n To create the necessary migration file, run\n ${cyan('astro db sync')}\n`; - -function generateDiffSummary(diff: deepDiff.Diff[]) { - // TODO: human readable summary - return JSON.stringify(diff, null, 2); -} - -function getNewMigrationNumber(allMigrationFiles: string[]): number { - const len = allMigrationFiles.length - 1; - return allMigrationFiles.reduce((acc, curr) => { - const num = Number.parseInt(curr.split('_')[0] ?? len, 10); - return num > acc ? num : acc; - }, 0); -} - -export async function getMigrations(dir: URL): Promise { - const migrationFiles = await readdir(dir).catch((err) => { - if (err.code === 'ENOENT') { - return []; - } - throw err; - }); - return migrationFiles; -} - -export async function loadMigration( - migration: string, - dir: URL -): Promise<{ diff: any[]; db: string[]; confirm?: string[] }> { - return JSON.parse(await readFile(new URL(migration, dir), 'utf-8')); -} - -export async function loadInitialSnapshot(dir: URL): Promise { - const snapshot = JSON.parse(await readFile(new URL(INITIAL_SNAPSHOT, dir), 'utf-8')); - // `experimentalVersion: 1` -- added the version column - if (snapshot.experimentalVersion === 1) { - return snapshot; - } - // `experimentalVersion: 0` -- initial format - if (!snapshot.schema) { - return { experimentalVersion: 1, schema: snapshot }; - } - throw new Error('Invalid snapshot format'); -} - -export async function initializeMigrationsDirectory(currentSnapshot: DBSnapshot, dir: URL) { - await mkdir(dir, { recursive: true }); - await writeFile(new URL(INITIAL_SNAPSHOT, dir), JSON.stringify(currentSnapshot, undefined, 2)); -} - -export async function initializeFromMigrations( - allMigrationFiles: string[], - dir: URL -): Promise { - const prevSnapshot = await loadInitialSnapshot(dir); - for (const migration of allMigrationFiles) { - if (migration === INITIAL_SNAPSHOT) continue; - const migrationContent = await loadMigration(migration, dir); - migrationContent.diff.forEach((change: any) => { - applyChange(prevSnapshot, {}, change); - }); - } - return prevSnapshot; -} - -export function createCurrentSnapshot({ tables = {} }: DBConfig): DBSnapshot { - const schema = JSON.parse(JSON.stringify(tables)); - return { experimentalVersion: 1, schema }; -} -export function createEmptySnapshot(): DBSnapshot { - return { experimentalVersion: 1, schema: {} }; -} diff --git a/packages/db/src/core/errors.ts b/packages/db/src/core/errors.ts index 706553a0673e..4931b2bc5fcc 100644 --- a/packages/db/src/core/errors.ts +++ b/packages/db/src/core/errors.ts @@ -10,14 +10,26 @@ export const MISSING_PROJECT_ID_ERROR = `${red('▶ Directory not linked.')} To link this directory to an Astro Studio project, run ${cyan('astro db link')}\n`; -export const MIGRATIONS_NOT_INITIALIZED = `${yellow( - '▶ No migrations found!' -)}\n\n To scaffold your migrations folder, run\n ${cyan('astro db sync')}\n`; - export const MISSING_EXECUTE_PATH_ERROR = `${red( '▶ No file path provided.' )} Provide a path by running ${cyan('astro db execute ')}\n`; +export const RENAME_TABLE_ERROR = (oldTable: string, newTable: string) => { + return ( + red('▶ Potential table rename detected: ' + oldTable + ', ' + newTable) + + `\n You cannot add and remove tables in the same schema update batch.` + + `\n To resolve, add a 'deprecated: true' flag to '${oldTable}' instead.` + ); +}; + +export const RENAME_COLUMN_ERROR = (oldSelector: string, newSelector: string) => { + return ( + red('▶ Potential column rename detected: ' + oldSelector + ', ' + newSelector) + + `\n You cannot add and remove columns in the same table.` + + `\n To resolve, add a 'deprecated: true' flag to '${oldSelector}' instead.` + ); +}; + export const FILE_NOT_FOUND_ERROR = (path: string) => `${red('▶ File not found:')} ${bold(path)}\n`; diff --git a/packages/db/src/core/integration/typegen.ts b/packages/db/src/core/integration/typegen.ts index 4fad1bc92189..1c4834d13e1f 100644 --- a/packages/db/src/core/integration/typegen.ts +++ b/packages/db/src/core/integration/typegen.ts @@ -26,9 +26,14 @@ ${Object.entries(tables) } function generateTableType(name: string, collection: DBTable): string { + const sanitizedColumnsList = Object.entries(collection.columns) + // Filter out deprecated columns from the typegen, so that they don't + // appear as queryable fields in the generated types / your codebase. + .filter(([key, val]) => !val.schema.deprecated); + const sanitizedColumns = Object.fromEntries(sanitizedColumnsList); let tableType = ` export const ${name}: import(${RUNTIME_IMPORT}).Table< ${JSON.stringify(name)}, - ${JSON.stringify(collection.columns)} + ${JSON.stringify(sanitizedColumns)} >;`; return tableType; } diff --git a/packages/db/src/core/types.ts b/packages/db/src/core/types.ts index f9de8f8ee519..a3ae42311355 100644 --- a/packages/db/src/core/types.ts +++ b/packages/db/src/core/types.ts @@ -21,7 +21,8 @@ const baseColumnSchema = z.object({ label: z.string().optional(), optional: z.boolean().optional().default(false), unique: z.boolean().optional().default(false), - + deprecated: z.boolean().optional().default(false), + // Defined when `defineReadableTable()` is called name: z.string().optional(), // TODO: rename to `tableName`. Breaking schema change @@ -184,6 +185,7 @@ export const tableSchema = z.object({ columns: columnsSchema, indexes: z.record(indexSchema).optional(), foreignKeys: z.array(foreignKeysSchema).optional(), + deprecated: z.boolean().optional().default(false), }); export const tablesSchema = z.preprocess((rawTables) => { @@ -258,6 +260,7 @@ export interface TableConfig references: () => MaybeArray>; }>; indexes?: Record>; + deprecated?: boolean; } interface IndexConfig extends z.input { diff --git a/packages/db/src/core/utils.ts b/packages/db/src/core/utils.ts index 674514711075..d57e3f660b34 100644 --- a/packages/db/src/core/utils.ts +++ b/packages/db/src/core/utils.ts @@ -21,7 +21,3 @@ export function getAstroStudioUrl(): string { export function getDbDirectoryUrl(root: URL | string) { return new URL('db/', root); } - -export function getMigrationsDirectoryUrl(root: URL | string) { - return new URL('migrations/', getDbDirectoryUrl(root)); -} diff --git a/packages/db/src/runtime/queries.ts b/packages/db/src/runtime/queries.ts index 4341fd1237ab..cd54381f8f65 100644 --- a/packages/db/src/runtime/queries.ts +++ b/packages/db/src/runtime/queries.ts @@ -66,6 +66,10 @@ export async function recreateTables({ db, tables }: { db: SqliteDB; tables: DBT ]); } +export function getDropTableIfExistsQuery(tableName: string) { + return `DROP TABLE IF EXISTS ${sqlite.escapeName(tableName)}`; +}; + export function getCreateTableQuery(tableName: string, table: DBTable) { let query = `CREATE TABLE ${sqlite.escapeName(tableName)} (`; diff --git a/packages/db/test/fixtures/basics/db/config.ts b/packages/db/test/fixtures/basics/db/config.ts index f216caab6e45..58657d43f9a0 100644 --- a/packages/db/test/fixtures/basics/db/config.ts +++ b/packages/db/test/fixtures/basics/db/config.ts @@ -4,6 +4,7 @@ import { column, defineDB, defineTable } from 'astro:db'; const Author = defineTable({ columns: { name: column.text(), + age2: column.number({optional: true}), }, }); diff --git a/packages/db/test/fixtures/basics/db/theme.ts b/packages/db/test/fixtures/basics/db/theme.ts index aecc67f7d5ae..26a38af07f34 100644 --- a/packages/db/test/fixtures/basics/db/theme.ts +++ b/packages/db/test/fixtures/basics/db/theme.ts @@ -9,7 +9,7 @@ export const Themes = defineTable({ updated: column.date({ default: NOW, }), - isDark: column.boolean({ default: sql`TRUE` }), + isDark: column.boolean({ default: sql`TRUE`, deprecated: true}), owner: column.text({ optional: true, default: sql`NULL` }), }, }); diff --git a/packages/db/test/unit/column-queries.test.js b/packages/db/test/unit/column-queries.test.js index c4d60d5c6f37..e8ef08bee536 100644 --- a/packages/db/test/unit/column-queries.test.js +++ b/packages/db/test/unit/column-queries.test.js @@ -24,29 +24,21 @@ const userInitial = tableSchema.parse( }) ); -const defaultAmbiguityResponses = { - collectionRenames: {}, - columnRenames: {}, -}; - -function userChangeQueries(oldTable, newTable, ambiguityResponses = defaultAmbiguityResponses) { +function userChangeQueries(oldTable, newTable) { return getCollectionChangeQueries({ collectionName: TABLE_NAME, oldCollection: oldTable, newCollection: newTable, - ambiguityResponses, }); } function configChangeQueries( oldCollections, newCollections, - ambiguityResponses = defaultAmbiguityResponses ) { return getMigrationQueries({ oldSnapshot: { schema: oldCollections, experimentalVersion: 1 }, newSnapshot: { schema: newCollections, experimentalVersion: 1 }, - ambiguityResponses, }); } @@ -63,7 +55,10 @@ describe('column queries', () => { const oldCollections = {}; const newCollections = { [TABLE_NAME]: userInitial }; const { queries } = await configChangeQueries(oldCollections, newCollections); - expect(queries).to.deep.equal([getCreateTableQuery(TABLE_NAME, userInitial)]); + expect(queries).to.deep.equal([ + `DROP TABLE IF EXISTS "${TABLE_NAME}"`, + `CREATE TABLE "${TABLE_NAME}" (_id INTEGER PRIMARY KEY, "name" text NOT NULL, "age" integer NOT NULL, "email" text NOT NULL UNIQUE, "mi" text)`, + ]); }); it('should drop table for removed tables', async () => { @@ -73,15 +68,42 @@ describe('column queries', () => { expect(queries).to.deep.equal([`DROP TABLE "${TABLE_NAME}"`]); }); - it('should rename table for renamed tables', async () => { + it('should error if possible table rename is detected', async () => { const rename = 'Peeps'; const oldCollections = { [TABLE_NAME]: userInitial }; const newCollections = { [rename]: userInitial }; - const { queries } = await configChangeQueries(oldCollections, newCollections, { - ...defaultAmbiguityResponses, - collectionRenames: { [rename]: TABLE_NAME }, + let error = null; + try { + await configChangeQueries(oldCollections, newCollections, { + collectionRenames: { [rename]: TABLE_NAME }, + }); + } catch (e) { + error = e.message; + } + expect(error).to.include.string('Potential table rename detected'); + }); + + it('should error if possible column rename is detected', async () => { + const blogInitial = tableSchema.parse({ + columns: { + title: column.text(), + }, + }); + const blogFinal = tableSchema.parse({ + columns: { + title2: column.text(), + }, }); - expect(queries).to.deep.equal([`ALTER TABLE "${TABLE_NAME}" RENAME TO "${rename}"`]); + let error = null; + try { + await configChangeQueries( + {[TABLE_NAME]: blogInitial}, + {[TABLE_NAME]: blogFinal}, + ); + } catch (e) { + error = e.message; + } + expect(error).to.include.string('Potential column rename detected'); }); }); @@ -139,27 +161,6 @@ describe('column queries', () => { ]); }); - describe('ALTER RENAME COLUMN', () => { - it('when renaming a column', async () => { - const userFinal = { - ...userInitial, - columns: { - ...userInitial.columns, - }, - }; - userFinal.columns.middleInitial = userFinal.columns.mi; - delete userFinal.columns.mi; - - const { queries } = await userChangeQueries(userInitial, userFinal, { - collectionRenames: {}, - columnRenames: { [TABLE_NAME]: { middleInitial: 'mi' } }, - }); - expect(queries).to.deep.equal([ - `ALTER TABLE "${TABLE_NAME}" RENAME COLUMN "mi" TO "middleInitial"`, - ]); - }); - }); - describe('Lossy table recreate', () => { it('when changing a column type', async () => { const userFinal = {