👷 Improve monthly clean database script

Archive results that were not properly archived

apps/builder/src/features/results/api/deleteResults.ts

@@ -3,7 +3,8 @@ import { authenticatedProcedure } from '@/helpers/server/trpc'
 import { TRPCError } from '@trpc/server'
 import { Typebot } from '@typebot.io/schemas'
 import { z } from 'zod'
-import { archiveResults } from '../helpers/archiveResults'
+import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
+import prisma from '@/lib/prisma'
 
 export const deleteResults = authenticatedProcedure
   .meta({
@@ -40,7 +41,7 @@ export const deleteResults = authenticatedProcedure
     })) as Pick<Typebot, 'groups'> | null
     if (!typebot)
       throw new TRPCError({ code: 'NOT_FOUND', message: 'Typebot not found' })
-    const { success } = await archiveResults({
+    const { success } = await archiveResults(prisma)({
       typebot,
       resultsFilter: {
         id: (idsArray?.length ?? 0) > 0 ? { in: idsArray } : undefined,

apps/builder/src/pages/api/typebots/[typebotId].ts

@@ -6,10 +6,10 @@ import { getAuthenticatedUser } from '@/features/auth/helpers/getAuthenticatedUs
 import { Typebot } from '@typebot.io/schemas'
 import { omit } from '@typebot.io/lib'
 import { getTypebot } from '@/features/typebot/helpers/getTypebot'
-import { archiveResults } from '@/features/results/helpers/archiveResults'
 import { isReadTypebotForbidden } from '@/features/typebot/helpers/isReadTypebotForbidden'
 import { removeTypebotOldProperties } from '@/features/typebot/helpers/removeTypebotOldProperties'
 import { roundGroupsCoordinate } from '@/features/typebot/helpers/roundGroupsCoordinate'
+import { archiveResults } from '@typebot.io/lib/api/helpers/archiveResults'
 
 const handler = async (req: NextApiRequest, res: NextApiResponse) => {
   const user = await getAuthenticatedUser(req, res)
@@ -56,7 +56,7 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => {
       },
     })) as Pick<Typebot, 'groups'> | null
     if (!typebot) return res.status(404).send({ typebot: null })
-    const { success } = await archiveResults({
+    const { success } = await archiveResults(prisma)({
       typebot,
       resultsFilter: { typebotId },
     })

packages/lib/api/helpers/archiveResults.ts

@@ -0,0 +1,126 @@
+import { Prisma, PrismaClient } from '@typebot.io/prisma'
+import { InputBlockType, Typebot } from '@typebot.io/schemas'
+import { Client } from 'minio'
+
+type ArchiveResultsProps = {
+  typebot: Pick<Typebot, 'groups'>
+  resultsFilter?: Omit<Prisma.ResultWhereInput, 'typebotId'> & {
+    typebotId: string
+  }
+}
+
+export const archiveResults =
+  (prisma: PrismaClient) =>
+  async ({ typebot, resultsFilter }: ArchiveResultsProps) => {
+    const batchSize = 100
+    const fileUploadBlockIds = typebot.groups
+      .flatMap((group) => group.blocks)
+      .filter((block) => block.type === InputBlockType.FILE)
+      .map((block) => block.id)
+
+    let currentTotalResults = 0
+
+    const resultsCount = await prisma.result.count({
+      where: {
+        ...resultsFilter,
+        OR: [{ isArchived: false }, { isArchived: null }],
+      },
+    })
+
+    if (resultsCount === 0) return { success: true }
+
+    let progress = 0
+
+    do {
+      progress += batchSize
+      console.log(`Archiving ${progress} / ${resultsCount} results...`)
+      const resultsToDelete = await prisma.result.findMany({
+        where: {
+          ...resultsFilter,
+          OR: [{ isArchived: false }, { isArchived: null }],
+        },
+        select: {
+          id: true,
+        },
+        take: batchSize,
+      })
+
+      if (resultsToDelete.length === 0) break
+
+      currentTotalResults = resultsToDelete.length
+
+      const resultIds = resultsToDelete.map((result) => result.id)
+
+      if (fileUploadBlockIds.length > 0) {
+        const filesToDelete = await prisma.answer.findMany({
+          where: {
+            resultId: { in: resultIds },
+            blockId: { in: fileUploadBlockIds },
+          },
+        })
+        if (filesToDelete.length > 0)
+          await deleteFilesFromBucket({
+            urls: filesToDelete.flatMap((a) => a.content.split(', ')),
+          })
+      }
+
+      await prisma.$transaction([
+        prisma.log.deleteMany({
+          where: {
+            resultId: { in: resultIds },
+          },
+        }),
+        prisma.answer.deleteMany({
+          where: {
+            resultId: { in: resultIds },
+          },
+        }),
+        prisma.result.updateMany({
+          where: {
+            id: { in: resultIds },
+          },
+          data: {
+            isArchived: true,
+            variables: [],
+          },
+        }),
+      ])
+    } while (currentTotalResults >= batchSize)
+
+    return { success: true }
+  }
+
+const deleteFilesFromBucket = async ({
+  urls,
+}: {
+  urls: string[]
+}): Promise<void> => {
+  if (
+    !process.env.S3_ENDPOINT ||
+    !process.env.S3_ACCESS_KEY ||
+    !process.env.S3_SECRET_KEY
+  )
+    throw new Error(
+      'S3 not properly configured. Missing one of those variables: S3_ENDPOINT, S3_ACCESS_KEY, S3_SECRET_KEY'
+    )
+
+  const useSSL =
+    process.env.S3_SSL && process.env.S3_SSL === 'false' ? false : true
+  const minioClient = new Client({
+    endPoint: process.env.S3_ENDPOINT,
+    port: process.env.S3_PORT ? parseInt(process.env.S3_PORT) : undefined,
+    useSSL,
+    accessKey: process.env.S3_ACCESS_KEY,
+    secretKey: process.env.S3_SECRET_KEY,
+    region: process.env.S3_REGION,
+  })
+
+  const bucket = process.env.S3_BUCKET ?? 'typebot'
+
+  return minioClient.removeObjects(
+    bucket,
+    urls
+      .filter((url) => url.includes(process.env.S3_ENDPOINT as string))
+      .map((url) => url.split(`/${bucket}/`)[1])
+  )
+}

packages/lib/package.json

@@ -8,14 +8,14 @@
   "devDependencies": {
     "@paralleldrive/cuid2": "2.2.0",
     "@playwright/test": "1.34.3",
+    "@typebot.io/prisma": "workspace:*",
+    "@typebot.io/schemas": "workspace:*",
+    "@typebot.io/tsconfig": "workspace:*",
     "@types/nodemailer": "6.4.8",
     "aws-sdk": "2.1384.0",
-    "@typebot.io/prisma": "workspace:*",
     "dotenv": "16.0.3",
-    "@typebot.io/schemas": "workspace:*",
     "next": "13.4.3",
     "nodemailer": "6.9.2",
-    "@typebot.io/tsconfig": "workspace:*",
     "typescript": "5.0.4"
   },
   "peerDependencies": {
@@ -24,6 +24,7 @@
     "nodemailer": "6.7.8"
   },
   "dependencies": {
-    "got": "12.6.0"
+    "got": "12.6.0",
+    "minio": "7.1.1"
   }
 }