From 79c870b48841fe747d26f6abc1c95d09084639ad Mon Sep 17 00:00:00 2001 From: haricnugraha Date: Tue, 29 Oct 2024 17:04:49 +0700 Subject: [PATCH] feat: add image preview for video --- events/publishers/audio.ts | 17 +++ events/subscribers/audio.ts | 102 +++++++++++++++++- events/subscribers/video.ts | 6 +- next.config.js | 7 +- pages/index.tsx | 21 ++-- .../migration.sql | 8 ++ prisma/schema.prisma | 3 +- scripts/worker.ts | 3 +- services/video/repository.ts | 15 +++ 9 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 events/publishers/audio.ts create mode 100644 prisma/migrations/20241029093522_add_segment_image_url/migration.sql diff --git a/events/publishers/audio.ts b/events/publishers/audio.ts new file mode 100644 index 0000000..a5b5d51 --- /dev/null +++ b/events/publishers/audio.ts @@ -0,0 +1,17 @@ +import { publish } from "@/services/pubsub/publisher"; +import { events } from "../"; + +type PublishAudioTranscribedParameters = { + id: string; + objectStorageName: string; +}; + +export async function publishAudioTranscribed({ + id, + objectStorageName, +}: PublishAudioTranscribedParameters) { + await publish(events.audio.transcribed, { + id, + objectStorageName, + }); +} diff --git a/events/subscribers/audio.ts b/events/subscribers/audio.ts index a8c96f3..89c6b8a 100644 --- a/events/subscribers/audio.ts +++ b/events/subscribers/audio.ts @@ -1,10 +1,17 @@ import { exec } from "node:child_process"; import fs from "node:fs/promises"; import { promisify } from "node:util"; -import { downloadFile, uploadFile } from "@/services/object-storage"; -import { update, updateWithSegments } from "@/services/video/repository"; import { VideoStatus } from "@prisma/client"; +import { intervalToDuration } from "date-fns"; import { config } from "@/config"; +import { downloadFile, uploadFile } from "@/services/object-storage"; +import { + getSegmentsBy, + update, + updateSegmentImageUrl, + updateWithSegments, +} from "@/services/video/repository"; +import { publishAudioTranscribed } from "../publishers/audio"; const execPromisify = promisify(exec); @@ -51,6 +58,11 @@ export async function onAudioUpload({ console.info(`Insert transcription from ${outputFSPath} to database`); await storeTranscription({ id, transcriptionFSPath: outputFSPath }); + await publishAudioTranscribed({ + id, + objectStorageName, + }); + // delete temporary file console.info(`Deleting temporary file ${destinationFSPath}`); fs.rm(destinationFSPath); @@ -112,3 +124,89 @@ async function storeTranscription({ })) ); } + +export async function onAudioTranscribe({ + id, + objectStorageName, +}: OnAudioUploadParams) { + const videoFileName = objectStorageName.replace("/videos/", ""); + const videoFSPath = `/tmp/${videoFileName}`; + console.info(`Downloading video ${objectStorageName} to ${videoFSPath}`); + const [_, segments] = await Promise.all([ + downloadFile({ + destinationFSPath: videoFSPath, + objectStorageName, + }), + getSegmentsBy(id, {}), + ]); + + await Promise.all( + segments.map(async ({ id, start }) => { + // capture the image + const imageName = `${Math.floor(start)}.jpg`; + const imageDestinationFSPath = `/tmp/images-${id}-${imageName}`; + await getImageAt({ imageDestinationFSPath, seconds: start, videoFSPath }); + + // store the image to object storage + const imageObjectStorageName = `/images/${id}/${imageName}`; + console.info( + `Image ${imageDestinationFSPath} uploading to ${imageObjectStorageName}` + ); + await uploadFile({ + objectStorageName: imageObjectStorageName, + targetFSPath: imageDestinationFSPath, + }); + + // store data to database + const { bucketName, host, port, useSSL } = config.objectStorage; + const imageUrl = `${ + useSSL ? "https://" : "http://" + }${host}:${port}/${bucketName}${imageObjectStorageName}`; + console.info(`Updating image URL to ${imageUrl} for segment ${id}`); + await updateSegmentImageUrl({ id, imageUrl }); + + // Delete temporary file + console.info(`Deleting temporary file ${imageDestinationFSPath}`); + fs.rm(imageDestinationFSPath); + }) + ); + + // Delete temporary file + console.info(`Deleting temporary file ${videoFSPath}`); + fs.rm(videoFSPath); + + return; +} + +type GetImageAtParameters = { + seconds: number; + videoFSPath: string; + imageDestinationFSPath: string; +}; + +async function getImageAt({ + seconds, + videoFSPath, + imageDestinationFSPath, +}: GetImageAtParameters): Promise { + const time = secondToDuration(seconds); + const cmd = `ffmpeg -ss ${time} -i ${videoFSPath} -frames:v 1 ${imageDestinationFSPath}`; + console.info( + `Capture image at ${time}, and store it to ${imageDestinationFSPath}` + ); + + await execPromisify(cmd); +} + +function secondToDuration(seconds: number) { + const { + hours, + minutes, + seconds: durationSeconds, + } = intervalToDuration({ start: 0, end: seconds * 1000 }); + const sanitize = (value?: number): string => { + return value ? `${value}` : "00"; + }; + + return `${sanitize(hours)}:${sanitize(minutes)}:${sanitize(durationSeconds)}`; +} diff --git a/events/subscribers/video.ts b/events/subscribers/video.ts index ae53993..bee5289 100644 --- a/events/subscribers/video.ts +++ b/events/subscribers/video.ts @@ -17,9 +17,9 @@ export async function onVideoUpload({ id, objectStorageName, }: OnVideoUploadParams): Promise { - const VideoFileName = objectStorageName.replace("/videos/", ""); - const videoFileNameWithoutFormat = VideoFileName.split(".")[0]; - const videoFSPath = `/tmp/${VideoFileName}`; + const videoFileName = objectStorageName.replace("/videos/", ""); + const videoFileNameWithoutFormat = videoFileName.split(".")[0]; + const videoFSPath = `/tmp/${videoFileName}`; await update(id, { status: VideoStatus.CONVERTING }); diff --git a/next.config.js b/next.config.js index a843cbe..3ed7818 100644 --- a/next.config.js +++ b/next.config.js @@ -1,6 +1,9 @@ /** @type {import('next').NextConfig} */ const nextConfig = { reactStrictMode: true, -} + images: { + domains: ["localhost"], + }, +}; -module.exports = nextConfig +module.exports = nextConfig; diff --git a/pages/index.tsx b/pages/index.tsx index 11161c2..362ab59 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -7,10 +7,12 @@ import { Col, Empty, Form, + Image, Input, Layout, List, Modal, + Popover, Row, Skeleton, Space, @@ -345,7 +347,7 @@ function VideoCard({ @@ -432,15 +434,22 @@ function SearchResult({ ( + renderItem={({ start, text, imageUrl }) => ( onSeeked(start)} style={{ cursor: "pointer" }} > - {humanReadableSeekPosition(start)}{" "} - - {text} - + } + placement="right" + > + + {humanReadableSeekPosition(start)} + {" "} + + {text} + + )} style={{ marginTop: "0.75rem" }} diff --git a/prisma/migrations/20241029093522_add_segment_image_url/migration.sql b/prisma/migrations/20241029093522_add_segment_image_url/migration.sql new file mode 100644 index 0000000..c095847 --- /dev/null +++ b/prisma/migrations/20241029093522_add_segment_image_url/migration.sql @@ -0,0 +1,8 @@ +-- DropForeignKey +ALTER TABLE "Segment" DROP CONSTRAINT "Segment_videoId_fkey"; + +-- AlterTable +ALTER TABLE "Segment" ADD COLUMN "imageUrl" TEXT NOT NULL DEFAULT ''; + +-- AddForeignKey +ALTER TABLE "Segment" ADD CONSTRAINT "Segment_videoId_fkey" FOREIGN KEY ("videoId") REFERENCES "Video"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index edaeb8e..705ce98 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -37,10 +37,11 @@ model Segment { start Float end Float text String + imageUrl String @default("") createdAt Int updatedAt Int - video Video @relation(fields: [videoId], references: [id]) + video Video @relation(fields: [videoId], references: [id], onDelete: Cascade) videoId String @@index([text]) diff --git a/scripts/worker.ts b/scripts/worker.ts index 3ad1051..1c62169 100644 --- a/scripts/worker.ts +++ b/scripts/worker.ts @@ -1,6 +1,6 @@ import { type NatsConnection, JSONCodec } from "nats"; import events from "@/events"; -import { onAudioUpload } from "@/events/subscribers/audio"; +import { onAudioTranscribe, onAudioUpload } from "@/events/subscribers/audio"; import { onVideoUpload } from "@/events/subscribers/video"; import { closeConnection, newConnection } from "@/services/pubsub/connect"; @@ -9,6 +9,7 @@ const jc = JSONCodec(); export const eventConsumers = [ { event: events.video.uploaded, consumer: onVideoUpload }, { event: events.video.converted, consumer: onAudioUpload }, + { event: events.audio.transcribed, consumer: onAudioTranscribe }, ]; async function main() { diff --git a/services/video/repository.ts b/services/video/repository.ts index 82e7556..9962e03 100644 --- a/services/video/repository.ts +++ b/services/video/repository.ts @@ -108,6 +108,21 @@ export async function updateWithSegments( }); } +type UpdateSegmentImageUrlParameters = { + id: string; + imageUrl: string; +}; + +export async function updateSegmentImageUrl({ + id, + imageUrl, +}: UpdateSegmentImageUrlParameters) { + return await prisma.segment.update({ + where: { id }, + data: { imageUrl, updatedAt: getUnixTimeStamp() }, + }); +} + export async function getSegmentsBy( videoId: string, { search }: GetVideoParams