feat: add image preview for video

events/publishers/audio.ts

@@ -0,0 +1,17 @@
+import { publish } from "@/services/pubsub/publisher";
+import { events } from "../";
+
+type PublishAudioTranscribedParameters = {
+  id: string;
+  objectStorageName: string;
+};
+
+export async function publishAudioTranscribed({
+  id,
+  objectStorageName,
+}: PublishAudioTranscribedParameters) {
+  await publish(events.audio.transcribed, {
+    id,
+    objectStorageName,
+  });
+}

events/subscribers/audio.ts

@@ -1,10 +1,17 @@
 import { exec } from "node:child_process";
 import fs from "node:fs/promises";
 import { promisify } from "node:util";
-import { downloadFile, uploadFile } from "@/services/object-storage";
-import { update, updateWithSegments } from "@/services/video/repository";
 import { VideoStatus } from "@prisma/client";
+import { intervalToDuration } from "date-fns";
 import { config } from "@/config";
+import { downloadFile, uploadFile } from "@/services/object-storage";
+import {
+  getSegmentsBy,
+  update,
+  updateSegmentImageUrl,
+  updateWithSegments,
+} from "@/services/video/repository";
+import { publishAudioTranscribed } from "../publishers/audio";
 
 const execPromisify = promisify(exec);
 
@@ -51,6 +58,11 @@ export async function onAudioUpload({
   console.info(`Insert transcription from ${outputFSPath} to database`);
   await storeTranscription({ id, transcriptionFSPath: outputFSPath });
 
+  await publishAudioTranscribed({
+    id,
+    objectStorageName,
+  });
+
   // delete temporary file
   console.info(`Deleting temporary file ${destinationFSPath}`);
   fs.rm(destinationFSPath);
@@ -112,3 +124,89 @@ async function storeTranscription({
     }))
   );
 }
+
+export async function onAudioTranscribe({
+  id,
+  objectStorageName,
+}: OnAudioUploadParams) {
+  const videoFileName = objectStorageName.replace("/videos/", "");
+  const videoFSPath = `/tmp/${videoFileName}`;
+  console.info(`Downloading video ${objectStorageName} to ${videoFSPath}`);
+  const [_, segments] = await Promise.all([
+    downloadFile({
+      destinationFSPath: videoFSPath,
+      objectStorageName,
+    }),
+    getSegmentsBy(id, {}),
+  ]);
+
+  await Promise.all(
+    segments.map(async ({ id, start }) => {
+      // capture the image
+      const imageName = `${Math.floor(start)}.jpg`;
+      const imageDestinationFSPath = `/tmp/images-${id}-${imageName}`;
+      await getImageAt({ imageDestinationFSPath, seconds: start, videoFSPath });
+
+      // store the image to object storage
+      const imageObjectStorageName = `/images/${id}/${imageName}`;
+      console.info(
+        `Image ${imageDestinationFSPath} uploading to ${imageObjectStorageName}`
+      );
+      await uploadFile({
+        objectStorageName: imageObjectStorageName,
+        targetFSPath: imageDestinationFSPath,
+      });
+
+      // store data to database
+      const { bucketName, host, port, useSSL } = config.objectStorage;
+      const imageUrl = `${
+        useSSL ? "https://" : "http://"
+      }${host}:${port}/${bucketName}${imageObjectStorageName}`;
+      console.info(`Updating image URL to ${imageUrl} for segment ${id}`);
+      await updateSegmentImageUrl({ id, imageUrl });
+
+      // Delete temporary file
+      console.info(`Deleting temporary file ${imageDestinationFSPath}`);
+      fs.rm(imageDestinationFSPath);
+    })
+  );
+
+  // Delete temporary file
+  console.info(`Deleting temporary file ${videoFSPath}`);
+  fs.rm(videoFSPath);
+
+  return;
+}
+
+type GetImageAtParameters = {
+  seconds: number;
+  videoFSPath: string;
+  imageDestinationFSPath: string;
+};
+
+async function getImageAt({
+  seconds,
+  videoFSPath,
+  imageDestinationFSPath,
+}: GetImageAtParameters): Promise<void> {
+  const time = secondToDuration(seconds);
+  const cmd = `ffmpeg -ss ${time} -i ${videoFSPath} -frames:v 1 ${imageDestinationFSPath}`;
+  console.info(
+    `Capture image at ${time}, and store it to ${imageDestinationFSPath}`
+  );
+
+  await execPromisify(cmd);
+}
+
+function secondToDuration(seconds: number) {
+  const {
+    hours,
+    minutes,
+    seconds: durationSeconds,
+  } = intervalToDuration({ start: 0, end: seconds * 1000 });
+  const sanitize = (value?: number): string => {
+    return value ? `${value}` : "00";
+  };
+
+  return `${sanitize(hours)}:${sanitize(minutes)}:${sanitize(durationSeconds)}`;
+}

events/subscribers/video.ts

@@ -17,9 +17,9 @@ export async function onVideoUpload({
   id,
   objectStorageName,
 }: OnVideoUploadParams): Promise<void> {
-  const VideoFileName = objectStorageName.replace("/videos/", "");
-  const videoFileNameWithoutFormat = VideoFileName.split(".")[0];
-  const videoFSPath = `/tmp/${VideoFileName}`;
+  const videoFileName = objectStorageName.replace("/videos/", "");
+  const videoFileNameWithoutFormat = videoFileName.split(".")[0];
+  const videoFSPath = `/tmp/${videoFileName}`;
 
   await update(id, { status: VideoStatus.CONVERTING });
 

next.config.js

@@ -1,6 +1,9 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   reactStrictMode: true,
-}
+  images: {
+    domains: ["localhost"],
+  },
+};
 
-module.exports = nextConfig
+module.exports = nextConfig;

pages/index.tsx

@@ -7,10 +7,12 @@ import {
   Col,
   Empty,
   Form,
+  Image,
   Input,
   Layout,
   List,
   Modal,
+  Popover,
   Row,
   Skeleton,
   Space,
@@ -345,7 +347,7 @@ function VideoCard({
       <video
         key={videoUrl}
         controls
-        style={{ width: "100%", paddingBottom: "0.5rem" }}
+        style={{ width: "100%", paddingBottom: "0.5rem", maxHeight: 300 }}
       >
         <source src={videoUrl} type={type} />
       </video>
@@ -432,15 +434,22 @@ function SearchResult({
     <List
       bordered
       dataSource={segments}
-      renderItem={({ start, text }) => (
+      renderItem={({ start, text, imageUrl }) => (
         <List.Item
           onClick={() => onSeeked(start)}
           style={{ cursor: "pointer" }}
         >
-          <Typography.Text>{humanReadableSeekPosition(start)}</Typography.Text>{" "}
-          <Typography.Text style={{ marginLeft: "0.5rem", color: "#1677ff" }}>
-            {text}
-          </Typography.Text>
+          <Popover
+            content={<Image alt={text} src={imageUrl} width={200} />}
+            placement="right"
+          >
+            <Typography.Text>
+              {humanReadableSeekPosition(start)}
+            </Typography.Text>{" "}
+            <Typography.Text style={{ marginLeft: "0.5rem", color: "#1677ff" }}>
+              {text}
+            </Typography.Text>
+          </Popover>
         </List.Item>
       )}
       style={{ marginTop: "0.75rem" }}

prisma/migrations/20241029093522_add_segment_image_url/migration.sql

@@ -0,0 +1,8 @@
+-- DropForeignKey
+ALTER TABLE "Segment" DROP CONSTRAINT "Segment_videoId_fkey";
+
+-- AlterTable
+ALTER TABLE "Segment" ADD COLUMN     "imageUrl" TEXT NOT NULL DEFAULT '';
+
+-- AddForeignKey
+ALTER TABLE "Segment" ADD CONSTRAINT "Segment_videoId_fkey" FOREIGN KEY ("videoId") REFERENCES "Video"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -37,10 +37,11 @@ model Segment {
   start     Float
   end       Float
   text      String
+  imageUrl  String @default("")
   createdAt Int
   updatedAt Int
 
-  video   Video  @relation(fields: [videoId], references: [id])
+  video   Video  @relation(fields: [videoId], references: [id], onDelete: Cascade)
   videoId String
 
   @@index([text])

scripts/worker.ts

@@ -1,6 +1,6 @@
 import { type NatsConnection, JSONCodec } from "nats";
 import events from "@/events";
-import { onAudioUpload } from "@/events/subscribers/audio";
+import { onAudioTranscribe, onAudioUpload } from "@/events/subscribers/audio";
 import { onVideoUpload } from "@/events/subscribers/video";
 import { closeConnection, newConnection } from "@/services/pubsub/connect";
 
@@ -9,6 +9,7 @@ const jc = JSONCodec();
 export const eventConsumers = [
   { event: events.video.uploaded, consumer: onVideoUpload },
   { event: events.video.converted, consumer: onAudioUpload },
+  { event: events.audio.transcribed, consumer: onAudioTranscribe },
 ];
 
 async function main() {

services/video/repository.ts

@@ -108,6 +108,21 @@ export async function updateWithSegments(
   });
 }
 
+type UpdateSegmentImageUrlParameters = {
+  id: string;
+  imageUrl: string;
+};
+
+export async function updateSegmentImageUrl({
+  id,
+  imageUrl,
+}: UpdateSegmentImageUrlParameters) {
+  return await prisma.segment.update({
+    where: { id },
+    data: { imageUrl, updatedAt: getUnixTimeStamp() },
+  });
+}
+
 export async function getSegmentsBy(
   videoId: string,
   { search }: GetVideoParams