feat: loom

pipes/loom/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": ["next/core-web-vitals"]
+}

pipes/loom/README.md

@@ -0,0 +1,10 @@
+this pipes uses screenpipe api to merge chunks of videos in to a single loom type video and you can use any LLM to create summary of it!
+
+<img width="1312" alt="Screenshot 2024-11-29 at 12 23 59 PM" src="https://github.com/user-attachments/assets/6f7ab5a2-f791-4d9c-928b-45c594a026cb">
+
+<img width="1312" alt="Screenshot 2024-11-29 at 12 23 59 PM" src="https://github.com/user-attachments/assets/28d64b33-dcc8-4682-bf2e-019dd2795eab">
+
+<img width="1312" alt="Screenshot 2024-11-29 at 12 23 59 PM" src="https://github.com/user-attachments/assets/fd509cb8-f9ed-4829-87f6-37cc47651deb">
+
+<img width="1312" alt="Screenshot 2024-11-29 at 12 23 59 PM" src="https://github.com/user-attachments/assets/129bbf30-2446-49e1-a321-c930cdfe1eee">
+

pipes/loom/app/api/copy/route.ts

@@ -0,0 +1,42 @@
+import os from 'os'
+import { constants } from 'fs';
+import { exec } from 'child_process';
+import { access } from 'fs/promises';
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function GET(req: NextRequest): Promise<NextResponse> {
+  const path = req.nextUrl.searchParams.get('path');
+  if (!path || typeof path !== 'string') {
+    return NextResponse.json({ error: 'path is required' }, { status: 400 });
+  }
+
+  try {
+    await access(path, constants.F_OK);
+  } catch (error) {
+    return NextResponse.json({ error: 'File does not exist' }, { status: 404 });
+  }
+
+  let command: string;
+  if (os.platform() === 'win32') {
+    command = `powershell.exe -NoProfile -WindowStyle hidden -File copyToClipboard.ps1 -FilePath "${path}"`;
+  } else if (os.platform() === 'darwin') {
+    command = `osascript -e 'set the clipboard to (read (POSIX file "${path}") as JPEG picture)'`;
+  } else {
+    return NextResponse.json({ error: 'Unsupported operating system' }, { status: 400 });
+  }
+
+
+  return new Promise((resolve) => {
+    exec(command, (error, stdout, stderr) => {
+      if(error) {
+        console.error(`Error: ${error.message}`);
+        resolve(NextResponse.json({ error: `${error.message}` }, { status: 400 }));
+      } else if(stderr) {
+        console.error(`Stderr: ${stderr}`);
+        resolve(NextResponse.json({ error: `${stderr}` }, { status: 500 }));
+      } else {
+        resolve(NextResponse.json({ message: 'successfully copied to clipboard', stdout: stdout }, { status: 200 }));
+      }
+    })
+  })
+}

pipes/loom/app/api/file/route.ts

@@ -0,0 +1,74 @@
+import fs from 'fs';
+import path from 'path';
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function GET(req: NextRequest) {
+  const videoPath = req.nextUrl.searchParams.get('path');
+  if (!videoPath || typeof videoPath !== 'string') {
+    return NextResponse.json({ error: 'file path is required' }, { status: 400 });
+  }
+
+  try {
+    const fullPath = path.resolve(videoPath);
+    console.log(`Attempting to access file: ${fullPath}`);
+
+    if (!fs.existsSync(fullPath)) {
+      console.error('Error: File not found');
+      return NextResponse.json({error: 'File not found'}, {status: 404});
+    }
+
+    const fileStream = fs.createReadStream(fullPath);
+    const contentType = getMimeType(fullPath);
+
+    const headers = new Headers();
+    headers.set('Content-Type', contentType);
+    headers.set('Accept-Ranges', 'bytes');
+
+    let controllerClosed = false;
+    const readableStream = new ReadableStream({
+      start(controller) {
+        fileStream.on('data', (chunk) => {
+          if(!controllerClosed) {
+            controller.enqueue(chunk);
+          }
+        });
+        fileStream.on('end', () => {
+          if (!controllerClosed) {
+            controller.close();
+            controllerClosed = true;
+          }
+        });
+        fileStream.on('error', (err) => {
+          if (!controllerClosed) {
+            controller.error(err);
+            controllerClosed = true;
+          }
+        });
+      },
+      cancel() {
+        controllerClosed = true;
+        fileStream.destroy();
+      }
+    })
+
+    return new NextResponse(readableStream, { headers });
+  } catch (error :any) {
+    console.error('Unexpected error:', error);
+    return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
+  }
+}
+
+const getMimeType = (path: string): string => {
+  const ext = path.split('.').pop()?.toLowerCase()
+  const isAudio = path.toLowerCase().includes('input') 
+    || path.toLowerCase().includes('output')
+  switch (ext) {
+    case 'mp4': return 'video/mp4'
+    case 'webm': return 'video/webm'
+    case 'ogg': return 'video/ogg'
+    case 'mp3': return 'audio/mpeg'
+    case 'wav': return 'audio/wav'
+    default: return isAudio ? 'audio/mpeg' : 'video/mp4'
+  }
+}
+

pipes/loom/app/api/settings/route.ts

@@ -0,0 +1,109 @@
+import path from "path";
+import { promises as fs } from "fs";
+import { pipe } from "@screenpipe/js";
+import { NextResponse } from "next/server";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function GET() {
+  try {
+    const settingsManager = pipe.settings;
+    if (!settingsManager) {
+      throw new Error("settingsManager not found");
+    }
+
+    const screenpipeDir = process.env.SCREENPIPE_DIR || process.cwd();
+    const settingsPath = path.join(
+      screenpipeDir,
+      "pipes",
+      "loom",
+      "pipe.json"
+    );
+
+    try {
+      const settingsContent = await fs.readFile(settingsPath, "utf8");
+      const persistedSettings = JSON.parse(settingsContent);
+
+      // Merge with current settings
+      const rawSettings = await settingsManager.getAll();
+      return NextResponse.json({
+        ...rawSettings,
+        customSettings: {
+          ...rawSettings.customSettings,
+          loom: {
+            ...(rawSettings.customSettings?.loom || {}),
+            ...persistedSettings,
+          },
+        },
+      });
+    } catch (err) {
+      // If no persisted settings, return normal settings
+      const rawSettings = await settingsManager.getAll();
+      return NextResponse.json(rawSettings);
+    }
+  } catch (error) {
+    console.error("failed to get settings:", error);
+    return NextResponse.json(
+      { error: "failed to get settings" },
+      { status: 500 }
+    );
+  }
+}
+
+export async function PUT(request: Request) {
+  try {
+    const settingsManager = pipe.settings;
+    if (!settingsManager) {
+      throw new Error("settingsManager not found");
+    }
+
+    const body = await request.json();
+    const { key, value, isPartialUpdate, reset, namespace } = body;
+
+    if (reset) {
+      if (namespace) {
+        if (key) {
+          await settingsManager.setCustomSetting(namespace, key, undefined);
+        } else {
+          await settingsManager.updateNamespaceSettings(namespace, {});
+        }
+      } else {
+        if (key) {
+          await settingsManager.resetKey(key);
+        } else {
+          await settingsManager.reset();
+        }
+      }
+      return NextResponse.json({ success: true });
+    }
+
+    if (namespace) {
+      if (isPartialUpdate) {
+        const currentSettings =
+          (await settingsManager.getNamespaceSettings(namespace)) || {};
+        await settingsManager.updateNamespaceSettings(namespace, {
+          ...currentSettings,
+          ...value,
+        });
+      } else {
+        await settingsManager.setCustomSetting(namespace, key, value);
+      }
+    } else if (isPartialUpdate) {
+      const serializedSettings = JSON.parse(JSON.stringify(value));
+      await settingsManager.update(serializedSettings);
+    } else {
+      const serializedValue = JSON.parse(JSON.stringify(value));
+      await settingsManager.set(key, serializedValue);
+    }
+
+    return NextResponse.json({ success: true });
+  } catch (error) {
+    console.error("failed to update settings:", error);
+    return NextResponse.json(
+      { error: "failed to update settings" },
+      { status: 500 }
+    );
+  }
+}
+

pipes/loom/app/api/stream/route.ts

@@ -0,0 +1,86 @@
+import { Message } from 'ai';
+import { OpenAI } from 'openai';
+import { ContentItem } from "@screenpipe/js";
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function POST(req: NextRequest) {
+  const { settings, chatMessages, floatingInput, selectedAgent, data } = await req.json();
+  const MAX_CONTENT_LENGTH = settings.aiMaxContextChars || 6000;
+
+  // had to trim down the context when the loom video is tooooo long :(
+  const removeDuplicateLines = (textContent: string[])  => {
+    const uniqueLines = Array.from(new Set(textContent));
+    const rmdups = uniqueLines.map((i) => i.replace(/(\S+)(\s+\1)+/g, "$1"))
+    let context = rmdups.join('\n');
+
+    if (context.length > MAX_CONTENT_LENGTH) {
+      context = context.slice(0, 6000);
+    }
+
+    return context;
+  };
+
+  const ocrTexts = data.map((item: ContentItem) => {
+    if(item.type === "OCR"){
+      return item.content.text;
+    }
+  });
+
+  const context = removeDuplicateLines(ocrTexts);
+
+  try {
+    const openai = new OpenAI({
+      apiKey: settings.aiProviderType === 'screenpipe-cloud'
+        ? settings.user.token : settings.openaiApiKey,
+      baseURL: settings.aiUrl,
+    });
+
+    const model = settings.aiModel;
+    const customPrompt = settings.customPrompt || '';
+    const customRules = settings.loom?.customRules || '';
+
+    const messages = [
+      {
+        role: 'user' as const,
+        content: `You are a helpful assistant specialized as a "${selectedAgent.name}". ${selectedAgent.systemPrompt}
+          Rules:
+          - Current time (JavaScript Date.prototype.toString): ${new Date().toString()}
+          - User timezone: ${Intl.DateTimeFormat().resolvedOptions().timeZone}
+          - User timezone offset: ${new Date().getTimezoneOffset()}
+          - ${customPrompt ? `Prompt: ${customPrompt}` : ''}
+          - Rules: ${customRules} `,
+      },
+      ...chatMessages.map((msg: Message) => ({
+        role: msg.role as 'user' | 'assistant' | 'system',
+        content: msg.content,
+      })),
+      {
+        role: 'user' as const,
+        content: `Context data: ${context}
+        User query: ${floatingInput}`,
+      },
+    ];
+
+    const stream = await openai.chat.completions.create(
+      {
+        model: model,
+        messages: messages,
+        stream: true,
+      },
+    );
+
+    let fullResponse = '';
+    for await (const chunk of stream) {
+      const content = chunk.choices[0]?.delta?.content || '';
+      fullResponse += content;
+    }
+
+    return NextResponse.json({ response: fullResponse }, { status: 200 });
+  } catch (error: any) {
+    console.error('Error generating AI response:', error);
+    return NextResponse.json(
+      { message: "Failed to generate AI response" },
+      { status: 500, statusText: `${error}`} 
+    );
+  }
+}