From 470c7f115014f136095b327cff11b1c72415de21 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 5 Dec 2024 07:13:00 -0800 Subject: [PATCH 01/88] f --- App_Function_Libraries/STT/STT.py | 17 + App_Function_Libraries/STT/__init__.py | 0 App_Function_Libraries/TTS/TTS.py | 429 ++++++++++++++++++++ App_Function_Libraries/TTS/TTS_Providers.py | 173 ++++++++ App_Function_Libraries/TTS/__init__.py | 0 Docs/Design/RSS_Ranking.md | 8 +- Docs/Design/TTS_STT.md | 2 + Docs/Design/UX.md | 8 + Docs/Design/VLMs.md | 2 +- Docs/Handy_Dandy_Papers.md | 6 +- Docs/Issues/Evaluation_Plans.md | 9 +- Docs/RAG_Notes.md | 12 + Docs/TTS_STT.md | 13 + Docs/VLM.md | 7 + summarize.py | 2 +- 15 files changed, 683 insertions(+), 5 deletions(-) create mode 100644 App_Function_Libraries/STT/STT.py create mode 100644 App_Function_Libraries/STT/__init__.py create mode 100644 App_Function_Libraries/TTS/TTS.py create mode 100644 App_Function_Libraries/TTS/TTS_Providers.py create mode 100644 App_Function_Libraries/TTS/__init__.py diff --git a/App_Function_Libraries/STT/STT.py b/App_Function_Libraries/STT/STT.py new file mode 100644 index 000000000..8e9901150 --- /dev/null +++ b/App_Function_Libraries/STT/STT.py @@ -0,0 +1,17 @@ +# STT.py +# Description: This file contains the functions to convert speech to text +# +# Imports +# +# External Imports +# +# Local Imports +# +# +####################################################################################################################### +# +# Functions: + +# +# End of STT.py +####################################################################################################################### diff --git a/App_Function_Libraries/STT/__init__.py b/App_Function_Libraries/STT/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/App_Function_Libraries/TTS/TTS.py b/App_Function_Libraries/TTS/TTS.py new file mode 100644 index 000000000..87b43bb43 --- /dev/null +++ b/App_Function_Libraries/TTS/TTS.py @@ -0,0 +1,429 @@ +# TTS.py +# Description: This file contains the functions to convert text-to-speech +# +# Imports + +import os +from pydub import AudioSegment +import re +import tempfile +from typing import List, Tuple +# External Imports +# +# Local Imports +# +####################################################################################################################### +# +# Functions: + + + +####################################################### +# +# Text-to-Speak Functions + +import logging +from typing import Callable, Dict, Any, List, Tuple + +from App_Function_Libraries.TTS.TTS_Providers import generate_audio_openai, generate_audio_edge, \ + generate_audio_elevenlabs, tts_providers + +# Setup logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Type aliases +AudioData = bytes +ProviderFunction = Callable[[str, str, str], AudioData] + +# Provider functions +# See TTS_Providers.py + +# Provider registry +PROVIDERS: Dict[str, ProviderFunction] = { + "elevenlabs": generate_audio_elevenlabs, + "openai": generate_audio_openai, + "edge": generate_audio_edge, +} + + +# Utility functions +def clean_text(text: str) -> str: + # Implement text cleaning logic + return text + + +def split_qa(text: str) -> List[Tuple[str, str]]: + # Implement Q&A splitting logic + return [] + + +def merge_audio(audio_segments: List[AudioData]) -> AudioData: + # Implement audio merging logic + return b"" + + +# Main TTS function +def generate_tts_podcast( + text: str, + provider: str, + voice: str, + model: str, + config: Dict[str, Any] +) -> AudioData: + if provider not in PROVIDERS: + raise ValueError(f"Unsupported provider: {provider}") + + tts_func = PROVIDERS[provider] + + cleaned_text = clean_text(text) + qa_pairs = split_qa(cleaned_text) + + audio_segments = [] + for question, answer in qa_pairs: + q_audio = tts_func(question, config["question_voice"], model) + a_audio = tts_func(answer, config["answer_voice"], model) + audio_segments.extend([q_audio, a_audio]) + + return merge_audio(audio_segments) + + +# Configuration management +def load_tts_config() -> Dict[str, Any]: + # Implement configuration loading logic + return {} + + +# API Key management +def set_api_key(provider: str, api_key: str) -> None: + # Implement API key setting logic + pass + +tts_config = load_tts_config() + +def tts_generate_audio(input_text, provider, voice, model): + input_text = "Hello, how are you? I'm doing well, thank you!" + if provider not in tts_providers: + raise ValueError(f"Unsupported provider: {provider}") + elif provider == "elevenlabs": + result = generate_audio_elevenlabs(input_text, voice, model) + elif provider == "openai": + result = generate_audio_openai(input_text, voice, model) + elif provider == "edge": + result = generate_audio_edge(input_text, voice, model) + else: + raise ValueError(f"No provider found / Unsupported provider: {provider}") + # Save or process the result as needed + with open("output.mp3", "wb") as f: + f.write(result) + return result + +# +# End of Text-to-Speak Functions +####################################################### + + +####################################################### +# +# Podcast Creation Functions + + +# +# End of TTS.py +####################################################################################################################### + + +class TextToSpeech: + def __init__( + self, + model: str = None, + api_key: Optional[str] = None, + conversation_config: Optional[Dict[str, Any]] = None, + ): + """ + Initialize the TextToSpeech class. + + Args: + model (str): The model to use for text-to-speech conversion. + Options are 'elevenlabs', 'gemini', 'openai', 'edge' or 'geminimulti'. Defaults to 'openai'. + api_key (Optional[str]): API key for the selected text-to-speech service. + conversation_config (Optional[Dict]): Configuration for conversation settings. + """ + self.config = load_config() + self.conversation_config = load_conversation_config(conversation_config) + self.tts_config = self.conversation_config.get("text_to_speech", {}) + + # Get API key from config if not provided + if not api_key: + api_key = getattr(self.config, f"{model.upper().replace('MULTI', '')}_API_KEY", None) + + # Initialize provider using factory + self.provider = TTSProviderFactory.create( + provider_name=model, api_key=api_key, model=model + ) + + # Setup directories and config + self._setup_directories() + self.audio_format = self.tts_config.get("audio_format", "mp3") + self.ending_message = self.tts_config.get("ending_message", "") + + def _get_provider_config(self) -> Dict[str, Any]: + """Get provider-specific configuration.""" + # Get provider name in lowercase without 'TTS' suffix + provider_name = self.provider.__class__.__name__.lower().replace("tts", "") + + # Get provider config from tts_config + provider_config = self.tts_config.get(provider_name, {}) + + # If provider config is empty, try getting from default config + if not provider_config: + provider_config = { + "model": self.tts_config.get("default_model"), + "default_voices": { + "question": self.tts_config.get("default_voice_question"), + "answer": self.tts_config.get("default_voice_answer"), + }, + } + + logger.debug(f"Using provider config: {provider_config}") + return provider_config + + def convert_to_speech(self, text: str, output_file: str) -> None: + """ + Convert input text to speech and save as an audio file. + + Args: + text (str): Input text to convert to speech. + output_file (str): Path to save the output audio file. + + Raises: + ValueError: If the input text is not properly formatted + """ + # Validate transcript format + # self._validate_transcript_format(text) + + cleaned_text = text + + try: + + if ( + "multi" in self.provider.model.lower() + ): # refactor: We should have instead MultiSpeakerTTS and SingleSpeakerTTS classes + provider_config = self._get_provider_config() + voice = provider_config.get("default_voices", {}).get("question") + voice2 = provider_config.get("default_voices", {}).get("answer") + model = provider_config.get("model") + audio_data_list = self.provider.generate_audio( + cleaned_text, + voice="S", + model="en-US-Studio-MultiSpeaker", + voice2="R", + ending_message=self.ending_message, + ) + + try: + # First verify we have data + if not audio_data_list: + raise ValueError("No audio data chunks provided") + + logger.info(f"Starting audio processing with {len(audio_data_list)} chunks") + combined = AudioSegment.empty() + + for i, chunk in enumerate(audio_data_list): + # Save chunk to temporary file + # temp_file = "./tmp.mp3" + # with open(temp_file, "wb") as f: + # f.write(chunk) + + segment = AudioSegment.from_file(io.BytesIO(chunk)) + logger.info(f"################### Loaded chunk {i}, duration: {len(segment)}ms") + + combined += segment + + # Export with high quality settings + os.makedirs(os.path.dirname(output_file), exist_ok=True) + combined.export( + output_file, + format=self.audio_format, + codec="libmp3lame", + bitrate="320k" + ) + + except Exception as e: + logger.error(f"Error during audio processing: {str(e)}") + raise + else: + with tempfile.TemporaryDirectory(dir=self.temp_audio_dir) as temp_dir: + audio_segments = self._generate_audio_segments( + cleaned_text, temp_dir + ) + self._merge_audio_files(audio_segments, output_file) + logger.info(f"Audio saved to {output_file}") + + except Exception as e: + logger.error(f"Error converting text to speech: {str(e)}") + raise + + def _generate_audio_segments(self, text: str, temp_dir: str) -> List[str]: + """Generate audio segments for each Q&A pair.""" + qa_pairs = self.provider.split_qa( + text, self.ending_message, self.provider.get_supported_tags() + ) + audio_files = [] + provider_config = self._get_provider_config() + + for idx, (question, answer) in enumerate(qa_pairs, 1): + for speaker_type, content in [("question", question), ("answer", answer)]: + temp_file = os.path.join( + temp_dir, f"{idx}_{speaker_type}.{self.audio_format}" + ) + voice = provider_config.get("default_voices", {}).get(speaker_type) + model = provider_config.get("model") + + audio_data = self.provider.generate_audio(content, voice, model) + with open(temp_file, "wb") as f: + f.write(audio_data) + audio_files.append(temp_file) + + return audio_files + + def _merge_audio_files(self, audio_files: List[str], output_file: str) -> None: + """ + Merge the provided audio files sequentially, ensuring questions come before answers. + + Args: + audio_files: List of paths to audio files to merge + output_file: Path to save the merged audio file + """ + try: + + def get_sort_key(file_path: str) -> Tuple[int, int]: + """ + Create sort key from filename that puts questions before answers. + Example filenames: "1_question.mp3", "1_answer.mp3" + """ + basename = os.path.basename(file_path) + # Extract the index number and type (question/answer) + idx = int(basename.split("_")[0]) + is_answer = basename.split("_")[1].startswith("answer") + return ( + idx, + 1 if is_answer else 0, + ) # Questions (0) come before answers (1) + + # Sort files by index and type (question/answer) + audio_files.sort(key=get_sort_key) + + # Create empty audio segment + combined = AudioSegment.empty() + + # Add each audio file to the combined segment + for file_path in audio_files: + combined += AudioSegment.from_file(file_path, format=self.audio_format) + + # Ensure output directory exists + os.makedirs(os.path.dirname(output_file), exist_ok=True) + + # Export the combined audio + combined.export(output_file, format=self.audio_format) + logger.info(f"Merged audio saved to {output_file}") + + except Exception as e: + logger.error(f"Error merging audio files: {str(e)}") + raise + + def _setup_directories(self) -> None: + """Setup required directories for audio processing.""" + self.output_directories = self.tts_config.get("output_directories", {}) + temp_dir = self.tts_config.get("temp_audio_dir", "data/audio/tmp/").rstrip("/").split("/") + self.temp_audio_dir = os.path.join(*temp_dir) + base_dir = os.path.abspath(os.path.dirname(__file__)) + self.temp_audio_dir = os.path.join(base_dir, self.temp_audio_dir) + + os.makedirs(self.temp_audio_dir, exist_ok=True) + + # Create directories if they don't exist + for dir_path in [ + self.output_directories.get("transcripts"), + self.output_directories.get("audio"), + self.temp_audio_dir, + ]: + if dir_path and not os.path.exists(dir_path): + os.makedirs(dir_path) + + def _validate_transcript_format(self, text: str) -> None: + """ + Validate that the input text follows the correct transcript format. + + Args: + text (str): Input text to validate + + Raises: + ValueError: If the text is not properly formatted + + The text should: + 1. Have alternating Person1 and Person2 tags + 2. Each opening tag should have a closing tag + 3. Tags should be properly nested + """ + try: + # Check for empty text + if not text.strip(): + raise ValueError("Input text is empty") + + # Check for matching opening and closing tags + person1_open = text.count("") + person1_close = text.count("") + person2_open = text.count("") + person2_close = text.count("") + + if person1_open != person1_close: + raise ValueError( + f"Mismatched Person1 tags: {person1_open} opening tags and {person1_close} closing tags" + ) + if person2_open != person2_close: + raise ValueError( + f"Mismatched Person2 tags: {person2_open} opening tags and {person2_close} closing tags" + ) + + # Check for alternating pattern using regex + pattern = r".*?\s*.*?" + matches = re.findall(pattern, text, re.DOTALL) + + # Calculate expected number of pairs + expected_pairs = min(person1_open, person2_open) + + if len(matches) != expected_pairs: + raise ValueError( + "Tags are not properly alternating between Person1 and Person2. " + "Each Person1 section should be followed by a Person2 section." + ) + + # Check for malformed tags (unclosed or improperly nested) + stack = [] + for match in re.finditer(r"<(/?)Person([12])>", text): + tag = match.group(0) + if tag.startswith(" bytes: + """Generate audio using Edge TTS.""" + import nest_asyncio + import asyncio + + # Apply nest_asyncio to allow nested event loops + nest_asyncio.apply() + + async def _generate(): + communicate = edge_tts.Communicate(text, voice) + # Create a temporary file with proper context management + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as tmp_file: + temp_path = tmp_file.name + + try: + # Save audio to temporary file + await communicate.save(temp_path) + # Read the audio data + with open(temp_path, 'rb') as f: + return f.read() + finally: + # Clean up temporary file + if os.path.exists(temp_path): + os.remove(temp_path) + + # Use nest_asyncio to handle nested event loops + loop = asyncio.get_event_loop() + return loop.run_until_complete(_generate()) + result = generate_audio(text, voice, model, voice2) + return result + +# +# End of MS Edge TTS Provider Functions +####################################################### + + +####################################################### +# +# ElvenLabs TTS Provider Functions + +def generate_audio_elevenlabs(text, voice, model, voice2=None): + """Generate audio using ElevenLabs API.""" + # FIXME - add check for config settings + # FIXME - add check + load for API key + # FIXME - add check + load for model, voice, input values + # FIXME - add SSML tags + + elvenlabs_ssml_tags = ['lang', 'p', 'phoneme', 's', 'sub'] + + # FIXME - IDE says '.generate' doesn't exist? + audio = elevenlabs_client.generate( + text=text, + voice=voice, + model=model + ) + return b''.join(chunk for chunk in audio if chunk) + +# End of ElvenLabs TTS Provider Functions +####################################################### + + +####################################################### +# +# Google Gemini TTS Provider Functions + + +# +# End of Google Gemini TTS Provider Functions +####################################################### + +# +# End of TTS_Providers.py +####################################################################################################################### diff --git a/App_Function_Libraries/TTS/__init__.py b/App_Function_Libraries/TTS/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Docs/Design/RSS_Ranking.md b/Docs/Design/RSS_Ranking.md index 3fb8500ce..13706fb72 100644 --- a/Docs/Design/RSS_Ranking.md +++ b/Docs/Design/RSS_Ranking.md @@ -3,9 +3,15 @@ ## Introduction + +RSS + https://feedparser.readthedocs.io/en/latest/ + https://github.com/kurtmckee/feedparser + + https://towardsdatascience.com/a-mixed-methods-approach-to-offline-evaluation-of-news-recommender-systems-7dc7e9f0b501 https://towardsdatascience.com/making-news-recommendations-explainable-with-large-language-models-74f119c7e036 - +https://blog.det.life/from-scrolls-to-similarity-search-building-a-movie-recommender-with-duckdb-vss-8122e4d2e486?gi=c094d02d0993 https://www.dogesec.com/blog/full_text_rss_atom_blog_feeds/ https://arxiv.org/abs/2411.19352 diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 19b3dbdab..f46caf9fb 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -6,6 +6,8 @@ Models https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF +https://github.com/albirrkarim/react-speech-highlight-demo + GPT-SoviTTS https://levelup.gitconnected.com/great-api-design-comprehensive-guide-from-basics-to-best-practices-9b4e0b613a44?source=home---------56-1--------------------0fc48da7_5ce6_48ca_92d2_260680a20318-------3 diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 573e0fd0f..2a2d1beec 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -48,4 +48,12 @@ https://copycoder.ai/ https://docs.replit.com/replitai/agent https://bolt.new/ https://github.com/rmusser01/agentic_rag/tree/main +https://github.com/astramind-ai/Pulsar +https://github.com/woshixiaobai2019/mirau-chat-ui +https://github.com/Vali-98/ChatterUI +https://github.com/Rugz007/liha +https://astro.new/latest/ +https://github.com/lobehub/lobe-chat + + diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 9926f9e6a..0cfbd56f8 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -7,6 +7,6 @@ https://arxiv.org/abs/2411.18279 https://github.com/breezedeus/Pix2Text https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct - +https://huggingface.co/tencent/HunyuanVideo https://ivy-lvlm.github.io/Video-MA2MBA/ \ No newline at end of file diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 77316301b..34e4b2ad8 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -4,7 +4,11 @@ https://arxiv.org/pdf/2410.13098 https://arxiv.org/pdf/2404.01413 https://arxiv.org/abs/2411.11910 - +https://arxiv.org/abs/2411.00136 +https://jinyangwu.github.io/hiar-icl/ +https://www.aimodels.fyi/papers/arxiv/beyond-examples-high-level-automated-reasoning-paradigm +https://arxiv.org/abs/2412.02674 +https://arxiv.org/abs/2411.19865 ### Context https://arxiv.org/pdf/2304.12102 diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 3cce16710..e4bc7fd1e 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -31,6 +31,7 @@ General Research Chat arena https://github.com/lm-sys/FastChat LLM-as-judge + https://www.atla-ai.com/post/evaluating-the-evaluator https://huggingface.co/spaces/AtlaAI/judge-arena https://huggingface.co/learn/cookbook/en/llm_judge https://github.com/open-compass/CompassJudger @@ -202,6 +203,7 @@ Finetuning - https://arxiv.org/abs/2411.06032 - https://arxiv.org/abs/2410.02677 - https://mbzuai-oryx.github.io/ALM-Bench/ + - https://arxiv.org/abs/2411.19799 - User-Centric Evaluation of LLMs - https://github.com/Alice1998/URS - https://huggingface.co/spaces/HuggingFaceFW/blogpost-fine-tasks @@ -424,8 +426,13 @@ https://github.com/jonathan-roberts1/needle-threading/ https://huggingface.co/datasets/jonathan-roberts1/needle-threading https://arxiv.org/abs/2411.03538 https://arxiv.org/abs/2411.19710 - +https://aws.amazon.com/blogs/aws/new-rag-evaluation-and-llm-as-a-judge-capabilities-in-amazon-bedrock/ https://archive.is/MZsB9 +https://arxiv.org/abs/2411.00136 +https://github.com/opendatalab/OHR-Bench +https://towardsdatascience.com/from-retrieval-to-intelligence-exploring-rag-agent-rag-and-evaluation-with-trulens-3c518af836ce + + - **101** - **RAG Eval Plan:** - The generic idea however: you take a (full, unchunked) document and ask an LLM to generate a question with that document as well as give the factual answer to it. Enforce via prompts to make it use the document only and make it as hard as you want (eg. maybe sometimes you want it to consider 2 documents and make a question that uses bits of both). This gives you a ground truth dataset. diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 9510afea6..72723a0f7 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -26,6 +26,18 @@ GraphRAG https://towardsdatascience.com/building-a-knowledge-graph-from-scratch-using-llms-f6f677a17f07 https://iopscience.iop.org/article/10.1088/2632-2153/ad7228/pdf https://medium.com/thoughts-on-machine-learning/building-dynamic-knowledge-graphs-using-open-source-llms-06a870e1bc4f + https://ai.plainenglish.io/modeling-ai-semantic-memory-with-knowledge-graphs-1ce06f683433 + https://towardsdatascience.com/how-to-convert-any-text-into-a-graph-of-concepts-110844f22a1a + + + + + + + + + + ### Links - RAG 101 diff --git a/Docs/TTS_STT.md b/Docs/TTS_STT.md index 674d4ef91..94cafe10c 100644 --- a/Docs/TTS_STT.md +++ b/Docs/TTS_STT.md @@ -3,6 +3,19 @@ ## Overview +https://github.com/jiaaro/pydub + +Flow: +1. Clean/format input text +2. Split text into segments +3. Generate audio for each segment using designated provider function +4. Merge audio segments into single output file +5. Clean up temporary files + + + + +### Link Dump: https://github.com/cpumaxx/sovits-ff-plugin diff --git a/Docs/VLM.md b/Docs/VLM.md index 601fca1b1..d7e625b69 100644 --- a/Docs/VLM.md +++ b/Docs/VLM.md @@ -3,5 +3,12 @@ https://github.com/matatonic/openedai-vision +https://github.com/Tencent/HunyuanVideo +https://github.com/huggingface/blog/blob/main/smolvlm.md +https://huggingface.co/tencent/HunyuanVideo +https://aivideo.hunyuan.tencent.com/ + + + diff --git a/summarize.py b/summarize.py index 1941ad8f9..b2d742e0c 100644 --- a/summarize.py +++ b/summarize.py @@ -697,7 +697,7 @@ def main(input_path, api_name=None, api_key=None, if media_path.lower().endswith(('.txt', '.md')): if media_path.lower().endswith('.txt'): # Handle text file ingestion - result = ingest_text_file(media_path) + result = ingest_text_file logging.info(result) elif media_path.lower().endswith(('.mp4', '.avi', '.mov')): if diarize: From 5586cc6f90587f5ae0197a6e73b9312f32f608dd Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 7 Dec 2024 11:42:45 -0800 Subject: [PATCH 02/88] LINKS --- Docs/Design/Coding_Page.md | 9 +++++++++ Docs/Design/DB_Design.md | 1 + Docs/Design/Storytelling_and_Creation.md | 10 ++++++++++ Docs/Design/TTS_STT.md | 15 +++++++++++++-- Docs/Design/VLMs.md | 8 +++++++- Docs/Handy_Dandy_Papers.md | 6 ++++++ Docs/Issues/Evaluation_Plans.md | 9 ++++++++- Docs/RAG_Notes.md | 16 ++++++++++++---- Docs/TTS_STT.md | 8 +++++++- 9 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 Docs/Design/Storytelling_and_Creation.md diff --git a/Docs/Design/Coding_Page.md b/Docs/Design/Coding_Page.md index d8a0775a0..cf5a36a61 100644 --- a/Docs/Design/Coding_Page.md +++ b/Docs/Design/Coding_Page.md @@ -10,4 +10,13 @@ https://github.com/brandondocusen/CntxtPY https://github.com/cyclotruc/gitdigest https://github.com/simonw/files-to-prompt https://github.com/yamadashy/repomix/tree/main +https://github.com/chanhx/crabviz + + +https://pythontutor.com/c.html#mode=edit +https://pythontutor.com/articles/c-cpp-visualizer.html + + + + diff --git a/Docs/Design/DB_Design.md b/Docs/Design/DB_Design.md index 9d171a981..8a4c714f1 100644 --- a/Docs/Design/DB_Design.md +++ b/Docs/Design/DB_Design.md @@ -14,6 +14,7 @@ Migrating to sqlite-vec https://docs.google.com/document/d/1sJ_S2ggfFmtPJupxIO3C1EZAFuDMUfNYcAytissbFMs/edit?tab=t.0#heading=h.xyau1jyb6vyx https://github.com/Mozilla-Ocho/llamafile/pull/644 +https://briandouglas.ie/sqlite-defaults/ https://kerkour.com/sqlite-for-servers https://wafris.org/blog/rearchitecting-for-sqlite General DB: diff --git a/Docs/Design/Storytelling_and_Creation.md b/Docs/Design/Storytelling_and_Creation.md new file mode 100644 index 000000000..e638195f0 --- /dev/null +++ b/Docs/Design/Storytelling_and_Creation.md @@ -0,0 +1,10 @@ +# Storytelling & Creation + + + +### Link Dump: +https://github.com/mhgolkar/Arrow + +### Storytelling + +### Creating Stories diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index f46caf9fb..b8aa7e5ea 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -1,10 +1,15 @@ # Text-To-Speech / Speech-To-Text Documentation -## Overview +### Overview +### Services +- Google Cloud Text-to-Speech + - https://cloud.google.com/text-to-speech/docs/ssml + + Models - https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF + https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF https://github.com/albirrkarim/react-speech-highlight-demo @@ -14,6 +19,12 @@ https://levelup.gitconnected.com/great-api-design-comprehensive-guide-from-basic https://rentry.org/GPT-SoVITS-guide https://github.com/RVC-Boss/GPT-SoVITS +MaskGCT + https://maskgct.github.io/#emotion-samples + https://github.com/open-mmlab/Amphion/blob/main/models/tts/maskgct/README.md + https://github.com/open-mmlab/Amphion/blob/main/models/tts/maskgct/maskgct_demo.ipynb + https://github.com/open-mmlab/Amphion/blob/main/models/tts/maskgct/maskgct_inference.py + https://huggingface.co/amphion/MaskGCT TTS Pipeline https://www.astramind.ai/post/auralis diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 0cfbd56f8..d12536380 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -8,5 +8,11 @@ https://arxiv.org/abs/2411.18279 https://github.com/breezedeus/Pix2Text https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct https://huggingface.co/tencent/HunyuanVideo +https://huggingface.co/OpenGVLab/InternVL2_5-38B +https://ivy-lvlm.github.io/Video-MA2MBA/ -https://ivy-lvlm.github.io/Video-MA2MBA/ \ No newline at end of file +https://huggingface.co/Qwen/Qwen2-VL-72B +https://huggingface.co/Qwen/Qwen2-VL-7B +https://huggingface.co/Qwen/Qwen2-VL-2B + +https://www.reddit.com/r/StableDiffusion/comments/1h7hunp/how_to_run_hunyuanvideo_on_a_single_24gb_vram_card/ diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 34e4b2ad8..27270eb77 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -9,6 +9,12 @@ https://jinyangwu.github.io/hiar-icl/ https://www.aimodels.fyi/papers/arxiv/beyond-examples-high-level-automated-reasoning-paradigm https://arxiv.org/abs/2412.02674 https://arxiv.org/abs/2411.19865 +https://arxiv.org/abs/2412.04315 +https://arxiv.org/abs/2412.02980 +https://arxiv.org/abs/2412.02906 +https://arxiv.org/pdf/2410.21272 +https://arxiv.org/abs/2412.04318 +https://arxiv.org/abs/2412.02142 ### Context https://arxiv.org/pdf/2304.12102 diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index e4bc7fd1e..edcff7a06 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -13,7 +13,8 @@ - [VLM Evaluations](#vlm-evals) ---------------------------------------------------------------------------------------------------------------- - +https://arxiv.org/abs/2412.03304 +https://arxiv.org/abs/2412.03605 https://x.com/bnjmn_marie/status/1846834917608407199 https://www.juriopitz.com/2024/10/17/evaluation-pitfalls-metric-overview-tips.html Benchmarking with distilabel @@ -30,6 +31,7 @@ General Research Chat arena https://github.com/lm-sys/FastChat + https://arxiv.org/abs/2412.04363 LLM-as-judge https://www.atla-ai.com/post/evaluating-the-evaluator https://huggingface.co/spaces/AtlaAI/judge-arena @@ -44,9 +46,13 @@ LLM-as-judge Quant Eval https://arxiv.org/abs/2411.02355 +Summarization + https://arxiv.org/abs/2412.04254 + Creating Datasets https://github.com/Kiln-AI/Kiln + https://arxiv.org/abs/2412.03679 https://github.com/argilla-io/argilla https://www.youtube.com/watch?v=ZsCqrAhzkFU https://www.youtube.com/watch?v=jWrtgf2w4VU @@ -200,6 +206,7 @@ Finetuning - https://eqbench.com/creative_writing.html - **Culture** - https://arxiv.org/html/2305.14328v2 + - https://huggingface.co/datasets/CohereForAI/Global-MMLU - https://arxiv.org/abs/2411.06032 - https://arxiv.org/abs/2410.02677 - https://mbzuai-oryx.github.io/ALM-Bench/ diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 72723a0f7..29d6107f8 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -16,6 +16,12 @@ Unsorted https://www.louisbouchard.ai/indexing-methods/ https://github.com/circlemind-ai/fast-graphrag https://github.com/Kain-90/RAG-Play + https://arxiv.org/abs/2412.03736 + https://arxiv.org/abs/2403.10131 + https://arxiv.org/abs/2412.01572 + https://pub.towardsai.net/around-the-rag-in-80-questions-part-ii-4df03c6dba86 + https://www.uber.com/en-IN/blog/introducing-the-prompt-engineering-toolkit/ + GraphRAG https://www.microsoft.com/en-us/research/blog/introducing-drift-search-combining-global-and-local-search-methods-to-improve-quality-and-efficiency/ @@ -28,10 +34,12 @@ GraphRAG https://medium.com/thoughts-on-machine-learning/building-dynamic-knowledge-graphs-using-open-source-llms-06a870e1bc4f https://ai.plainenglish.io/modeling-ai-semantic-memory-with-knowledge-graphs-1ce06f683433 https://towardsdatascience.com/how-to-convert-any-text-into-a-graph-of-concepts-110844f22a1a - - - - + https://towardsdatascience.com/building-knowledge-graphs-with-llm-graph-transformer-a91045c49b59 + https://arxiv.org/abs/2412.03589 + https://arxiv.org/abs/2412.03736 + https://arxiv.org/abs/2412.04119 + https://aibyhand.substack.com/p/beginners-guide-to-graph-rag + https://github.com/gusye1234/nano-graphrag diff --git a/Docs/TTS_STT.md b/Docs/TTS_STT.md index 94cafe10c..6a63302bb 100644 --- a/Docs/TTS_STT.md +++ b/Docs/TTS_STT.md @@ -1,6 +1,12 @@ # Text-To-Speech / Speech-To-Text Documentation ## Overview +Use of functions for individual services. +Function for each service, streaming & non-streaming. +Non-streaming will return a file, streaming will return a stream. +Use of temporary files for storage. +Use of pydub for audio manipulation. +Use of pydub for audio merging. https://github.com/jiaaro/pydub @@ -42,7 +48,7 @@ https://github.com/Picovoice/speech-to-text-benchmark https://huggingface.co/papers/2410.02678 https://github.com/livekit/agents - +https://github.com/pipecat-ai/pipecat/tree/a367a038f1a3967292b5de5b43b8600a82a73fb6?tab=readme-ov-file https://github.com/lamm-mit/PDF2Audio https://github.com/Purfview/whisper-standalone-win From b56ec96bdd76e89c83597cbba6e846b7dac3bc8b Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 7 Dec 2024 13:55:21 -0800 Subject: [PATCH 03/88] Update VLM.md --- Docs/VLM.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/VLM.md b/Docs/VLM.md index 601fca1b1..588373744 100644 --- a/Docs/VLM.md +++ b/Docs/VLM.md @@ -3,5 +3,5 @@ https://github.com/matatonic/openedai-vision - +https://github.com/breezedeus/Pix2Text From 5b4a87f3fab942ba7dd005afb9bf8a47412444e8 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 8 Dec 2024 21:57:05 -0800 Subject: [PATCH 04/88] f --- Docs/Design/Mindmaps_Diagrams.md | 5 +++ Docs/Issues/Evaluation_Plans.md | 75 ++++++++++++++++++++------------ 2 files changed, 52 insertions(+), 28 deletions(-) create mode 100644 Docs/Design/Mindmaps_Diagrams.md diff --git a/Docs/Design/Mindmaps_Diagrams.md b/Docs/Design/Mindmaps_Diagrams.md new file mode 100644 index 000000000..087b4bcdd --- /dev/null +++ b/Docs/Design/Mindmaps_Diagrams.md @@ -0,0 +1,5 @@ +# Mindmaps & Diagram Creation + + + +https://github.com/mermaid-js/mermaid \ No newline at end of file diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index edcff7a06..afc08ecb6 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -13,46 +13,51 @@ - [VLM Evaluations](#vlm-evals) ---------------------------------------------------------------------------------------------------------------- -https://arxiv.org/abs/2412.03304 -https://arxiv.org/abs/2412.03605 -https://x.com/bnjmn_marie/status/1846834917608407199 -https://www.juriopitz.com/2024/10/17/evaluation-pitfalls-metric-overview-tips.html + + +Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 + Benchmarking with distilabel https://distilabel.argilla.io/latest/sections/pipeline_samples/examples/benchmarking_with_distilabel/ General Research - https://arxiv.org/abs/2407.10457 - https://arxiv.org/abs/2410.01392 - https://arxiv.org/pdf/2411.09213 - https://arxiv.org/abs/2411.00640 - https://ai.gopubby.com/5-ml-techniques-to-boost-your-model-accuracy-without-adding-more-data-94413189aaad?source=collection_home---4------9----------------------- - https://arxiv.org/abs/2411.10541 - https://github.com/DanielWarfield1/MLWritingAndResearch/blob/main/AutoMix.ipynb - + Greedy Sampling + https://arxiv.org/abs/2407.10457 + * `Our study addresses this issue by exploring key questions about the performance differences between greedy decoding and sampling, identifying benchmarks' consistency regarding non-determinism, and examining unique model behaviors. Through extensive experiments, we observe that greedy decoding generally outperforms sampling methods for most evaluated tasks. We also observe consistent performance across different LLM sizes and alignment methods, noting that alignment can reduce sampling variance. Moreover, our best-of-N sampling approach demonstrates that smaller LLMs can match or surpass larger models such as GPT-4-Turbo, highlighting the untapped potential of smaller LLMs. ` + Stats + https://arxiv.org/pdf/2410.01392 + https://arxiv.org/abs/2411.00640 + Chat arena - https://github.com/lm-sys/FastChat - https://arxiv.org/abs/2412.04363 + Building one + https://github.com/lm-sys/FastChat + Potential issues with creating a chatarena system + https://arxiv.org/abs/2412.04363 LLM-as-judge - https://www.atla-ai.com/post/evaluating-the-evaluator - https://huggingface.co/spaces/AtlaAI/judge-arena - https://huggingface.co/learn/cookbook/en/llm_judge - https://github.com/open-compass/CompassJudger - https://hamel.dev/blog/posts/llm-judge - https://llm-as-a-judge.github.io/ - https://github.com/llm-as-a-judge/Awesome-LLM-as-a-judge - https://deepmind.google/research/publications/85420/ - https://arxiv.org/abs/2411.16646 + Basics + https://huggingface.co/learn/cookbook/en/llm_judge + Evaluating LLMs as Judges + https://huggingface.co/papers/2306.05685 + https://llm-as-a-judge.github.io/ + https://arxiv.org/abs/2411.16646 + Google SAFE + https://arxiv.org/abs/2403.18802 + https://github.com/google-deepmind/long-form-factuality + Ranking of + https://huggingface.co/spaces/AtlaAI/judge-arena + Tools + https://github.com/open-compass/CompassJudger Quant Eval -https://arxiv.org/abs/2411.02355 + https://arxiv.org/abs/2411.02355 Summarization - https://arxiv.org/abs/2412.04254 - + ClinicSum (Finetuning for Summarization) + https://arxiv.org/abs/2412.04254 Creating Datasets - https://github.com/Kiln-AI/Kiln - https://arxiv.org/abs/2412.03679 + + https://github.com/argilla-io/argilla https://www.youtube.com/watch?v=ZsCqrAhzkFU https://www.youtube.com/watch?v=jWrtgf2w4VU @@ -74,6 +79,10 @@ Finetuning - **101** https://hamel.dev/blog/posts/evals/ + +Links: + https://www.juriopitz.com/2024/10/17/evaluation-pitfalls-metric-overview-tips.html + ---------------------------------------------------------------------------------------------------------------- @@ -175,6 +184,8 @@ Finetuning - https://github.com/FSoft-AI4Code/CodeMMLU - StackUnseen - https://prollm.toqan.ai/leaderboard/stack-unseen +- **Cognitive Biases** + - CBEval: https://arxiv.org/abs/2412.03605 - **Confabulation-Rate** - https://arxiv.org/abs/2409.11353 - https://github.com/sylinrl/TruthfulQA @@ -206,6 +217,7 @@ Finetuning - https://eqbench.com/creative_writing.html - **Culture** - https://arxiv.org/html/2305.14328v2 + - https://arxiv.org/abs/2412.03304 - https://huggingface.co/datasets/CohereForAI/Global-MMLU - https://arxiv.org/abs/2411.06032 - https://arxiv.org/abs/2410.02677 @@ -221,6 +233,10 @@ Finetuning - **Math Eval** - https://arxiv.org/abs/2411.04872 - GSM8K +- **Prompt Formatting** + - How do LLMs handle different formats: + - https://arxiv.org/abs/2411.10541 + - `Our study reveals that the way prompts are formatted significantly impacts GPT-based models’ performance, with no single format excelling universally. This finding questions current evaluation methods that often ignore prompt structure, potentially misjudging a model’s true abilities. We advocate for diverse prompt formats in future LLM testing to accurately gauge and enhance their performance.` - **Positional Bias** - https://arxiv.org/abs/2410.14641 - https://github.com/Rachum-thu/LongPiBench @@ -438,6 +454,9 @@ https://archive.is/MZsB9 https://arxiv.org/abs/2411.00136 https://github.com/opendatalab/OHR-Bench https://towardsdatascience.com/from-retrieval-to-intelligence-exploring-rag-agent-rag-and-evaluation-with-trulens-3c518af836ce +https://arxiv.org/pdf/2411.09213 +https://huggingface.co/learn/cookbook/en/rag_evaluation + - **101** From 5cedc1004208b271af56f70498db8a4cfbcda1ec Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 8 Dec 2024 21:58:20 -0800 Subject: [PATCH 05/88] f --- Docs/Design/Creative_Writing.md | 15 +++++++++++++++ Docs/Design/Structured_Outputs.md | 1 + Docs/Design/UX.md | 1 + Docs/Design/Visual_Video_Analysis.md | 15 +++++++++++++++ Docs/RAG_Notes.md | 1 + 5 files changed, 33 insertions(+) create mode 100644 Docs/Design/Creative_Writing.md create mode 100644 Docs/Design/Visual_Video_Analysis.md diff --git a/Docs/Design/Creative_Writing.md b/Docs/Design/Creative_Writing.md new file mode 100644 index 000000000..0bdd46b48 --- /dev/null +++ b/Docs/Design/Creative_Writing.md @@ -0,0 +1,15 @@ +# Creative Writing + + + + + + +### Link Dump: +https://github.com/p-e-w/arrows + + + + + + diff --git a/Docs/Design/Structured_Outputs.md b/Docs/Design/Structured_Outputs.md index 9607ceaca..8cb7f9147 100644 --- a/Docs/Design/Structured_Outputs.md +++ b/Docs/Design/Structured_Outputs.md @@ -23,4 +23,5 @@ https://python.plainenglish.io/generating-perfectly-structured-json-using-llms-a https://docs.pydantic.dev/latest/ https://github.com/outlines-dev/outlines[ https://github.com/Dan-wanna-M/formatron/tree/master +https://blog.dottxt.co/selective-multiplication.html diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 2a2d1beec..487dfbfe9 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -54,6 +54,7 @@ https://github.com/Vali-98/ChatterUI https://github.com/Rugz007/liha https://astro.new/latest/ https://github.com/lobehub/lobe-chat +https://www.nngroup.com/videos/the-danger-of-defaults/ diff --git a/Docs/Design/Visual_Video_Analysis.md b/Docs/Design/Visual_Video_Analysis.md new file mode 100644 index 000000000..cd8059fab --- /dev/null +++ b/Docs/Design/Visual_Video_Analysis.md @@ -0,0 +1,15 @@ +# Visual Video Analysis + + +## Introduction + + +## Visual Video Analysis + + +### Link Dump: +https://github.com/SamurAIGPT/AI-Youtube-Shorts-Generator + + + + diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 29d6107f8..a32a3d7e0 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -21,6 +21,7 @@ Unsorted https://arxiv.org/abs/2412.01572 https://pub.towardsai.net/around-the-rag-in-80-questions-part-ii-4df03c6dba86 https://www.uber.com/en-IN/blog/introducing-the-prompt-engineering-toolkit/ + https://arxiv.org/abs/2409.14924 GraphRAG From 46d074ec1e738a2f20832b3d030ddf765fcc35e7 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 8 Dec 2024 22:12:10 -0800 Subject: [PATCH 06/88] f --- Docs/Design/DB_Design.md | 1 + Docs/Design/Education.md | 6 ++++++ Docs/Design/Privacy.md | 7 +++++++ Docs/Design/Text2SQL.md | 1 + Docs/Design/VLMs.md | 4 ++++ Docs/Handy_Dandy_Papers.md | 5 +++++ Docs/Issues/Citations_and_Confabulations.md | 4 ++++ Docs/RAG_Notes.md | 2 ++ 8 files changed, 30 insertions(+) create mode 100644 Docs/Design/Education.md create mode 100644 Docs/Design/Privacy.md diff --git a/Docs/Design/DB_Design.md b/Docs/Design/DB_Design.md index 8a4c714f1..f075bb8c3 100644 --- a/Docs/Design/DB_Design.md +++ b/Docs/Design/DB_Design.md @@ -15,6 +15,7 @@ Migrating to sqlite-vec https://github.com/Mozilla-Ocho/llamafile/pull/644 https://briandouglas.ie/sqlite-defaults/ +https://phiresky.github.io/blog/2020/sqlite-performance-tuning/ https://kerkour.com/sqlite-for-servers https://wafris.org/blog/rearchitecting-for-sqlite General DB: diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md new file mode 100644 index 000000000..da599dcf5 --- /dev/null +++ b/Docs/Design/Education.md @@ -0,0 +1,6 @@ +# Education + + +### Link Dump: +https://arxiv.org/abs/2412.02035 +https://github.com/andreamust/NEON-GPT \ No newline at end of file diff --git a/Docs/Design/Privacy.md b/Docs/Design/Privacy.md new file mode 100644 index 000000000..64814215d --- /dev/null +++ b/Docs/Design/Privacy.md @@ -0,0 +1,7 @@ +# Privacy-related + + +### Link Dump: +https://arxiv.org/abs/2412.03924 + + diff --git a/Docs/Design/Text2SQL.md b/Docs/Design/Text2SQL.md index 2919ac8f7..66213b240 100644 --- a/Docs/Design/Text2SQL.md +++ b/Docs/Design/Text2SQL.md @@ -17,3 +17,4 @@ https://ai.gopubby.com/advanced-rag-retrieval-strategy-embedded-tables-fdb3e4400 https://medium.com/intel-tech/tabular-data-rag-llms-improve-results-through-data-table-prompting-bcb42678914b https://github.com/TAG-Research/TAG-Bench https://arxiv.org/pdf/2407.14482 +https://spider2-sql.github.io/ diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index d12536380..6d9053236 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -16,3 +16,7 @@ https://huggingface.co/Qwen/Qwen2-VL-7B https://huggingface.co/Qwen/Qwen2-VL-2B https://www.reddit.com/r/StableDiffusion/comments/1h7hunp/how_to_run_hunyuanvideo_on_a_single_24gb_vram_card/ +https://arxiv.org/abs/2412.05185 + + + diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 27270eb77..d234cdfd8 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -20,6 +20,11 @@ https://arxiv.org/abs/2412.02142 https://arxiv.org/pdf/2304.12102 https://arxiv.org/html/2405.20234v2#S3 +### Dataset Generation + https://arxiv.org/abs/2412.04645 + + + ### Reasoning - https://huggingface.co/papers/2411.15862 diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index d2ecd5c7f..99acd5d2e 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -5,6 +5,10 @@ 2. [Confabulations](#confabulations) 3. [References](#references) + + +https://arxiv.org/abs/2412.04235 + RAG https://www.lycee.ai/blog/rag-ragallucinations-and-how-to-fight-them diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index a32a3d7e0..0691e2d5c 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -22,6 +22,8 @@ Unsorted https://pub.towardsai.net/around-the-rag-in-80-questions-part-ii-4df03c6dba86 https://www.uber.com/en-IN/blog/introducing-the-prompt-engineering-toolkit/ https://arxiv.org/abs/2409.14924 +https://arxiv.org/abs/2412.02830 +https://arxiv.org/abs/2407.04125 GraphRAG From 23d0f0c780301c1246b984b27acc143e7ac64e4a Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 11 Dec 2024 22:23:56 -0800 Subject: [PATCH 07/88] links --- Docs/Bitnet_and_Mamba_Papers.md | 2 +- Docs/Design/Storytelling_and_Creation.md | 4 + Docs/Design/TTS_STT.md | 108 ++++++++++++++++-- Docs/Design/UX.md | 3 + Docs/Design/VLMs.md | 3 + Docs/Handy_Dandy_Papers.md | 5 + Docs/Issues/Evaluation_Plans.md | 1 + Docs/Issues/ToDo.md | 3 + .../Programming/Programming-ainomege.md | 69 +++++++++++ Docs/TTS_STT.md | 101 ---------------- 10 files changed, 190 insertions(+), 109 deletions(-) create mode 100644 Docs/Prompts/Programming/Programming-ainomege.md delete mode 100644 Docs/TTS_STT.md diff --git a/Docs/Bitnet_and_Mamba_Papers.md b/Docs/Bitnet_and_Mamba_Papers.md index 8c47e9288..4e028a9cb 100644 --- a/Docs/Bitnet_and_Mamba_Papers.md +++ b/Docs/Bitnet_and_Mamba_Papers.md @@ -5,7 +5,7 @@ ### Bitnet - Unsorted - https://arxiv.org/abs/2411.11843 - + https://github.com/KYuuto1006/DQT ### Mamba diff --git a/Docs/Design/Storytelling_and_Creation.md b/Docs/Design/Storytelling_and_Creation.md index e638195f0..4bae7ab7d 100644 --- a/Docs/Design/Storytelling_and_Creation.md +++ b/Docs/Design/Storytelling_and_Creation.md @@ -4,6 +4,10 @@ ### Link Dump: https://github.com/mhgolkar/Arrow +https://github.com/neph1/LlamaTale +https://jianzongwu.github.io/projects/diffsensei/ +https://github.com/jianzongwu/DiffSensei + ### Storytelling diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index b8aa7e5ea..7d97bcf08 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -1,23 +1,42 @@ # Text-To-Speech / Speech-To-Text Documentation -### Overview +## Overview +Use of functions for individual services. +Function for each service, streaming & non-streaming. +Non-streaming will return a file, streaming will return a stream. +Use of temporary files for storage. +Use of pydub for audio manipulation. +Use of pydub for audio merging. +Flow: +1. Clean/format input text +2. Split text into segments +3. Generate audio for each segment using designated provider function +4. Merge audio segments into single output file +5. Clean up temporary files ### Services - Google Cloud Text-to-Speech - https://cloud.google.com/text-to-speech/docs/ssml -Models - https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF + + + +### Link Dump: https://github.com/albirrkarim/react-speech-highlight-demo +Models + https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF +Merging Audio + https://github.com/jiaaro/pydub GPT-SoviTTS -https://levelup.gitconnected.com/great-api-design-comprehensive-guide-from-basics-to-best-practices-9b4e0b613a44?source=home---------56-1--------------------0fc48da7_5ce6_48ca_92d2_260680a20318-------3 -https://rentry.org/GPT-SoVITS-guide -https://github.com/RVC-Boss/GPT-SoVITS + https://levelup.gitconnected.com/great-api-design-comprehensive-guide-from-basics-to-best-practices-9b4e0b613a44?source=home---------56-1--------------------0fc48da7_5ce6_48ca_92d2_260680a20318-------3 + https://rentry.org/GPT-SoVITS-guide + https://github.com/RVC-Boss/GPT-SoVITS + https://github.com/cpumaxx/sovits-ff-plugin MaskGCT https://maskgct.github.io/#emotion-samples @@ -29,4 +48,79 @@ MaskGCT TTS Pipeline https://www.astramind.ai/post/auralis -https://github.com/cpumaxx/sovits-ff-plugin \ No newline at end of file +https://github.com/cpumaxx/sovits-ff-plugin + + + +Train using: https://github.com/Mangio621/Mangio-RVC-Fork/releases, +import the .pth into https://huggingface.co/wok000/vcclient000/tree/main to convert your voice in near real time with about a .25s delay + +https://www.hackster.io/lhl/voicechat2-local-ai-voice-chat-4c48f2 + +https://github.com/abus-aikorea/voice-pro + +https://github.com/myshell-ai/MeloTTS +https://github.com/idiap/coqui-ai-TTS +https://docs.inferless.com/cookbook/serverless-customer-service-bot + + +https://huggingface.co/spaces/lamm-mit/PDF2Audio + +https://huggingface.co/spaces/bencser/episodegen + +https://github.com/Picovoice/speech-to-text-benchmark + +https://huggingface.co/papers/2410.02678 + +https://github.com/livekit/agents +https://github.com/pipecat-ai/pipecat/tree/a367a038f1a3967292b5de5b43b8600a82a73fb6?tab=readme-ov-file + +https://github.com/lamm-mit/PDF2Audio +https://github.com/Purfview/whisper-standalone-win +https://github.com/ictnlp/LLaMA-Omni + +https://github.com/SWivid/F5-TTS + + +https://github.com/matatonic/openedai-speech + +https://github.com/RVC-Boss/GPT-SoVITS +https://github.com/RVC-Boss/GPT-SoVITS/wiki/GPT%E2%80%90SoVITS%E2%80%90v2%E2%80%90features-(%E6%96%B0%E7%89%B9%E6%80%A7) +https://rentry.org/GPT-SoVITS-guide +https://rentry.org/GPT-SoVITS-guide +It's just the 3 buttons (speech-to-text, ssl, semantics) and then training. + +The default training settings on the gradio UI are fine but I save epoch 12-16-24 on SoVITS for testing as that's the sweet spot range. + +Next thing that matters a lot is the ref audio you pick, and you can also drop your entire dataset into the "multiple references to average tone" box, which can improve the voice + +Only thing I changed was remove the space at the beginning of each lines in your list file + +(Look at batch size/ list file) + +And make sure you get the latest version https://github.com/RVC-Boss/GPT-SoVITS/releases + +https://github.com/souzatharsis/podcastfy + +https://github.com/THUDM/GLM-4-Voice/tree/main + +https://huggingface.co/cydxg/glm-4-voice-9b-int4/blob/main/README_en.md + +https://github.com/meta-llama/llama-recipes/tree/main/recipes%2Fquickstart%2FNotebookLlama + + +https://sakshi113.github.io/mmau_homepage/ + +https://github.com/fishaudio/fish-speech/tree/main +https://github.com/fishaudio/fish-speech/blob/main/Start_Agent.md +https://huggingface.co/fishaudio/fish-agent-v0.1-3b/tree/main + +https://github.com/pixelpump/Ai-Interview-Assistant-Python + +https://github.com/Standard-Intelligence/hertz-dev + +https://github.com/edwko/OuteTTS +https://huggingface.co/OuteAI/OuteTTS-0.2-500M-GGUF +https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF + +https://www.twilio.com/en-us/blog/twilio-openai-realtime-api-launch-integration \ No newline at end of file diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 487dfbfe9..ee19af526 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -55,6 +55,9 @@ https://github.com/Rugz007/liha https://astro.new/latest/ https://github.com/lobehub/lobe-chat https://www.nngroup.com/videos/the-danger-of-defaults/ +https://writings.stephenwolfram.com/2024/12/useful-to-the-point-of-being-revolutionary-introducing-wolfram-notebook-assistant/ + + diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 6d9053236..336144f91 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -17,6 +17,9 @@ https://huggingface.co/Qwen/Qwen2-VL-2B https://www.reddit.com/r/StableDiffusion/comments/1h7hunp/how_to_run_hunyuanvideo_on_a_single_24gb_vram_card/ https://arxiv.org/abs/2412.05185 +https://huggingface.co/collections/OpenGVLab/internvl-25-673e1019b66e2218f68d7c1c + +https://huggingface.co/papers/2412.07626 diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index d234cdfd8..ee3cab7d2 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -15,6 +15,8 @@ https://arxiv.org/abs/2412.02906 https://arxiv.org/pdf/2410.21272 https://arxiv.org/abs/2412.04318 https://arxiv.org/abs/2412.02142 +https://arxiv.org/abs/2412.05265 + ### Context https://arxiv.org/pdf/2304.12102 @@ -27,6 +29,9 @@ https://arxiv.org/abs/2412.02142 ### Reasoning - https://huggingface.co/papers/2411.15862 +https://arxiv.org/abs/2411.19865 +https://arxiv.org/abs/2412.06769 + ### Test-Time Compute - https://github.com/GAIR-NLP/O1-Journey diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index afc08ecb6..8f21c4784 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -184,6 +184,7 @@ Links: - https://github.com/FSoft-AI4Code/CodeMMLU - StackUnseen - https://prollm.toqan.ai/leaderboard/stack-unseen + - https://huggingface.co/papers/2412.05210 - **Cognitive Biases** - CBEval: https://arxiv.org/abs/2412.03605 - **Confabulation-Rate** diff --git a/Docs/Issues/ToDo.md b/Docs/Issues/ToDo.md index f43519309..460d4bd13 100644 --- a/Docs/Issues/ToDo.md +++ b/Docs/Issues/ToDo.md @@ -37,3 +37,6 @@ List of stuff I don't feel like filing an issue for (yet). https://ai.gopubby.com/demystifying-pdf-parsing-04-ocr-free-large-multimodal-model-based-method-0fdab50db048 https://pub.towardsai.net/demystifying-pdf-parsing-05-unifying-separate-tasks-into-a-small-model-d3739db021f7 https://ai.gopubby.com/demystifying-pdf-parsing-06-representative-industry-solutions-5d4a1cfe311b + +- Prompts + - https://github.com/ProfSynapse/Synapse_CoR \ No newline at end of file diff --git a/Docs/Prompts/Programming/Programming-ainomege.md b/Docs/Prompts/Programming/Programming-ainomege.md new file mode 100644 index 000000000..c33f65196 --- /dev/null +++ b/Docs/Prompts/Programming/Programming-ainomege.md @@ -0,0 +1,69 @@ +### TITLE ### +Programming - ainomege + +### AUTHOR ### +ainomege@reddit + +### SYSTEM ### +[CORE IDENTITY] You are a collaborative software developer on the user's team, functioning as both a thoughtful implementer and constructive critic. Your primary directive is to engage in iterative, test-driven development while maintaining unwavering commitment to clean, maintainable code. + +[BASE BEHAVIORS] + + REQUIREMENT VALIDATION Before generating any solution, automatically: { IDENTIFY { - Core functionality required - Immediate use cases - Essential constraints } QUESTION when detecting { - Ambiguous requirements - Speculative features - Premature optimization attempts - Mixed responsibilities } } + + SOLUTION GENERATION PROTOCOL When generating solutions: { ENFORCE { Single_Responsibility: "Each component handles exactly one concern" Open_Closed: "Extensions yes, modifications no" Liskov_Substitution: "Subtypes must be substitutable" Interface_Segregation: "Specific interfaces over general ones" Dependency_Inversion: "Depend on abstractions only" } VALIDATE_AGAINST { Complexity_Check: "Could this be simpler?" Necessity_Check: "Is this needed now?" Responsibility_Check: "Is this the right component?" Interface_Check: "Is this the minimum interface?" } } + + COLLABORATIVE DEVELOPMENT PROTOCOL On receiving task: { PHASE_1: REQUIREMENTS { ACTIVELY_PROBE { - Business context and goals - User needs and scenarios - Technical constraints - Integration requirements }} PHASE_2: SOLUTION_DESIGN { FIRST { - Propose simplest viable solution - Identify potential challenges - Highlight trade-offs }} PHASE_3: TEST_DRIVEN_IMPLEMENTATION { ITERATE { 1. Write failing test 2. Implement minimal code 3. Verify test passes 4. Refactor if needed }} }Copy Copy Copy CONTINUE_UNTIL { - All critical requirements are clear - Edge cases are identified - Assumptions are validated } THEN { - Challenge own assumptions - Suggest alternative approaches - Evaluate simpler options } SEEK_AGREEMENT on { - Core approach - Implementation strategy - Success criteria } MAINTAIN { - Test coverage - Code clarity - SOLID principles } + + CODE GENERATION RULES When writing code: { PRIORITIZE { Clarity > Cleverness Simplicity > Flexibility Current_Needs > Future_Possibilities Explicit > Implicit } ENFORCE { - Single responsibility per unit - Clear interface boundaries - Minimal dependencies - Explicit error handling } } + + QUALITY CONTROL Before presenting solution: { VERIFY { Simplicity: "Is this the simplest possible solution?" Necessity: "Is every component necessary?" Responsibility: "Are concerns properly separated?" Extensibility: "Can this be extended without modification?" Dependency: "Are dependencies properly abstracted?" } } + +[FORBIDDEN PATTERNS] DO NOT: + + Add "just in case" features + + Create abstractions without immediate use + + Mix multiple responsibilities + + Implement future requirements + + Optimize prematurely + +[RESPONSE STRUCTURE] Always structure responses as: { 1. Requirement Clarification 2. Core Solution Design 3. Implementation Details 4. Key Design Decisions 5. Validation Results } + +[COLLABORATIVE EXECUTION MODE] { BEHAVE_AS { Team_Member: "Proactively engage in development process" Critical_Thinker: "Challenge assumptions and suggest improvements" Quality_Guardian: "Maintain high standards through TDD" } + +MAINTAIN { + - KISS (Keep It Simple, Stupid) + - YAGNI (You Aren't Gonna Need It) + - SOLID Principles + - DRY (Don't Repeat Yourself) +} + +DEMONSTRATE { + Ownership: "Take responsibility for code quality" + Initiative: "Proactively identify issues and solutions" + Collaboration: "Engage in constructive dialogue" +} + +} + +[ERROR HANDLING] When detecting violations: { 1. Identify specific principle breach 2. Explain violation clearly 3. Provide simplest correction 4. Verify correction maintains requirements } + +[CONTINUOUS VALIDATION] During all interactions: { MONITOR for: - Scope creep - Unnecessary complexity - Mixed responsibilities - Premature optimization + +CORRECT by: +- Returning to core requirements +- Simplifying design +- Separating concerns +- Focusing on immediate needs + +} + +### USER ### +This is where you place the user prompt text + +### KEYWORDS ### +ainomege,programming diff --git a/Docs/TTS_STT.md b/Docs/TTS_STT.md deleted file mode 100644 index 6a63302bb..000000000 --- a/Docs/TTS_STT.md +++ /dev/null @@ -1,101 +0,0 @@ -# Text-To-Speech / Speech-To-Text Documentation - -## Overview -Use of functions for individual services. -Function for each service, streaming & non-streaming. -Non-streaming will return a file, streaming will return a stream. -Use of temporary files for storage. -Use of pydub for audio manipulation. -Use of pydub for audio merging. - - -https://github.com/jiaaro/pydub - -Flow: -1. Clean/format input text -2. Split text into segments -3. Generate audio for each segment using designated provider function -4. Merge audio segments into single output file -5. Clean up temporary files - - - - -### Link Dump: - -https://github.com/cpumaxx/sovits-ff-plugin - - - -Train using: https://github.com/Mangio621/Mangio-RVC-Fork/releases, -import the .pth into https://huggingface.co/wok000/vcclient000/tree/main to convert your voice in near real time with about a .25s delay - -https://www.hackster.io/lhl/voicechat2-local-ai-voice-chat-4c48f2 - -https://github.com/abus-aikorea/voice-pro - -https://github.com/myshell-ai/MeloTTS -https://github.com/idiap/coqui-ai-TTS -https://docs.inferless.com/cookbook/serverless-customer-service-bot - - -https://huggingface.co/spaces/lamm-mit/PDF2Audio - -https://huggingface.co/spaces/bencser/episodegen - -https://github.com/Picovoice/speech-to-text-benchmark - -https://huggingface.co/papers/2410.02678 - -https://github.com/livekit/agents -https://github.com/pipecat-ai/pipecat/tree/a367a038f1a3967292b5de5b43b8600a82a73fb6?tab=readme-ov-file - -https://github.com/lamm-mit/PDF2Audio -https://github.com/Purfview/whisper-standalone-win -https://github.com/ictnlp/LLaMA-Omni - -https://github.com/SWivid/F5-TTS - - -https://github.com/matatonic/openedai-speech - -https://github.com/RVC-Boss/GPT-SoVITS -https://github.com/RVC-Boss/GPT-SoVITS/wiki/GPT%E2%80%90SoVITS%E2%80%90v2%E2%80%90features-(%E6%96%B0%E7%89%B9%E6%80%A7) -https://rentry.org/GPT-SoVITS-guide -https://rentry.org/GPT-SoVITS-guide -It's just the 3 buttons (speech-to-text, ssl, semantics) and then training. - -The default training settings on the gradio UI are fine but I save epoch 12-16-24 on SoVITS for testing as that's the sweet spot range. - -Next thing that matters a lot is the ref audio you pick, and you can also drop your entire dataset into the "multiple references to average tone" box, which can improve the voice - -Only thing I changed was remove the space at the beginning of each lines in your list file - -(Look at batch size/ list file) - -And make sure you get the latest version https://github.com/RVC-Boss/GPT-SoVITS/releases - -https://github.com/souzatharsis/podcastfy - -https://github.com/THUDM/GLM-4-Voice/tree/main - -https://huggingface.co/cydxg/glm-4-voice-9b-int4/blob/main/README_en.md - -https://github.com/meta-llama/llama-recipes/tree/main/recipes%2Fquickstart%2FNotebookLlama - - -https://sakshi113.github.io/mmau_homepage/ - -https://github.com/fishaudio/fish-speech/tree/main -https://github.com/fishaudio/fish-speech/blob/main/Start_Agent.md -https://huggingface.co/fishaudio/fish-agent-v0.1-3b/tree/main - -https://github.com/pixelpump/Ai-Interview-Assistant-Python - -https://github.com/Standard-Intelligence/hertz-dev - -https://github.com/edwko/OuteTTS -https://huggingface.co/OuteAI/OuteTTS-0.2-500M-GGUF -https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF - -https://www.twilio.com/en-us/blog/twilio-openai-realtime-api-launch-integration \ No newline at end of file From 99e3f7335da5cbcffb453d0f37ac45465d9424df Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 12 Dec 2024 11:17:00 -0800 Subject: [PATCH 08/88] links --- App_Function_Libraries/TTS/TTS.py | 4 +++- Docs/Design/Character_Chat.md | 3 +++ Docs/Design/Structured_Outputs.md | 1 + Docs/Issues/Evaluation_Plans.md | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/App_Function_Libraries/TTS/TTS.py b/App_Function_Libraries/TTS/TTS.py index 87b43bb43..d82f68af4 100644 --- a/App_Function_Libraries/TTS/TTS.py +++ b/App_Function_Libraries/TTS/TTS.py @@ -102,7 +102,9 @@ def set_api_key(provider: str, api_key: str) -> None: tts_config = load_tts_config() def tts_generate_audio(input_text, provider, voice, model): - input_text = "Hello, how are you? I'm doing well, thank you!" + # if no input_text is passed, use a default text as a shorthand for validating X service works. + if not input_text: + input_text = "Hello, how are you? I'm doing well, thank you!" if provider not in tts_providers: raise ValueError(f"Unsupported provider: {provider}") elif provider == "elevenlabs": diff --git a/Docs/Design/Character_Chat.md b/Docs/Design/Character_Chat.md index f66c43af9..34ce60544 100644 --- a/Docs/Design/Character_Chat.md +++ b/Docs/Design/Character_Chat.md @@ -3,3 +3,6 @@ # Link Dump: https://github.com/caspianmoon/memoripy +https://arxiv.org/abs/2407.03974 + + diff --git a/Docs/Design/Structured_Outputs.md b/Docs/Design/Structured_Outputs.md index 8cb7f9147..becc52b40 100644 --- a/Docs/Design/Structured_Outputs.md +++ b/Docs/Design/Structured_Outputs.md @@ -24,4 +24,5 @@ https://docs.pydantic.dev/latest/ https://github.com/outlines-dev/outlines[ https://github.com/Dan-wanna-M/formatron/tree/master https://blog.dottxt.co/selective-multiplication.html +https://blog.dottxt.co/say-what-you-mean.html diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 8f21c4784..76a4fa48d 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -31,6 +31,8 @@ General Research Chat arena Building one https://github.com/lm-sys/FastChat + https://github.com/Teachings/llm_tools_benchmark + https://github.com/Nutlope/codearena Potential issues with creating a chatarena system https://arxiv.org/abs/2412.04363 LLM-as-judge @@ -213,6 +215,8 @@ Links: - https://github.com/Quehry/HelloBench - https://arxiv.org/abs/2409.16191 - https://github.com/jonathan-roberts1/needle-threading/ + - Michelangelo + - https://arxiv.org/abs/2409.12640 - **Creative Writing** - EQ Bench - https://eqbench.com/creative_writing.html From 7796c56f7f214f4a16d8665fd516179369bd2f9c Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 13 Dec 2024 21:04:30 -0800 Subject: [PATCH 09/88] Scrollbars elude me Don't know why scrollbars aren't working. Google documentation for Gemini voice is lacking. --- App_Function_Libraries/Gradio_Related.py | 4 + .../Gradio_UI/Website_scraping_tab.py | 7 +- App_Function_Libraries/STT/STT.py | 4 + App_Function_Libraries/TTS/TTS.py | 70 ++- App_Function_Libraries/TTS/TTS_Providers.py | 438 +++++++++++++----- App_Function_Libraries/Utils/Utils.py | 68 ++- .../Web_Scraping/Article_Extractor_Lib.py | 54 ++- CONTRIBUTING.md | 3 +- Docs/Design/Researcher.md | 11 + Docs/Design/Storytelling_and_Creation.md | 1 + Docs/Design/UX.md | 15 +- Docs/Design/VLMs.md | 9 + Docs/Handy_Dandy_Papers.md | 1 + Docs/Issues/Evaluation_Plans.md | 2 +- Docs/VLM.md | 11 - Server_API/Dockerfile | 10 - .../app/api/v1/endpoints/video_processing.py | 25 - Server_API/app/main.py | 6 +- summarize.py | 3 + 19 files changed, 547 insertions(+), 195 deletions(-) delete mode 100644 Docs/VLM.md delete mode 100644 Server_API/Dockerfile delete mode 100644 Server_API/app/api/v1/endpoints/video_processing.py diff --git a/App_Function_Libraries/Gradio_Related.py b/App_Function_Libraries/Gradio_Related.py index 55e175749..5f5d7b420 100644 --- a/App_Function_Libraries/Gradio_Related.py +++ b/App_Function_Libraries/Gradio_Related.py @@ -403,6 +403,10 @@ def launch_ui(share_public=None, server_mode=False): padding: 10px; margin-top: 10px; } + .scrollable-textbox textarea { + height: 600px !important; + overflow-y: auto !important; + } """ config = load_and_log_configs() diff --git a/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py b/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py index 5204547e0..c32dfb286 100644 --- a/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py +++ b/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py @@ -408,7 +408,12 @@ def create_website_scraping_tab(): with gr.Column(): progress_output = gr.Textbox(label="Progress", lines=3) - result_output = gr.Textbox(label="Result", lines=20) + result_output = gr.Textbox( + label="Web Scraping Results", + lines=20, + elem_classes="scrollable-textbox", + show_copy_button=True + ) def update_ui_for_scrape_method(method): url_level_update = gr.update(visible=(method == "URL Level")) diff --git a/App_Function_Libraries/STT/STT.py b/App_Function_Libraries/STT/STT.py index 8e9901150..1541360bf 100644 --- a/App_Function_Libraries/STT/STT.py +++ b/App_Function_Libraries/STT/STT.py @@ -12,6 +12,10 @@ # # Functions: + + +#https://github.com/QwenLM/Qwen2-Audio + # # End of STT.py ####################################################################################################################### diff --git a/App_Function_Libraries/TTS/TTS.py b/App_Function_Libraries/TTS/TTS.py index d82f68af4..4c050c779 100644 --- a/App_Function_Libraries/TTS/TTS.py +++ b/App_Function_Libraries/TTS/TTS.py @@ -7,7 +7,7 @@ from pydub import AudioSegment import re import tempfile -from typing import List, Tuple +from typing import List, Tuple, Optional # External Imports # # Local Imports @@ -17,6 +17,72 @@ # Functions: +####################################################### +# +# TTS Provider Check Functions + +tts_providers = ["elevenlabs", "openai", "edge", "google", "sovitts", "qwen"] + +def test_all_tts_providers(): + try: + # Load configuration + #config = load_config() + + # Override default TTS model to use edge for tests + test_config = {"text_to_speech": {"default_tts_model": "edge"}} + + # Read input text from file + with open( + "tests/data/transcript_336aa9f955cd4019bc1287379a5a2820.txt", "r" + ) as file: + input_text = file.read() + + # Test ElevenLabs + tts_elevenlabs = TextToSpeech(model="elevenlabs") + elevenlabs_output_file = "tests/data/response_elevenlabs.mp3" + tts_elevenlabs.convert_to_speech(input_text, elevenlabs_output_file) + logging.info( + f"ElevenLabs TTS completed. Output saved to {elevenlabs_output_file}" + ) + + # Test OpenAI + tts_openai = TextToSpeech(model="openai") + openai_output_file = "tests/data/response_openai.mp3" + tts_openai.convert_to_speech(input_text, openai_output_file) + logging.info(f"OpenAI TTS completed. Output saved to {openai_output_file}") + + # Test Edge + tts_edge = TextToSpeech(model="edge") + edge_output_file = "tests/data/response_edge.mp3" + tts_edge.convert_to_speech(input_text, edge_output_file) + logging.info(f"Edge TTS completed. Output saved to {edge_output_file}") + + # Test Google + tts_google = TextToSpeech(model="google") + google_output_file = "tests/data/response_google.mp3" + tts_google.convert_to_speech(input_text, google_output_file) + logging.info(f"Google TTS completed. Output saved to {google_output_file}") + + # Test Sovi TTS + tts_sovitts = TextToSpeech(model="sovitts") + sovitts_output_file = "tests/data/response_sovitts.mp3" + tts_sovitts.convert_to_speech(input_text, sovitts_output_file) + logging.info(f"Sovi TTS completed. Output saved to {sovitts_output_file}") + + # Test Qwen TTS + tts_qwen = TextToSpeech(model="qwen") + qwen_output_file = "tests/data/response_qwen.mp3" + tts_qwen.convert_to_speech(input_text, qwen_output_file) + logging.info(f"Qwen TTS completed. Output saved to {qwen_output_file}") + + except Exception as e: + logging.error(f"An error occurred during text-to-speech conversion: {str(e)}") + raise + +# +# End of TTS Provider Check Functions +####################################################### + ####################################################### # @@ -101,7 +167,7 @@ def set_api_key(provider: str, api_key: str) -> None: tts_config = load_tts_config() -def tts_generate_audio(input_text, provider, voice, model): +def tts_generate_audio_single_speaker(input_text, provider, voice, model): # if no input_text is passed, use a default text as a shorthand for validating X service works. if not input_text: input_text = "Hello, how are you? I'm doing well, thank you!" diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 7fd611483..c1c8259b7 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -2,87 +2,163 @@ # Description: This file contains the functions to allow for usage of different TTS providers. # # Imports +import logging import os import tempfile -from typing import List +import uuid + +# # External Imports import edge_tts -from elevenlabs import client as elevenlabs_client -import openai +import requests # # Local Imports +from App_Function_Libraries.Utils.Utils import load_and_log_configs, loaded_config_data # ####################################################################################################################### # # Functions: -tts_providers = ["elevenlabs", "openai", "edge"] +####################################################### +# +# OpenAI TTS Provider Functions -def test_all_tts_providers(): +# https://github.com/leokwsw/OpenAI-TTS-Gradio/blob/main/app.py +def generate_audio_openai(api_key, input_text, voice, model, response_format="mp3", output_file="speech.mp3"): + """ + Generate audio using OpenAI's Text-to-Speech API. + + Args: + api_key (str): OpenAI API key. + input_text (str): Text input for speech synthesis. + voice (str): Voice to use for the synthesis. + model (str): Model to use for the synthesis (e.g., "tts-1"). + response_format (str): Format of the response audio file (default is "mp3"). + output_file (str): Name of the output file to save the audio. + + Returns: + str: Path to the saved audio file if successful. + + Raises: + ValueError: If required inputs are missing or invalid. + RuntimeError: If the API request fails. + """ + # Validate inputs + loaded_config_data = load_and_log_configs() + openai_api_key = api_key + + # API key validation try: - # Load configuration - #config = load_config() - - # Override default TTS model to use edge for tests - test_config = {"text_to_speech": {"default_tts_model": "edge"}} + if not openai_api_key: + logging.info("OpenAI: API key not provided as parameter") + logging.info("OpenAI: Attempting to use API key from config file") + openai_api_key = loaded_config_data['api_keys']['openai'] - # Read input text from file - with open( - "tests/data/transcript_336aa9f955cd4019bc1287379a5a2820.txt", "r" - ) as file: - input_text = file.read() + if not openai_api_key: + logging.error("OpenAI: API key not found or is empty") + return "OpenAI: API Key Not Provided/Found in Config file or is empty" - # Test ElevenLabs - tts_elevenlabs = TextToSpeech(model="elevenlabs") - elevenlabs_output_file = "tests/data/response_elevenlabs.mp3" - tts_elevenlabs.convert_to_speech(input_text, elevenlabs_output_file) - logger.info( - f"ElevenLabs TTS completed. Output saved to {elevenlabs_output_file}" - ) + logging.debug(f"OpenAI: Using API Key: {openai_api_key[:5]}...{openai_api_key[-5:]}") + except Exception as e: + logging.error(f"OpenAI: Error loading API Key: {str(e)}") + return f"OpenAI: Error loading API Key: {str(e)}" - # Test OpenAI - tts_openai = TextToSpeech(model="openai") - openai_output_file = "tests/data/response_openai.mp3" - tts_openai.convert_to_speech(input_text, openai_output_file) - logger.info(f"OpenAI TTS completed. Output saved to {openai_output_file}") + # Input data handling + try: + if not input_text: + raise ValueError("Text input is required.") + logging.debug(f"OpenAI: Raw input data type: {type(input_text)}") + logging.debug(f"OpenAI: Raw input data (first 500 chars): {str(input_text)[:500]}...") + except Exception as e: + logging.error(f"OpenAI: Error loading input text: {str(e)}") + return f"OpenAI: Error loading input text: {str(e)}" - # Test Edge - tts_edge = TextToSpeech(model="edge") - edge_output_file = "tests/data/response_edge.mp3" - tts_edge.convert_to_speech(input_text, edge_output_file) - logger.info(f"Edge TTS completed. Output saved to {edge_output_file}") + # Voice selection handling + try: + if not voice: + logging.info("OpenAI: Speaker Voice not provided as parameter") + logging.info("OpenAI: Attempting to use Speaker Voice from config file") + voice = loaded_config_data['tts_settings']['default_openai_tts_voice'] + if not voice: + raise ValueError("Voice is required. Default voice not found in config file and no voice selection was passed.") except Exception as e: - logger.error(f"An error occurred during text-to-speech conversion: {str(e)}") - raise + logging.error(f"OpenAI: Error loading Speaker Voice: {str(e)}") + return f"OpenAI: Error loading Speaker Voice: {str(e)}" -####################################################### -# -# OpenAI TTS Provider Functions + # Model selection handling + try: + if not model: + logging.info("OpenAI: Model not provided as parameter") + logging.info("OpenAI: Attempting to use Model from config file") + model = loaded_config_data['tts_settings']['default_openai+tts_model'] -def generate_audio_openai(text, voice, model): - """OpenAI Text-to-Speech provider / Generate audio using OpenAI API.""" - # Provider-specific SSML tags - openai_ssml_tags: List[str] = ['break', 'emphasis'] - - # FIXME - Add API Key check + model loading - # if api_key: - # openai.api_key = api_key - # elif not openai.api_key: - # raise ValueError("OpenAI API key must be provided or set in environment") - # self.model = model - - # FIXME - add check for model, voice, input values - try: - response = openai.audio.speech.create( - model=model, - voice=voice, - input=text - ) - return response.content + if not model: + raise ValueError("Model is required. Default model not found in config and no model selection was passed.") except Exception as e: + logging.error(f"OpenAI: Error Selecting Model: {str(e)}") + return f"OpenAI: Error Selecting Model: {str(e)}" + + # API endpoint + endpoint = "https://api.openai.com/v1/audio/speech" + + # Headers for the API request + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + # Payload for the API request + payload = { + "model": model, + "input": input_text, + "voice": voice, + } + + try: + # Make the request to the API + response = requests.post(endpoint, headers=headers, json=payload, stream=True) + response.raise_for_status() # Raise an error for HTTP status codes >= 400 + + # Save the audio response to a file + with open(output_file, "wb") as f: + f.write(response.content) + + print(f"Audio successfully generated and saved to {output_file}.") + return output_file + + except requests.exceptions.RequestException as e: raise RuntimeError(f"Failed to generate audio: {str(e)}") from e + +def test_generate_audio_openai(api_key, input_text, voice, model, response_format="mp3", output_file="speech.mp3"): + try: + if not api_key: + logging.info("OpenAI: API key not provided as parameter") + logging.info("OpenAI: Attempting to use API key from config file") + openai_api_key = loaded_config_data['api_keys']['openai'] + + if not api_key: + logging.error("OpenAI: API key not found or is empty") + return "OpenAI: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"OpenAI: Using API Key: {api_key[:5]}...{api_key[-5:]}") + except Exception as e: + logging.error(f"OpenAI: Error loading API Key: {str(e)}") + return f"OpenAI: Error loading API Key: {str(e)}" + + input_text = "The quick brown fox jumped over the lazy dog." + + voice = "alloy" + + model = "tts-1" + + try: + output_file = generate_audio_openai(api_key, input_text, voice, model) + print(f"Generated audio file: {output_file}") + except Exception as e: + print(f"Error: {e}") + # # End of OpenAI TTS Provider Functions ####################################################### @@ -90,44 +166,43 @@ def generate_audio_openai(text, voice, model): ####################################################### # -# MS Edge TTS Provider Functions - -# https://github.com/rany2/edge-tts - -def generate_audio_edge(text, voice, model=None, voice2=None): - # Edge doesn't use a selectable model for TTS - # FIXME - SSML tags - common_ssml_tags = ['lang', 'p', 'phoneme', 's', 'sub'] - def generate_audio(self, text: str, voice: str, model: str, voice2: str = None) -> bytes: - """Generate audio using Edge TTS.""" - import nest_asyncio - import asyncio - - # Apply nest_asyncio to allow nested event loops - nest_asyncio.apply() - - async def _generate(): - communicate = edge_tts.Communicate(text, voice) - # Create a temporary file with proper context management - with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as tmp_file: - temp_path = tmp_file.name - - try: - # Save audio to temporary file - await communicate.save(temp_path) - # Read the audio data - with open(temp_path, 'rb') as f: - return f.read() - finally: - # Clean up temporary file - if os.path.exists(temp_path): - os.remove(temp_path) - - # Use nest_asyncio to handle nested event loops - loop = asyncio.get_event_loop() - return loop.run_until_complete(_generate()) - result = generate_audio(text, voice, model, voice2) - return result +# MS Edge TTS Provider Functions - NOPE +# +# # https://github.com/rany2/edge-tts +# +# def generate_audio_edge(text, voice, model=None, voice2=None): +# # FIXME - SSML tags +# common_ssml_tags = ['lang', 'p', 'phoneme', 's', 'sub'] +# def generate_audio(self, text: str, voice: str, model: str, voice2: str = None) -> bytes: +# """Generate audio using Edge TTS.""" +# import nest_asyncio +# import asyncio +# +# # Apply nest_asyncio to allow nested event loops +# nest_asyncio.apply() +# +# async def _generate(): +# communicate = edge_tts.Communicate(text, voice) +# # Create a temporary file with proper context management +# with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as tmp_file: +# temp_path = tmp_file.name +# +# try: +# # Save audio to temporary file +# await communicate.save(temp_path) +# # Read the audio data +# with open(temp_path, 'rb') as f: +# return f.read() +# finally: +# # Clean up temporary file +# if os.path.exists(temp_path): +# os.remove(temp_path) +# +# # Use nest_asyncio to handle nested event loops +# loop = asyncio.get_event_loop() +# return loop.run_until_complete(_generate()) +# result = generate_audio(text, voice, model, voice2) +# return result # # End of MS Edge TTS Provider Functions @@ -137,23 +212,160 @@ async def _generate(): ####################################################### # # ElvenLabs TTS Provider Functions +# FIXME - all of this -def generate_audio_elevenlabs(text, voice, model, voice2=None): +#https://elevenlabs.io/docs/api-reference/text-to-speech +def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): """Generate audio using ElevenLabs API.""" - # FIXME - add check for config settings - # FIXME - add check + load for API key - # FIXME - add check + load for model, voice, input values + CHUNK_SIZE = 1024 + # API key validation + elevenlabs_api_key = api_key + try: + if not elevenlabs_api_key: + logging.info("ElevenLabs: API key not provided as parameter") + logging.info("ElevenLabs: Attempting to use API key from config file") + elevenlabs_api_key = loaded_config_data['api_keys']['elevenlabs'] + + if not elevenlabs_api_key: + logging.error("ElevenLabs: API key not found or is empty") + return "ElevenLabs: API Key Not Provided/Found in Config file or is empty" + + logging.debug(f"ElevenLabs: Using API Key: {elevenlabs_api_key[:5]}...{elevenlabs_api_key[-5:]}") + except Exception as e: + logging.error(f"ElevenLabs: Error loading API Key: {str(e)}") + return f"ElevenLabs: Error loading API Key: {str(e)}" + + # Input data handling + try: + if not input_text: + raise ValueError("Text input is required.") + logging.debug(f"ElevenLabs: Raw input data type: {type(input_text)}") + logging.debug(f"ElevenLabs: Raw input data (first 500 chars): {str(input_text)[:500]}...") + except Exception as e: + logging.error(f"ElevenLabs: Error loading input text: {str(e)}") + return f"ElevenLabs: Error loading input text: {str(e)}" + + # Handle Voice ID + try: + if not voice: + logging.info("ElevenLabs: Speaker ID(Voice) not provided as parameter") + logging.info("ElevenLabs: Attempting to use Speaker ID(Voice) from config file") + voice = loaded_config_data['tts_settings']['default_eleven_tts_voice'] + + if not voice: + raise ValueError("Voice is required. Default voice not found in config file and no voice selection was passed.") + except Exception as e: + logging.error(f"ElevenLabs: Error loading Speaker ID(Voice): {str(e)}") + return f"ElevenLabs: Error loading Speaker ID(Voice): {str(e)}" + + # Handle Model ID/Selection + # Set Voice Model + model="eleven_turbo_v2_5", # use the turbo model for low latency + try: + if not model: + logging.info("ElevenLabs: Model not provided as parameter") + logging.info("ElevenLabs: Attempting to use Model from config file") + model = loaded_config_data['tts_settings']['default_eleven_tts_model'] + + if not model: + raise ValueError("Model is required. Default model not found in config file and no model selection was passed.") + except Exception as e: + logging.error(f"ElevenLabs: Error Selecting Model: {str(e)}") + return f"ElevenLabs: Error Selecting Model: {str(e)}" + # FIXME - add SSML tags - elvenlabs_ssml_tags = ['lang', 'p', 'phoneme', 's', 'sub'] + # File output (non-streaming) + output_format="mp3_22050_32", + + # Set the parameters for the TTS conversion + try: + default_eleven_tts_voice_stability = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_stability', 0.0) + except Exception as e: + logging.error(f"ElevenLabs: Error loading Stability: {str(e)}") + return f"ElevenLabs: Error loading Stability: {str(e)}" - # FIXME - IDE says '.generate' doesn't exist? - audio = elevenlabs_client.generate( - text=text, - voice=voice, - model=model - ) - return b''.join(chunk for chunk in audio if chunk) + try: + # Similarity Boost + default_eleven_tts_voice_similiarity_boost = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_similiarity_boost', 1.0) + except Exception as e: + logging.error(f"ElevenLabs: Error loading Similarity Boost: {str(e)}") + return f"ElevenLabs: Error loading Similarity Boost: {str(e)}" + + try: + # Style + default_eleven_tts_voice_style = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_style', 0.0) + except Exception as e: + logging.error(f"ElevenLabs: Error loading Style: {str(e)}") + return f"ElevenLabs: Error loading Style: {str(e)}" + + try: + # Use Speaker Boost + default_eleven_tts_voice_use_speaker_boost = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_use_speaker_boost', True) + except Exception as e: + logging.error(f"ElevenLabs: Error loading Use Speaker Boost: {str(e)}") + return f"ElevenLabs: Error loading Use Speaker Boost: {str(e)}" + + try: + # Output Format + default_eleven_tts_output_format = loaded_config_data['tts_settings'].get('default_eleven_tts_output_format', "mp3_44100_192") + except Exception as e: + logging.error(f"ElevenLabs: Error loading Output Format: {str(e)}") + return f"ElevenLabs: Error loading Output Format: {str(e)}" + + # Make the API request + # Construct the URL for the Text-to-Speech API request + tts_url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream?output_format=mp3_44100_192" + + # Set up headers for the API request, including the API key for authentication + headers = { + "Accept": "application/json", + "xi-api-key": elevenlabs_api_key + } + + # Set up the data payload for the API request, including the text and voice settings + data = { + "text": input_text, + "model_id": model, + "output_format": default_eleven_tts_output_format, + "voice_settings": { + "stability": default_eleven_tts_voice_stability, + "similarity_boost": default_eleven_tts_voice_similiarity_boost, + "style": default_eleven_tts_voice_style, + "use_speaker_boost": default_eleven_tts_voice_use_speaker_boost + } + } + try: + # Make the POST request to the TTS API with headers and data, enabling streaming response + with requests.post(tts_url, headers=headers, json=data, stream=True) as response: + # Check if the request was successful + if response.ok: + # Create a temporary file - FIXME + with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file: + # Read the response in chunks and write to the file + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + tmp_file.write(chunk) + temp_file_path = tmp_file.name + # Inform the user of success + print(f"Audio stream saved successfully to {temp_file_path}.") + if not response.ok: + logging.error(f"API request failed: {response.status_code} - {response.text}") + return f"API request failed: {response.status_code} - {response.text}" + except requests.exceptions.RequestException as e: + raise RuntimeError(f"Failed to generate audio: {str(e)}") from e + + save_file_path = f"{uuid.uuid4()}.mp3" + + # Writing the audio to a file + with open(save_file_path, "wb") as f: + for chunk in response: + if chunk: + f.write(chunk) + + print(f"{save_file_path}: A new audio file was saved successfully!") + + # Return the path of the saved audio file + return save_file_path # End of ElvenLabs TTS Provider Functions ####################################################### @@ -163,11 +375,23 @@ def generate_audio_elevenlabs(text, voice, model, voice2=None): # # Google Gemini TTS Provider Functions +# https://github.com/google-gemini/cookbook/blob/main/quickstarts/Audio.ipynb +# Fuck google. lets wait for their docs to not be complete fucking shit. # # End of Google Gemini TTS Provider Functions ####################################################### + +####################################################### +# +# gpt-soviTTS TTS Provider Functions +# https://github.com/RVC-Boss/GPT-SoVITS + +# +# End of gpt-soviTTS TTS Provider Functions +####################################################### + # # End of TTS_Providers.py ####################################################################################################################### diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 06f1fa773..0ccc0b897 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -219,7 +219,11 @@ def load_and_log_configs(): google_api_key = config.get('API', 'google_api_key', fallback=None) logging.debug( - f"Loaded Mistral API Key: {google_api_key[:5]}...{google_api_key[-5:] if google_api_key else None}") + f"Loaded Google API Key: {google_api_key[:5]}...{google_api_key[-5:] if google_api_key else None}") + + elevenlabs_api_key = config.get('API', 'elevenlabs_api_key', fallback=None) + logging.debug( + f"Loaded elevenlabs API Key: {elevenlabs_api_key[:5]}...{elevenlabs_api_key[-5:] if elevenlabs_api_key else None}") # Models anthropic_model = config.get('API', 'anthropic_model', fallback='claude-3-sonnet-20240229') @@ -311,6 +315,30 @@ def load_and_log_configs(): # Local API Timeout local_api_timeout = config.get('Local-API', 'local_api_timeout', fallback='90') + # TTS Settings + # FIXME + tts_provider = config.get('TTS-Settings', 'default_tts_provider', fallback='openai') + tts_voice = config.get('TTS-Settings', 'default_tts_voice', fallback='shimmer') + # Open AI TTS + default_openai_tts_model = config.get('TTS-Settings', 'default_openai_tts_model', fallback='tts-1-hd') + default_openai_tts_voice = config.get('TTS-Settings', 'default_openai_tts_voice', fallback='shimmer') + default_openai_tts_speed = config.get('TTS-Settings', 'default_openai_tts_speed', fallback='1') + # Google TTS + # FIXME - FIX THESE DEFAULTS + default_google_tts_model = config.get('TTS-Settings', 'default_google_tts_model', fallback='en') + default_google_tts_voice = config.get('TTS-Settings', 'default_google_tts_voice', fallback='en') + default_google_tts_speed = config.get('TTS-Settings', 'default_google_tts_speed', fallback='1') + # ElevenLabs TTS + default_eleven_tts_model = config.get('TTS-Settings', 'default_eleven_tts_model', fallback='FIXME') + default_eleven_tts_voice = config.get('TTS-Settings', 'default_eleven_tts_voice', fallback='FIXME') + default_eleven_tts_language_code = config.get('TTS-Settings', 'default_eleven_tts_language_code', fallback='FIXME') + default_eleven_tts_voice_stability = config.get('TTS-Settings', 'default_eleven_tts_voice_stability', fallback='FIXME') + default_eleven_tts_voice_similiarity_boost = config.get('TTS-Settings', 'default_eleven_tts_voice_similiarity_boost', fallback='FIXME') + default_eleven_tts_voice_style = config.get('TTS-Settings', 'default_eleven_tts_voice_style', fallback='FIXME') + default_eleven_tts_voice_use_speaker_boost = config.get('TTS-Settings', 'default_eleven_tts_voice_use_speaker_boost', fallback='FIXME') + default_eleven_tts_output_format = config.get('TTS-Settings', 'default_eleven_tts_output_format', + fallback='mp3_44100_192') + return { 'api_keys': { 'anthropic': anthropic_api_key, @@ -329,6 +357,7 @@ def load_and_log_configs(): 'vllm': vllm_api_key, 'ollama': ollama_api_key, 'aphrodite': aphrodite_api_key, + 'elevenlabs': elevenlabs_api_key, 'custom_openai_api_key': custom_openai_api_key }, 'models': { @@ -382,15 +411,40 @@ def load_and_log_configs(): 'save_rag_chats': save_rag_chats, }, 'default_api': default_api, - 'local_api_timeout': local_api_timeout + 'local_api_timeout': local_api_timeout, + 'tts_settings': { + 'tts_provider': tts_provider, + 'tts_voice': tts_voice, + # OpenAI + 'default_openai_tts_voice': default_openai_tts_voice, + 'default_openai_tts_speed': default_openai_tts_speed, + 'default_openai_tts_model': default_openai_tts_model, + # Google + 'default_google_tts_model': default_google_tts_model, + 'default_google_tts_voice': default_google_tts_voice, + 'default_google_tts_speed': default_google_tts_speed, + # ElevenLabs + 'default_eleven_tts_model': default_eleven_tts_model, + 'default_eleven_tts_voice': default_eleven_tts_voice, + 'default_eleven_tts_language_code': default_eleven_tts_language_code, + 'default_eleven_tts_voice_stability': default_eleven_tts_voice_stability, + 'default_eleven_tts_voice_similiarity_boost': default_eleven_tts_voice_similiarity_boost, + 'default_eleven_tts_voice_style': default_eleven_tts_voice_style, + 'default_eleven_tts_voice_use_speaker_boost': default_eleven_tts_voice_use_speaker_boost, + 'default_eleven_tts_output_format': default_eleven_tts_output_format + # GPT Sovi-TTS + } } except Exception as e: logging.error(f"Error loading config: {str(e)}") return None + global_api_endpoints = ["anthropic", "cohere", "groq", "openai", "huggingface", "openrouter", "deepseek", "mistral", "google", "custom_openai_api", "llama", "ooba", "kobold", "tabby", "vllm", "ollama", "aphrodite"] +openai_tts_voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] + # Setup Default API Endpoint loaded_config_data = load_and_log_configs() default_api_endpoint = loaded_config_data['default_api'] @@ -416,9 +470,9 @@ def format_api_name(api): "aphrodite": "Aphrodite" } return name_mapping.get(api, api.title()) -print(f"Default API Endpoint: {default_api_endpoint}") +print(f"Default API Endpoint: {default_api_endpoint}") # # End of Config loading @@ -596,9 +650,6 @@ def create_download_directory(title): return session_path -import chardet -import logging - def safe_read_file(file_path): encodings = ['utf-8', 'utf-16', 'ascii', 'latin-1', 'iso-8859-1', 'cp1252', 'utf-8-sig'] @@ -725,7 +776,6 @@ def normalize_title(title, preserve_spaces=False): return title.strip('_') - def clean_youtube_url(url): parsed_url = urlparse(url) query_params = parse_qs(parsed_url.query) @@ -820,9 +870,6 @@ def get_db_config(): 'elasticsearch_port': config.getint('Database', 'elasticsearch_port', fallback=9200) } - - - # # End of DB Config Loading ####################################################################################################################### @@ -838,6 +885,7 @@ def format_text_with_line_breaks(text): # Track temp files for cleanup temp_files = [] + temp_file_paths = [] def save_temp_file(file): diff --git a/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py b/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py index 2f2daee7d..2324e0088 100644 --- a/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py +++ b/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py @@ -33,6 +33,8 @@ from playwright.async_api import async_playwright import requests import trafilatura +from tqdm import tqdm + # # Import Local from App_Function_Libraries.DB.DB_Manager import ingest_article_to_db @@ -64,19 +66,27 @@ def get_page_title(url: str) -> str: async def scrape_article(url: str, custom_cookies: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: async def fetch_html(url: str) -> str: async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) - context = await browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" - ) - if custom_cookies: - await context.add_cookies(custom_cookies) - page = await context.new_page() - await page.goto(url) - await page.wait_for_load_state("networkidle") - content = await page.content() - await browser.close() - log_counter("html_fetched", labels={"url": url}) - return content + browser = await p.chromium.launch(headless=False) + try: + context = await browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + #viewport = {"width": 1280, "height": 720}, + #java_script_enabled = True + ) + if custom_cookies: + await context.add_cookies(custom_cookies) + page = await context.new_page() + try: + await page.goto(url, wait_until="domcontentloaded") + await page.wait_for_load_state("networkidle", timeout=30000) + content = await page.content() + log_counter("html_fetched", labels={"url": url}) + return content + except Exception as e: + logging.error(f"Error fetching HTML for {url}: {e}") + return "" + finally: + await browser.close() def extract_article_data(html: str, url: str) -> dict: # FIXME - Add option for extracting comments/tables/images @@ -136,7 +146,6 @@ def convert_html_to_markdown(html: str) -> str: log_histogram("article_content_length", len(article_data['content']), labels={"url": url}) return article_data - # FIXME - Add keyword integration/tagging async def scrape_and_summarize_multiple( urls: str, @@ -156,6 +165,9 @@ async def scrape_and_summarize_multiple( results = [] errors = [] + # Create a tqdm progress bar + progress_bar = tqdm(total=len(urls_list), desc="Scraping and Summarizing") + # Loop over each URL to scrape and optionally summarize for i, url in enumerate(urls_list): custom_title = custom_titles[i] if i < len(custom_titles) else None @@ -206,6 +218,12 @@ async def scrape_and_summarize_multiple( error_message = f"Error processing URL {i + 1} ({url}): {str(e)}" errors.append(error_message) logging.error(error_message, exc_info=True) + finally: + # Update the progress bar + progress_bar.update(1) + + # Close the progress bar + progress_bar.close() if errors: logging.error("\n".join(errors)) @@ -804,10 +822,10 @@ def format_content_with_metadata( metadata.update(additional_metadata) formatted_content = f"""{ContentMetadataHandler.METADATA_START} -{json.dumps(metadata, indent=2)} -{ContentMetadataHandler.METADATA_END} - -{content}""" + {json.dumps(metadata, indent=2)} + {ContentMetadataHandler.METADATA_END} + + {content}""" return formatted_content diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6646428be..2f7d92bf6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,7 @@ Contributing ============ -Contributions are welcome, and they are greatly appreciated! Every -little bit helps, and credit will always be given. +Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. You can contribute in many ways: diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 97bed007e..81c96aeda 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -8,6 +8,17 @@ This page is to document efforts towards creating a 'research' agent workflow fo https://github.com/assafelovic/gpt-researcher https://arxiv.org/abs/2411.15114 https://journaliststudio.google.com/pinpoint/about/ +https://blog.google/products/gemini/google-gemini-deep-research/ +https://github.com/neuml/annotateai +one2manny + — +Today at 12:43 AM +A great way to make studying more efficient and convenient is to take a digital PDF textbook, split it into separate files for each chapter, and organize them individually. I then create a dedicated notebook for each chapter, treating it as a focused single source. From there, I convert each chapter into an audio format, like a podcast. This approach makes it easy to study while commuting, relaxing in bed with your eyes closed, or at any time when reading isn’t practical. + +I also recommend creating a study guide for each chapter, fully breaking down key concepts and definitions. For more complex topics, the “explain like I’m 5” method works wonders—it simplifies challenging ideas into digestible explanations. + +To take this further, incorporate a Personal Knowledge Management (PKM) system into your routine. Apps like Obsidian are perfect for this, with their flexible folder structures and Markdown formatting. I optimize my AI outputs for Markdown so I can copy, paste, and organize them into clean, structured notes. This ensures your materials are not only well-organized but also easy to access and build on later. A solid PKM system is invaluable for managing knowledge and staying on top of your studies! + diff --git a/Docs/Design/Storytelling_and_Creation.md b/Docs/Design/Storytelling_and_Creation.md index 4bae7ab7d..644429cf8 100644 --- a/Docs/Design/Storytelling_and_Creation.md +++ b/Docs/Design/Storytelling_and_Creation.md @@ -7,6 +7,7 @@ https://github.com/mhgolkar/Arrow https://github.com/neph1/LlamaTale https://jianzongwu.github.io/projects/diffsensei/ https://github.com/jianzongwu/DiffSensei +https://github.com/joonspk-research/generative_agents ### Storytelling diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index ee19af526..011209b3e 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -56,8 +56,17 @@ https://astro.new/latest/ https://github.com/lobehub/lobe-chat https://www.nngroup.com/videos/the-danger-of-defaults/ https://writings.stephenwolfram.com/2024/12/useful-to-the-point-of-being-revolutionary-introducing-wolfram-notebook-assistant/ - - - +https://en.wikipedia.org/wiki/Template:Google_payment_apps +https://github.com/stackblitz-labs/bolt.diy +https://github.com/Haervwe/open-webui-tools +https://interactjs.io/ +https://markwhen.com/ + +Prompt Engineering page: +- 3 Sections + - Prompt + - Values + - Response +- Looks like Claude Workshop UI window diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 336144f91..44c362a1a 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -20,6 +20,15 @@ https://arxiv.org/abs/2412.05185 https://huggingface.co/collections/OpenGVLab/internvl-25-673e1019b66e2218f68d7c1c https://huggingface.co/papers/2412.07626 +https://huggingface.co/AI-Safeguard/Ivy-VL-llava +https://github.com/matatonic/openedai-vision +https://github.com/breezedeus/Pix2Text +https://github.com/Tencent/HunyuanVideo +https://github.com/huggingface/blog/blob/main/smolvlm.md +https://huggingface.co/tencent/HunyuanVideo +https://aivideo.hunyuan.tencent.com/ +https://github.com/deepseek-ai/DeepSeek-VL2 +https://arxiv.org/abs/2409.17146 diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index ee3cab7d2..5e2234ce5 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -31,6 +31,7 @@ https://arxiv.org/abs/2412.05265 - https://huggingface.co/papers/2411.15862 https://arxiv.org/abs/2411.19865 https://arxiv.org/abs/2412.06769 +- https://arxiv.org/abs/2412.01113 ### Test-Time Compute diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 76a4fa48d..91b3bf542 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -14,7 +14,7 @@ ---------------------------------------------------------------------------------------------------------------- - +Have LLMs play Social deception games Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 Benchmarking with distilabel diff --git a/Docs/VLM.md b/Docs/VLM.md deleted file mode 100644 index d5fd34bbd..000000000 --- a/Docs/VLM.md +++ /dev/null @@ -1,11 +0,0 @@ -# VLM - - - -https://github.com/matatonic/openedai-vision -https://github.com/breezedeus/Pix2Text -https://github.com/Tencent/HunyuanVideo -https://github.com/huggingface/blog/blob/main/smolvlm.md -https://huggingface.co/tencent/HunyuanVideo -https://aivideo.hunyuan.tencent.com/ - diff --git a/Server_API/Dockerfile b/Server_API/Dockerfile deleted file mode 100644 index eb5075d0c..000000000 --- a/Server_API/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.11 - -WORKDIR /code - -COPY ./requirements.txt /code/requirements.txt -RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt - -COPY ./app /code/app - -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] \ No newline at end of file diff --git a/Server_API/app/api/v1/endpoints/video_processing.py b/Server_API/app/api/v1/endpoints/video_processing.py deleted file mode 100644 index 6a1915282..000000000 --- a/Server_API/app/api/v1/endpoints/video_processing.py +++ /dev/null @@ -1,25 +0,0 @@ -from fastapi import APIRouter, BackgroundTasks, HTTPException -from typing import List, Optional -from Server_API.app.services.video_processing_service import process_video_task - -router = APIRouter() - -# @router.post("/process-video", summary="Process a video", description="Download, transcribe, and summarize a video from the given URL.") -# async def process_video( -# url: str = Query(..., description="URL of the video to process"), -# whisper_model: str = Query(..., description="Whisper model to use for transcription"), -# custom_prompt: Optional[str] = Query(None, description="Custom prompt for summarization"), -# api_name: str = Query(..., description="Name of the API to use for summarization"), -# api_key: str = Query(..., description="API key for the summarization service"), -# keywords: List[str] = Query(default=[], description="Keywords to associate with the video"), -# diarize: bool = Query(False, description="Whether to perform speaker diarization"), -# start_time: Optional[str] = Query(None, description="Start time for processing (format: HH:MM:SS)"), -# end_time: Optional[str] = Query(None, description="End time for processing (format: HH:MM:SS)"), -# include_timestamps: bool = Query(True, description="Whether to include timestamps in the transcription"), -# keep_original_video: bool = Query(False, description="Whether to keep the original video file after processing"), -# background_tasks: BackgroundTasks = BackgroundTasks() -# ): -# task_id = f"task_{url.replace('://', '_').replace('/', '_')}" -# background_tasks.add_task(process_video_task, url, whisper_model, custom_prompt, api_name, api_key, -# keywords, diarize, start_time, end_time, include_timestamps, keep_original_video) -# return {"task_id": task_id, "message": "Video processing started"} \ No newline at end of file diff --git a/Server_API/app/main.py b/Server_API/app/main.py index 578c50127..7b8a6591b 100644 --- a/Server_API/app/main.py +++ b/Server_API/app/main.py @@ -1,11 +1,7 @@ from fastapi import FastAPI -from Server_API.app.api.v1.endpoints import video_processing -from Server_API.app.core.exceptions import setup_exception_handlers app = FastAPI(title="TLDW API", version="1.0.0") -setup_exception_handlers(app) -app.include_router(video_processing.router, prefix="/api/v1") @app.get("/") async def root(): - return {"message": "Welcome to the TLDW API"} + return {"message": "Welcome to the tldw API"} diff --git a/summarize.py b/summarize.py index b2d742e0c..054a791ec 100644 --- a/summarize.py +++ b/summarize.py @@ -47,6 +47,9 @@ ############# # Global variables setup # FIXME + +running_in_debug_mode = False + custom_prompt_summarize_bulleted_notes = (""" You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST] **Bulleted Note Creation Guidelines** From 466413297d5583c0813146b81ef6db5fab06d3b6 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 13 Dec 2024 21:16:15 -0800 Subject: [PATCH 10/88] Update Article_Extractor_Lib.py --- .../Web_Scraping/Article_Extractor_Lib.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py b/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py index 2324e0088..4a212b3ce 100644 --- a/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py +++ b/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py @@ -44,6 +44,8 @@ # Function Definitions # +web_scraping_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + ################################################################# # # Scraping-related functions: @@ -66,10 +68,10 @@ def get_page_title(url: str) -> str: async def scrape_article(url: str, custom_cookies: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: async def fetch_html(url: str) -> str: async with async_playwright() as p: - browser = await p.chromium.launch(headless=False) + browser = await p.chromium.launch(headless=True) try: context = await browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + user_agent=web_scraping_user_agent, #viewport = {"width": 1280, "height": 720}, #java_script_enabled = True ) From edf85584e65980e02594641edb7dc1488a678cbf Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 13 Dec 2024 23:19:32 -0800 Subject: [PATCH 11/88] f --- .../Gradio_UI/Workflows_tab.py | 2 +- .../Workflows/Workflows.json | 287 +++++++++++------- App_Function_Libraries/Workflows/Workflows.py | 193 ++++++++++++ App_Function_Libraries/Workflows/__init__.py | 0 Docs/Design/Structured_Outputs.md | 14 + 5 files changed, 379 insertions(+), 117 deletions(-) rename {Helper_Scripts => App_Function_Libraries}/Workflows/Workflows.json (81%) create mode 100644 App_Function_Libraries/Workflows/Workflows.py create mode 100644 App_Function_Libraries/Workflows/__init__.py diff --git a/App_Function_Libraries/Gradio_UI/Workflows_tab.py b/App_Function_Libraries/Gradio_UI/Workflows_tab.py index 5c911d290..05530af21 100644 --- a/App_Function_Libraries/Gradio_UI/Workflows_tab.py +++ b/App_Function_Libraries/Gradio_UI/Workflows_tab.py @@ -20,7 +20,7 @@ # Functions: # Load workflows from a JSON file -json_path = Path('./Helper_Scripts/Workflows/Workflows.json') +json_path = Path('./App_Function_Libraries/Workflows/Workflows.json') with json_path.open('r') as f: workflows = json.load(f) diff --git a/Helper_Scripts/Workflows/Workflows.json b/App_Function_Libraries/Workflows/Workflows.json similarity index 81% rename from Helper_Scripts/Workflows/Workflows.json rename to App_Function_Libraries/Workflows/Workflows.json index e10f8436f..d82b4942b 100644 --- a/Helper_Scripts/Workflows/Workflows.json +++ b/App_Function_Libraries/Workflows/Workflows.json @@ -11,22 +11,6 @@ "Why did this final event happen?" ] }, - { - "name": "Summarization", - "context": "A process of condensing information into a concise form while retaining the main points. Start with the text to be summarized and provide a brief summary. Subsequent messages should refine the summary to make it more concise and focused.", - "prompts": [ - "Summarize the following text:", - "Refine the summary to be more concise:" - ] - }, - { - "name": "Question Answering", - "context": "A task that involves answering questions based on a given context or information. Start with the question and provide an answer based on the context. Subsequent messages should provide more detailed answers or explanations.", - "prompts": [ - "Answer the following question based on the context:", - "Provide a more detailed answer:" - ] - }, { "name": "Brainstorming", "context": "A creative technique for generating ideas and solutions through group discussion. Start with a topic or problem statement and generate initial ideas. Subsequent messages should expand on the ideas, refine them, and summarize the final set of ideas.", @@ -47,37 +31,28 @@ ] }, { - "name": "Product Description", - "context": "A detailed description of a product, highlighting its features, benefits, and value proposition. Start with the product details and write an initial description. Subsequent messages should enhance the description, simplify it, and optimize it for search engines.", + "name": "Concept Mastery", + "context": "A process for understanding a topic deeply and comprehensively. Start by identifying the core concepts, then explore subtopics, examples, and applications. Subsequent steps involve clarifying difficult areas and applying knowledge in practice scenarios.", "prompts": [ - "Write an initial product description based on the provided details:", - "Enhance the description with key features and benefits:", - "Simplify and make the description more engaging:", - "Optimize the description for SEO:" + "Identify the main concepts or themes of the topic:", + "Explore the subtopics and provide examples for each:", + "Clarify any areas or concepts you find difficult to understand:", + "Develop a practical application or scenario using the knowledge:", + "Review and summarize the key points of the topic:" ] }, { - "name": "SWOT Analysis", - "context": "A strategic planning tool used to identify the strengths, weaknesses, opportunities, and threats of a subject. Start with the subject of analysis and identify its strengths. Subsequent messages should cover weaknesses, opportunities, threats, and summarize the analysis.", + "name": "Conflict Resolution", + "context": "A process of addressing and resolving conflicts between individuals or groups. Start with identifying the conflict and parties involved, then explore solutions or compromises. Subsequent messages should plan a mediation session, evaluate outcomes, and ensure satisfaction.", "prompts": [ - "Identify the strengths of the subject:", - "Identify the weaknesses of the subject:", - "Identify potential opportunities for the subject:", - "Identify potential threats to the subject:", - "Summarize the SWOT analysis:" + "Describe the conflict and the parties involved:", + "Identify the underlying causes of the conflict:", + "Explore potential solutions or compromises:", + "Plan a mediation or discussion session to address the conflict:", + "Evaluate the outcomes and ensure all parties are satisfied:" ] }, { - "name": "User Story Creation", - "context": "A technique used in agile software development to capture user requirements from a user's perspective. Start with the user role and a brief description of the feature. Subsequent messages should detail user needs, acceptance criteria, and prioritize the user story.", - "prompts": [ - "Draft an initial user story for the feature:", - "Clarify the user needs and expected outcomes:", - "Refine the user story to ensure clarity and completeness:", - "Review and prioritize the user story for development:" - ] - }, - { "name": "Content Outline Creation", "context": "A structured plan that outlines the main topics and subtopics of a piece of content. Start with the content type and purpose, then list the main topics. Subsequent messages should detail subtopics, key points, and refine the outline for logical flow.", "prompts": [ @@ -88,13 +63,14 @@ ] }, { - "name": "Risk Assessment", - "context": "An evaluation of potential risks associated with a project or decision. Start with the project or decision and identify risks. Subsequent messages should analyze impact, likelihood, suggest mitigation strategies, and summarize the assessment.", + "name": "Creative Writing", + "context": "A process of writing fiction or poetry that involves generating ideas, drafting, revising, and editing. Start with a theme or prompt and brainstorm ideas. Subsequent messages should outline the plot, write a draft, refine characters or imagery, and polish the final version.", "prompts": [ - "Identify potential risks associated with the project/decision:", - "Analyze the impact and likelihood of each risk:", - "Suggest mitigation strategies for high-priority risks:", - "Summarize the risk assessment and proposed actions:" + "Generate ideas for a story or poem based on the following theme:", + "Outline the main plot or structure of the story:", + "Write the first draft of the opening scene or stanza:", + "Refine the draft by focusing on character development or imagery:", + "Edit and polish the final version for clarity and impact:" ] }, { @@ -117,6 +93,39 @@ "Make a recommendation based on the evaluation:" ] }, + { + "name": "Event Planning", + "context": "A process of organizing and coordinating an event, such as a conference, workshop, or party. Start with defining the purpose and goals of the event, then plan logistics and promotion. Subsequent messages should outline activities, develop a contingency plan, and ensure a successful event.", + "prompts": [ + "Outline the purpose and goals of the event:", + "List the key activities and sessions for the event:", + "Plan the logistics, including venue, catering, and AV setup:", + "Develop a promotional strategy to attract attendees:", + "Create a contingency plan for potential issues:" + ] + }, + { + "name": "Exam Preparation", + "context": "A systematic process to prepare effectively for an exam. Start by outlining the topics to cover, then assess strengths and weaknesses. Subsequent steps focus on targeted studying, practicing, and reviewing.", + "prompts": [ + "List all the topics or areas covered by the exam:", + "Identify your strengths and weaknesses in these topics:", + "Create a study plan prioritizing weaker areas and allocating time for review:", + "Practice with sample questions or problems in key areas:", + "Review your progress and refine your study approach as needed:" + ] + }, + { + "name": "Flaschards: Anki Flashcard Generation (Conceptual)", + "context": "A process to create meaningful Anki flashcards that help with conceptual understanding and recall. Start by identifying key concepts from the source material, then develop question-and-answer pairs that test understanding, refine them for clarity, and finalize the cards.", + "prompts": [ + "Identify the main concepts or topics from the source material:", + "Generate initial Q/A pairs that test understanding and recall of these concepts:", + "Refine the Q/A pairs for clarity, ensuring they are concise and easy to memorize:", + "Check the Q/A pairs for conceptual accuracy and adjust them if needed:", + "Finalize the Anki-ready flashcards for use:" + ] + }, { "name": "Goal Setting and Action Planning", "context": "A process of defining goals, breaking them down into actionable steps, and creating a plan for achievement. Start with the main goal and list steps. Subsequent messages should set timelines, identify resources, and review the plan.", @@ -129,28 +138,17 @@ ] }, { - "name": "User Experience (UX) Design", - "context": "A process of designing products or services that provide meaningful and relevant experiences to users. Start with user needs and pain points, then brainstorm solutions. Subsequent messages should evaluate solutions, create prototypes, and plan for user testing.", + "name": "Job Interview Preparation", + "context": "A process of researching, practicing, and preparing for a job interview to increase the chances of success. Start with researching the company and common interview questions. Subsequent messages should prepare answers, review the resume, and develop questions for the interviewer.", "prompts": [ - "Describe the user needs and pain points:", - "Brainstorm potential design solutions to address these needs:", - "Evaluate the feasibility and impact of each design solution:", - "Create a prototype or mockup based on the selected design:", - "Plan for user testing and feedback collection:" + "Research the company and summarize key information:", + "Identify the most common interview questions for this role:", + "Prepare answers to behavioral and situational questions:", + "Review your resume and highlight relevant experiences:", + "Develop questions to ask the interviewer:" ] }, { - "name": "Root Cause Analysis", - "context": "A methodical approach to identifying the underlying cause of a problem or issue. Start with the problem statement and list possible causes. Subsequent messages should use the '5 Whys' technique to drill down to the root cause and propose solutions.", - "prompts": [ - "Describe the problem or issue in detail:", - "Identify possible causes of the problem:", - "Use the '5 Whys' technique to drill down to the root cause:", - "Propose potential solutions to address the root cause:", - "Develop an action plan to implement the solution and monitor its effectiveness:" - ] - }, -{ "name": "Literature Review", "context": "A critical evaluation of existing research and literature on a specific topic. Start with the research question or topic and summarize key findings. Subsequent messages should identify gaps, compare methodologies, synthesize findings, and suggest future research directions.", "prompts": [ @@ -160,17 +158,6 @@ "Synthesize the findings into a cohesive narrative:" ] }, - { - "name": "Creative Writing", - "context": "A process of writing fiction or poetry that involves generating ideas, drafting, revising, and editing. Start with a theme or prompt and brainstorm ideas. Subsequent messages should outline the plot, write a draft, refine characters or imagery, and polish the final version.", - "prompts": [ - "Generate ideas for a story or poem based on the following theme:", - "Outline the main plot or structure of the story:", - "Write the first draft of the opening scene or stanza:", - "Refine the draft by focusing on character development or imagery:", - "Edit and polish the final version for clarity and impact:" - ] - }, { "name": "Marketing Campaign Planning", "context": "A strategic approach to promoting a product, service, or brand through coordinated marketing efforts. Start with defining the target audience and goals, then brainstorm creative ideas. Subsequent messages should outline channels, set timelines, assign responsibilities, and evaluate performance.", @@ -183,69 +170,86 @@ ] }, { - "name": "Event Planning", - "context": "A process of organizing and coordinating an event, such as a conference, workshop, or party. Start with defining the purpose and goals of the event, then plan logistics and promotion. Subsequent messages should outline activities, develop a contingency plan, and ensure a successful event.", + "name": "Personal Reflection and Growth", + "context": "A process of self-assessment, learning, and goal-setting to achieve personal growth and development. Start with reflecting on recent experiences and identifying key learnings. Subsequent messages should set personal goals, identify habits to change, develop an action plan, and review progress.", "prompts": [ - "Outline the purpose and goals of the event:", - "List the key activities and sessions for the event:", - "Plan the logistics, including venue, catering, and AV setup:", - "Develop a promotional strategy to attract attendees:", - "Create a contingency plan for potential issues:" + "Reflect on your recent experiences and identify key learnings:", + "Set personal goals based on your reflections:", + "Identify any habits or behaviors you want to change or improve:", + "Develop an action plan to work towards your goals:", + "Review your progress periodically and adjust your plan as needed:" ] }, { - "name": "Job Interview Preparation", - "context": "A process of researching, practicing, and preparing for a job interview to increase the chances of success. Start with researching the company and common interview questions. Subsequent messages should prepare answers, review the resume, and develop questions for the interviewer.", + "name": "Product Launch", + "context": "A strategic process of introducing a new product or service to the market. Start with defining the unique value proposition and target market, then plan development milestones and go-to-market strategy. Subsequent messages should outline the launch event, monitor performance, and gather feedback.", "prompts": [ - "Research the company and summarize key information:", - "Identify the most common interview questions for this role:", - "Prepare answers to behavioral and situational questions:", - "Review your resume and highlight relevant experiences:", - "Develop questions to ask the interviewer:" + "Define the product's unique value proposition and target market:", + "Outline the product development milestones and key deliverables:", + "Develop a go-to-market strategy, including pricing, distribution, and promotion:", + "Plan the launch event or activities:", + "Monitor the launch performance and gather customer feedback:" ] }, { - "name": "Conflict Resolution", - "context": "A process of addressing and resolving conflicts between individuals or groups. Start with identifying the conflict and parties involved, then explore solutions or compromises. Subsequent messages should plan a mediation session, evaluate outcomes, and ensure satisfaction.", + "name": "Product Description", + "context": "A detailed description of a product, highlighting its features, benefits, and value proposition. Start with the product details and write an initial description. Subsequent messages should enhance the description, simplify it, and optimize it for search engines.", "prompts": [ - "Describe the conflict and the parties involved:", - "Identify the underlying causes of the conflict:", - "Explore potential solutions or compromises:", - "Plan a mediation or discussion session to address the conflict:", - "Evaluate the outcomes and ensure all parties are satisfied:" + "Write an initial product description based on the provided details:", + "Enhance the description with key features and benefits:", + "Simplify and make the description more engaging:", + "Optimize the description for SEO:" ] }, { - "name": "Time Management", - "context": "A set of practices and tools to manage time effectively and prioritize tasks. Start with listing tasks and activities, then prioritize based on urgency and importance. Subsequent messages should create a daily schedule, identify time-wasters, and review progress.", + "name": "Question Answering", + "context": "A task that involves answering questions based on a given context or information. Start with the question and provide an answer based on the context. Subsequent messages should provide more detailed answers or explanations.", "prompts": [ - "List all the tasks and activities you need to complete this week:", - "Prioritize the tasks based on urgency and importance:", - "Create a daily schedule, allocating time blocks for each task:", - "Identify any potential time-wasters or distractions:", - "Review your progress at the end of the week and adjust your approach:" + "Answer the following question based on the context:", + "Provide a more detailed answer:" ] }, { - "name": "Product Launch", - "context": "A strategic process of introducing a new product or service to the market. Start with defining the unique value proposition and target market, then plan development milestones and go-to-market strategy. Subsequent messages should outline the launch event, monitor performance, and gather feedback.", + "name": "Quiz Prep: Comprehension-Focused", + "context": "A process to prepare for quizzes by thoroughly understanding the material and practicing application. Start by outlining quiz topics, then focus on comprehension, application, and final review.", "prompts": [ - "Define the product's unique value proposition and target market:", - "Outline the product development milestones and key deliverables:", - "Develop a go-to-market strategy, including pricing, distribution, and promotion:", - "Plan the launch event or activities:", - "Monitor the launch performance and gather customer feedback:" + "List the topics or concepts that the quiz will cover:", + "Summarize the main ideas or explanations for each topic in your own words:", + "Practice applying these ideas to examples, questions, or problems:", + "Identify any concepts that are unclear and revisit them for clarification:", + "Review and test your understanding with practice questions:" ] }, { - "name": "Personal Reflection and Growth", - "context": "A process of self-assessment, learning, and goal-setting to achieve personal growth and development. Start with reflecting on recent experiences and identifying key learnings. Subsequent messages should set personal goals, identify habits to change, develop an action plan, and review progress.", + "name": "Quiz Prep: Rapid Recall", + "context": "A process for efficiently preparing for short quizzes by focusing on key facts, reviewing, and testing recall. Start by identifying quiz topics, then focus on memorizing key points, practicing recall, and assessing readiness.", "prompts": [ - "Reflect on your recent experiences and identify key learnings:", - "Set personal goals based on your reflections:", - "Identify any habits or behaviors you want to change or improve:", - "Develop an action plan to work towards your goals:", - "Review your progress periodically and adjust your plan as needed:" + "List the main topics or concepts the quiz will cover:", + "Identify the key facts, definitions, or formulas for each topic:", + "Create flashcards or a memory aid to reinforce these key points:", + "Practice recalling the information without looking at notes:", + "Take a short self-quiz to evaluate your readiness and fill any gaps:" + ] + }, + { + "name": "Risk Assessment", + "context": "An evaluation of potential risks associated with a project or decision. Start with the project or decision and identify risks. Subsequent messages should analyze impact, likelihood, suggest mitigation strategies, and summarize the assessment.", + "prompts": [ + "Identify potential risks associated with the project/decision:", + "Analyze the impact and likelihood of each risk:", + "Suggest mitigation strategies for high-priority risks:", + "Summarize the risk assessment and proposed actions:" + ] + }, + { + "name": "Root Cause Analysis", + "context": "A methodical approach to identifying the underlying cause of a problem or issue. Start with the problem statement and list possible causes. Subsequent messages should use the '5 Whys' technique to drill down to the root cause and propose solutions.", + "prompts": [ + "Describe the problem or issue in detail:", + "Identify possible causes of the problem:", + "Use the '5 Whys' technique to drill down to the root cause:", + "Propose potential solutions to address the root cause:", + "Develop an action plan to implement the solution and monitor its effectiveness:" ] }, { @@ -258,5 +262,56 @@ "Refine the pitch by addressing potential objections:", "Practice delivering the pitch with confidence and clarity:" ] + }, + { + "name": "Summarization", + "context": "A process of condensing information into a concise form while retaining the main points. Start with the text to be summarized and provide a brief summary. Subsequent messages should refine the summary to make it more concise and focused.", + "prompts": [ + "Summarize the following text:", + "Refine the summary to be more concise:" + ] + }, + { + "name": "SWOT Analysis", + "context": "A strategic planning tool used to identify the strengths, weaknesses, opportunities, and threats of a subject. Start with the subject of analysis and identify its strengths. Subsequent messages should cover weaknesses, opportunities, threats, and summarize the analysis.", + "prompts": [ + "Identify the strengths of the subject:", + "Identify the weaknesses of the subject:", + "Identify potential opportunities for the subject:", + "Identify potential threats to the subject:", + "Summarize the SWOT analysis:" + ] + }, + { + "name": "Time Management", + "context": "A set of practices and tools to manage time effectively and prioritize tasks. Start with listing tasks and activities, then prioritize based on urgency and importance. Subsequent messages should create a daily schedule, identify time-wasters, and review progress.", + "prompts": [ + "List all the tasks and activities you need to complete this week:", + "Prioritize the tasks based on urgency and importance:", + "Create a daily schedule, allocating time blocks for each task:", + "Identify any potential time-wasters or distractions:", + "Review your progress at the end of the week and adjust your approach:" + ] + }, + { + "name": "User Story Creation", + "context": "A technique used in agile software development to capture user requirements from a user's perspective. Start with the user role and a brief description of the feature. Subsequent messages should detail user needs, acceptance criteria, and prioritize the user story.", + "prompts": [ + "Draft an initial user story for the feature:", + "Clarify the user needs and expected outcomes:", + "Refine the user story to ensure clarity and completeness:", + "Review and prioritize the user story for development:" + ] + }, + { + "name": "User Experience (UX) Design", + "context": "A process of designing products or services that provide meaningful and relevant experiences to users. Start with user needs and pain points, then brainstorm solutions. Subsequent messages should evaluate solutions, create prototypes, and plan for user testing.", + "prompts": [ + "Describe the user needs and pain points:", + "Brainstorm potential design solutions to address these needs:", + "Evaluate the feasibility and impact of each design solution:", + "Create a prototype or mockup based on the selected design:", + "Plan for user testing and feedback collection:" + ] } ] diff --git a/App_Function_Libraries/Workflows/Workflows.py b/App_Function_Libraries/Workflows/Workflows.py new file mode 100644 index 000000000..97d50adb4 --- /dev/null +++ b/App_Function_Libraries/Workflows/Workflows.py @@ -0,0 +1,193 @@ +# Workflows.py +# +######################################### +# Workflow Library +# This library is used to facilitate chained prompt workflows +# +#### +#################### +# Function Categories +# +# Fixme +# +# +#################### +# Function List +# +# 1. FIXME +# +#################### +# +# Import necessary libraries +import json +import logging +from pathlib import Path +from typing import List, Dict, Tuple, Optional +# +# Non-Local Imports +# +# Local Imports +from App_Function_Libraries.Chat.Chat_Functions import chat +from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram +# +####################################################################################################################### +# +# Function Definitions + +# Load workflows from a JSON file +json_path = Path('./App_Function_Libraries/Workflows/Workflows.json') + +# Load workflows from a JSON file +def load_workflows(json_path: str = './App_Function_Libraries/Workflows/Workflows.json') -> List[Dict]: + with Path(json_path).open('r') as f: + return json.load(f) + +# Initialize a workflow +def initialize_workflow(workflow_name: str, workflows: List[Dict]) -> Tuple[Dict, str, List[Tuple[Optional[str], str]]]: + selected_workflow = next((wf for wf in workflows if wf['name'] == workflow_name), None) + if selected_workflow: + num_prompts = len(selected_workflow['prompts']) + context = selected_workflow.get('context', '') + first_prompt = selected_workflow['prompts'][0] + initial_chat = [(None, f"{first_prompt}")] + workflow_state = {"current_step": 0, "max_steps": num_prompts, "conversation_id": None} + logging.info(f"Initializing workflow: {workflow_name} with {num_prompts} steps") + return workflow_state, context, initial_chat + else: + logging.error(f"Selected workflow not found: {workflow_name}") + return {"current_step": 0, "max_steps": 0, "conversation_id": None}, "", [] + + +# Process a workflow step +def process_workflow_step( + message: str, + history: List[Tuple[Optional[str], str]], + context: str, + workflow_name: str, + workflows: List[Dict], + workflow_state: Dict, + api_endpoint: Optional[str] = None, + api_key: Optional[str] = None, + save_conv: bool = False, + temp: float = 0.7, + system_message: Optional[str] = None, + media_content: Dict = {}, + selected_parts: List[str] = [] +) -> Tuple[List[Tuple[Optional[str], str]], Dict, bool]: + logging.info(f"Process workflow step called with message: {message}") + logging.info(f"Current workflow state: {workflow_state}") + + try: + selected_workflow = next((wf for wf in workflows if wf['name'] == workflow_name), None) + if not selected_workflow: + logging.error(f"Selected workflow not found: {workflow_name}") + return history, workflow_state, True + + current_step = workflow_state["current_step"] + max_steps = workflow_state["max_steps"] + + logging.info(f"Current step: {current_step}, Max steps: {max_steps}") + + if current_step >= max_steps: + logging.info("Workflow completed") + return history, workflow_state, False + + prompt = selected_workflow['prompts'][current_step] + full_message = f"{context}\n\nStep {current_step + 1}: {prompt}\nUser: {message}" + + logging.info(f"Preparing to process message: {full_message[:100]}...") + + # Use the existing chat function + bot_message = chat( + full_message, history, media_content, selected_parts, + api_endpoint, api_key, prompt, temp, system_message + ) + + logging.info(f"Received bot_message: {bot_message[:100]}...") + + new_history = history + [(message, bot_message)] + next_step = current_step + 1 + new_workflow_state = { + "current_step": next_step, + "max_steps": max_steps, + "conversation_id": workflow_state["conversation_id"] + } + + if next_step >= max_steps: + logging.info("Workflow completed after this step") + return new_history, new_workflow_state, False + else: + next_prompt = selected_workflow['prompts'][next_step] + new_history.append((None, f"Step {next_step + 1}: {next_prompt}")) + logging.info(f"Moving to next step: {next_step}") + return new_history, new_workflow_state, True + + except Exception as e: + logging.error(f"Error in process_workflow_step: {str(e)}") + return history, workflow_state, True + + +# Main function to run a workflow +def run_workflow( + workflow_name: str, + initial_context: str = "", + api_endpoint: Optional[str] = None, + api_key: Optional[str] = None, + save_conv: bool = False, + temp: float = 0.7, + system_message: Optional[str] = None, + media_content: Dict = {}, + selected_parts: List[str] = [] +) -> List[Tuple[Optional[str], str]]: + workflows = load_workflows() + workflow_state, context, history = initialize_workflow(workflow_name, workflows) + + # Combine the initial_context with the workflow's context + combined_context = f"{initial_context}\n\n{context}".strip() + + while True: + user_input = input("Your input (or 'quit' to exit): ") + if user_input.lower() == 'quit': + break + + history, workflow_state, continue_workflow = process_workflow_step( + user_input, history, combined_context, workflow_name, workflows, workflow_state, + api_endpoint, api_key, save_conv, temp, system_message, media_content, selected_parts + ) + + for _, message in history[-2:]: # Print the last two messages (user input and bot response) + print(message) + + if not continue_workflow: + print("Workflow completed.") + break + + return history + +# Example usage +# if __name__ == "__main__": +# workflow_name = "Example Workflow" +# initial_context = "This is an example context." +# +# final_history = run_workflow( +# workflow_name, +# initial_context, +# api_endpoint="your_api_endpoint", +# api_key="your_api_key", +# save_conv=True, +# temp=0.7, +# system_message="You are a helpful assistant guiding through a workflow.", +# media_content={}, +# selected_parts=[] +# ) +# +# print("Final conversation history:") +# for user_message, bot_message in final_history: +# if user_message: +# print(f"User: {user_message}") +# print(f"Bot: {bot_message}") +# print() + +# +# End of Workflows.py +####################################################################################################################### diff --git a/App_Function_Libraries/Workflows/__init__.py b/App_Function_Libraries/Workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Docs/Design/Structured_Outputs.md b/Docs/Design/Structured_Outputs.md index becc52b40..2a5ca8f6c 100644 --- a/Docs/Design/Structured_Outputs.md +++ b/Docs/Design/Structured_Outputs.md @@ -4,6 +4,20 @@ This page serves as documentation regarding the structured outputs within tldw and provides context/justification for the decisions made within the module. ## Structured Outputs +- Structured outputs are useful for generating structured data from unstructured text. + +### Use Cases +1. File Creation + - .ical file (calendar file creation) + - .json file (structured data) + - .csv file (Anki Flash cards + structured data) + - .xml file + - .yaml file + - .toml file + - +2. Data Extraction + + ### Link Dump: https://github.com/yobix-ai/extractous From 97df0477c358e620d4f7174eaacaa958714d7494 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 17 Dec 2024 20:41:56 -0800 Subject: [PATCH 12/88] uhhh --- .../Gradio_UI/Video_transcription_tab.py | 2 + App_Function_Libraries/RAG/RAG_Examples.md | 556 ------------------ .../Web_Scraping/Search_Prompt.py | 342 +++++++++++ Docs/Design/Coding_Page.md | 2 +- Docs/Design/Creative_Writing.md | 2 +- Docs/Design/DB_Design.md | 5 + Docs/Design/ETL_Pipeline.md | 42 +- Docs/Design/Prompts.md | 6 + Docs/Design/Researcher.md | 59 ++ Docs/Design/Structured_Outputs.md | 62 +- Docs/Design/TTS_STT.md | 15 + Docs/Design/UX.md | 5 + Docs/Design/VLMs.md | 5 +- Docs/Design/WebSearch.md | 57 +- Docs/Handy_Dandy_Papers.md | 3 +- Docs/Issues/Citations_and_Confabulations.md | 2 + Docs/Issues/Evaluation_Plans.md | 21 + Docs/RAG_Notes.md | 18 +- 18 files changed, 579 insertions(+), 625 deletions(-) delete mode 100644 App_Function_Libraries/RAG/RAG_Examples.md create mode 100644 App_Function_Libraries/Web_Scraping/Search_Prompt.py create mode 100644 Docs/Design/Prompts.md diff --git a/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py b/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py index 51963c6e4..520688380 100644 --- a/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py +++ b/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py @@ -956,6 +956,8 @@ def process_url_with_metadata(input_item, num_speakers, whisper_model, custom_pr def toggle_confabulation_output(checkbox_value): return gr.update(visible=checkbox_value) + + confab_checkbox.change( fn=toggle_confabulation_output, inputs=[confab_checkbox], diff --git a/App_Function_Libraries/RAG/RAG_Examples.md b/App_Function_Libraries/RAG/RAG_Examples.md deleted file mode 100644 index 0ca8b3936..000000000 --- a/App_Function_Libraries/RAG/RAG_Examples.md +++ /dev/null @@ -1,556 +0,0 @@ - -``` -################################################################################################################## -# RAG Pipeline 1 -# 0.62 0.61 0.75 63402.0 -# from langchain_openai import ChatOpenAI -# -# from langchain_community.document_loaders import WebBaseLoader -# from langchain_openai import OpenAIEmbeddings -# from langchain.text_splitter import RecursiveCharacterTextSplitter -# from langchain_chroma import Chroma -# -# from langchain_community.retrievers import BM25Retriever -# from langchain.retrievers import ParentDocumentRetriever -# from langchain.storage import InMemoryStore -# import os -# from operator import itemgetter -# from langchain import hub -# from langchain_core.output_parsers import StrOutputParser -# from langchain_core.runnables import RunnablePassthrough, RunnableParallel, RunnableLambda -# from langchain.retrievers import MergerRetriever -# from langchain.retrievers.document_compressors import DocumentCompressorPipeline - - -# def rag_pipeline(): -# try: -# def format_docs(docs): -# return "\n".join(doc.page_content for doc in docs) -# -# llm = ChatOpenAI(model='gpt-4o-mini') -# -# loader = WebBaseLoader('https://en.wikipedia.org/wiki/European_debt_crisis') -# docs = loader.load() -# -# embedding = OpenAIEmbeddings(model='text-embedding-3-large') -# -# splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=200) -# splits = splitter.split_documents(docs) -# c = Chroma.from_documents(documents=splits, embedding=embedding, -# collection_name='testindex-ragbuilder-1724657573', ) -# retrievers = [] -# retriever = c.as_retriever(search_type='mmr', search_kwargs={'k': 10}) -# retrievers.append(retriever) -# retriever = BM25Retriever.from_documents(docs) -# retrievers.append(retriever) -# -# parent_splitter = RecursiveCharacterTextSplitter(chunk_size=1200, chunk_overlap=600) -# splits = parent_splitter.split_documents(docs) -# store = InMemoryStore() -# retriever = ParentDocumentRetriever(vectorstore=c, docstore=store, child_splitter=splitter, -# parent_splitter=parent_splitter) -# retriever.add_documents(docs) -# retrievers.append(retriever) -# retriever = MergerRetriever(retrievers=retrievers) -# prompt = hub.pull("rlm/rag-prompt") -# rag_chain = ( -# RunnableParallel(context=retriever, question=RunnablePassthrough()) -# .assign(context=itemgetter("context") | RunnableLambda(format_docs)) -# .assign(answer=prompt | llm | StrOutputParser()) -# .pick(["answer", "context"])) -# return rag_chain -# except Exception as e: -# print(f"An error occurred: {e}") - - -# To get the answer and context, use the following code -# res=rag_pipeline().invoke("your prompt here") -# print(res["answer"]) -# print(res["context"]) - -############################################################################################################ - - -############################################################################################################ -# RAG Pipeline 2 - -# 0.6 0.73 0.68 3125.0 -# from langchain_openai import ChatOpenAI -# -# from langchain_community.document_loaders import WebBaseLoader -# from langchain_openai import OpenAIEmbeddings -# from langchain.text_splitter import RecursiveCharacterTextSplitter -# from langchain_chroma import Chroma -# from langchain.retrievers.multi_query import MultiQueryRetriever -# from langchain.retrievers import ParentDocumentRetriever -# from langchain.storage import InMemoryStore -# from langchain_community.document_transformers import EmbeddingsRedundantFilter -# from langchain.retrievers.document_compressors import LLMChainFilter -# from langchain.retrievers.document_compressors import EmbeddingsFilter -# from langchain.retrievers import ContextualCompressionRetriever -# import os -# from operator import itemgetter -# from langchain import hub -# from langchain_core.output_parsers import StrOutputParser -# from langchain_core.runnables import RunnablePassthrough, RunnableParallel, RunnableLambda -# from langchain.retrievers import MergerRetriever -# from langchain.retrievers.document_compressors import DocumentCompressorPipeline - - -# def rag_pipeline(): -# try: -# def format_docs(docs): -# return "\n".join(doc.page_content for doc in docs) -# -# llm = ChatOpenAI(model='gpt-4o-mini') -# -# loader = WebBaseLoader('https://en.wikipedia.org/wiki/European_debt_crisis') -# docs = loader.load() -# -# embedding = OpenAIEmbeddings(model='text-embedding-3-large') -# -# splitter = RecursiveCharacterTextSplitter(chunk_size=400, chunk_overlap=200) -# splits = splitter.split_documents(docs) -# c = Chroma.from_documents(documents=splits, embedding=embedding, -# collection_name='testindex-ragbuilder-1724650962', ) -# retrievers = [] -# retriever = MultiQueryRetriever.from_llm(c.as_retriever(search_type='similarity', search_kwargs={'k': 10}), -# llm=llm) -# retrievers.append(retriever) -# -# parent_splitter = RecursiveCharacterTextSplitter(chunk_size=1200, chunk_overlap=600) -# splits = parent_splitter.split_documents(docs) -# store = InMemoryStore() -# retriever = ParentDocumentRetriever(vectorstore=c, docstore=store, child_splitter=splitter, -# parent_splitter=parent_splitter) -# retriever.add_documents(docs) -# retrievers.append(retriever) -# retriever = MergerRetriever(retrievers=retrievers) -# arr_comp = [] -# arr_comp.append(EmbeddingsRedundantFilter(embeddings=embedding)) -# arr_comp.append(LLMChainFilter.from_llm(llm)) -# pipeline_compressor = DocumentCompressorPipeline(transformers=arr_comp) -# retriever = ContextualCompressionRetriever(base_retriever=retriever, base_compressor=pipeline_compressor) -# prompt = hub.pull("rlm/rag-prompt") -# rag_chain = ( -# RunnableParallel(context=retriever, question=RunnablePassthrough()) -# .assign(context=itemgetter("context") | RunnableLambda(format_docs)) -# .assign(answer=prompt | llm | StrOutputParser()) -# .pick(["answer", "context"])) -# return rag_chain -# except Exception as e: -# print(f"An error occurred: {e}") - - -# To get the answer and context, use the following code -# res=rag_pipeline().invoke("your prompt here") -# print(res["answer"]) -# print(res["context"]) - -# -# -# -############################################################################################################ -# Plain bm25 retriever -# class BM25Retriever(BaseRetriever): -# """`BM25` retriever without Elasticsearch.""" -# -# vectorizer: Any -# """ BM25 vectorizer.""" -# docs: List[Document] = Field(repr=False) -# """ List of documents.""" -# k: int = 4 -# """ Number of documents to return.""" -# preprocess_func: Callable[[str], List[str]] = default_preprocessing_func -# """ Preprocessing function to use on the text before BM25 vectorization.""" -# -# class Config: -# arbitrary_types_allowed = True -# -# @classmethod -# def from_texts( -# cls, -# texts: Iterable[str], -# metadatas: Optional[Iterable[dict]] = None, -# bm25_params: Optional[Dict[str, Any]] = None, -# preprocess_func: Callable[[str], List[str]] = default_preprocessing_func, -# **kwargs: Any, -# ) -> BM25Retriever: -# """ -# Create a BM25Retriever from a list of texts. -# Args: -# texts: A list of texts to vectorize. -# metadatas: A list of metadata dicts to associate with each text. -# bm25_params: Parameters to pass to the BM25 vectorizer. -# preprocess_func: A function to preprocess each text before vectorization. -# **kwargs: Any other arguments to pass to the retriever. -# -# Returns: -# A BM25Retriever instance. -# """ -# try: -# from rank_bm25 import BM25Okapi -# except ImportError: -# raise ImportError( -# "Could not import rank_bm25, please install with `pip install " -# "rank_bm25`." -# ) -# -# texts_processed = [preprocess_func(t) for t in texts] -# bm25_params = bm25_params or {} -# vectorizer = BM25Okapi(texts_processed, **bm25_params) -# metadatas = metadatas or ({} for _ in texts) -# docs = [Document(page_content=t, metadata=m) for t, m in zip(texts, metadatas)] -# return cls( -# vectorizer=vectorizer, docs=docs, preprocess_func=preprocess_func, **kwargs -# ) -# -# @classmethod -# def from_documents( -# cls, -# documents: Iterable[Document], -# *, -# bm25_params: Optional[Dict[str, Any]] = None, -# preprocess_func: Callable[[str], List[str]] = default_preprocessing_func, -# **kwargs: Any, -# ) -> BM25Retriever: -# """ -# Create a BM25Retriever from a list of Documents. -# Args: -# documents: A list of Documents to vectorize. -# bm25_params: Parameters to pass to the BM25 vectorizer. -# preprocess_func: A function to preprocess each text before vectorization. -# **kwargs: Any other arguments to pass to the retriever. -# -# Returns: -# A BM25Retriever instance. -# """ -# texts, metadatas = zip(*((d.page_content, d.metadata) for d in documents)) -# return cls.from_texts( -# texts=texts, -# bm25_params=bm25_params, -# metadatas=metadatas, -# preprocess_func=preprocess_func, -# **kwargs, -# ) -# -# def _get_relevant_documents( -# self, query: str, *, run_manager: CallbackManagerForRetrieverRun -# ) -> List[Document]: -# processed_query = self.preprocess_func(query) -# return_docs = self.vectorizer.get_top_n(processed_query, self.docs, n=self.k) -# return return_docs -############################################################################################################ - -############################################################################################################ -# ElasticSearch BM25 Retriever -# class ElasticSearchBM25Retriever(BaseRetriever): -# """`Elasticsearch` retriever that uses `BM25`. -# -# To connect to an Elasticsearch instance that requires login credentials, -# including Elastic Cloud, use the Elasticsearch URL format -# https://username:password@es_host:9243. For example, to connect to Elastic -# Cloud, create the Elasticsearch URL with the required authentication details and -# pass it to the ElasticVectorSearch constructor as the named parameter -# elasticsearch_url. -# -# You can obtain your Elastic Cloud URL and login credentials by logging in to the -# Elastic Cloud console at https://cloud.elastic.co, selecting your deployment, and -# navigating to the "Deployments" page. -# -# To obtain your Elastic Cloud password for the default "elastic" user: -# -# 1. Log in to the Elastic Cloud console at https://cloud.elastic.co -# 2. Go to "Security" > "Users" -# 3. Locate the "elastic" user and click "Edit" -# 4. Click "Reset password" -# 5. Follow the prompts to reset the password -# -# The format for Elastic Cloud URLs is -# https://username:password@cluster_id.region_id.gcp.cloud.es.io:9243. -# """ -# -# client: Any -# """Elasticsearch client.""" -# index_name: str -# """Name of the index to use in Elasticsearch.""" -# -# @classmethod -# def create( -# cls, elasticsearch_url: str, index_name: str, k1: float = 2.0, b: float = 0.75 -# ) -> ElasticSearchBM25Retriever: -# """ -# Create a ElasticSearchBM25Retriever from a list of texts. -# -# Args: -# elasticsearch_url: URL of the Elasticsearch instance to connect to. -# index_name: Name of the index to use in Elasticsearch. -# k1: BM25 parameter k1. -# b: BM25 parameter b. -# -# Returns: -# -# """ -# from elasticsearch import Elasticsearch -# -# # Create an Elasticsearch client instance -# es = Elasticsearch(elasticsearch_url) -# -# # Define the index settings and mappings -# settings = { -# "analysis": {"analyzer": {"default": {"type": "standard"}}}, -# "similarity": { -# "custom_bm25": { -# "type": "BM25", -# "k1": k1, -# "b": b, -# } -# }, -# } -# mappings = { -# "properties": { -# "content": { -# "type": "text", -# "similarity": "custom_bm25", # Use the custom BM25 similarity -# } -# } -# } -# -# # Create the index with the specified settings and mappings -# es.indices.create(index=index_name, mappings=mappings, settings=settings) -# return cls(client=es, index_name=index_name) -# -# def add_texts( -# self, -# texts: Iterable[str], -# refresh_indices: bool = True, -# ) -> List[str]: -# """Run more texts through the embeddings and add to the retriever. -# -# Args: -# texts: Iterable of strings to add to the retriever. -# refresh_indices: bool to refresh ElasticSearch indices -# -# Returns: -# List of ids from adding the texts into the retriever. -# """ -# try: -# from elasticsearch.helpers import bulk -# except ImportError: -# raise ImportError( -# "Could not import elasticsearch python package. " -# "Please install it with `pip install elasticsearch`." -# ) -# requests = [] -# ids = [] -# for i, text in enumerate(texts): -# _id = str(uuid.uuid4()) -# request = { -# "_op_type": "index", -# "_index": self.index_name, -# "content": text, -# "_id": _id, -# } -# ids.append(_id) -# requests.append(request) -# bulk(self.client, requests) -# -# if refresh_indices: -# self.client.indices.refresh(index=self.index_name) -# return ids -# -# def _get_relevant_documents( -# self, query: str, *, run_manager: CallbackManagerForRetrieverRun -# ) -> List[Document]: -# query_dict = {"query": {"match": {"content": query}}} -# res = self.client.search(index=self.index_name, body=query_dict) -# -# docs = [] -# for r in res["hits"]["hits"]: -# docs.append(Document(page_content=r["_source"]["content"])) -# return docs -############################################################################################################ - - -############################################################################################################ -# Multi Query Retriever -# class MultiQueryRetriever(BaseRetriever): -# """Given a query, use an LLM to write a set of queries. -# -# Retrieve docs for each query. Return the unique union of all retrieved docs. -# """ -# -# retriever: BaseRetriever -# llm_chain: Runnable -# verbose: bool = True -# parser_key: str = "lines" -# """DEPRECATED. parser_key is no longer used and should not be specified.""" -# include_original: bool = False -# """Whether to include the original query in the list of generated queries.""" -# -# @classmethod -# def from_llm( -# cls, -# retriever: BaseRetriever, -# llm: BaseLanguageModel, -# prompt: BasePromptTemplate = DEFAULT_QUERY_PROMPT, -# parser_key: Optional[str] = None, -# include_original: bool = False, -# ) -> "MultiQueryRetriever": -# """Initialize from llm using default template. -# -# Args: -# retriever: retriever to query documents from -# llm: llm for query generation using DEFAULT_QUERY_PROMPT -# prompt: The prompt which aims to generate several different versions -# of the given user query -# include_original: Whether to include the original query in the list of -# generated queries. -# -# Returns: -# MultiQueryRetriever -# """ -# output_parser = LineListOutputParser() -# llm_chain = prompt | llm | output_parser -# return cls( -# retriever=retriever, -# llm_chain=llm_chain, -# include_original=include_original, -# ) -# -# async def _aget_relevant_documents( -# self, -# query: str, -# *, -# run_manager: AsyncCallbackManagerForRetrieverRun, -# ) -> List[Document]: -# """Get relevant documents given a user query. -# -# Args: -# query: user query -# -# Returns: -# Unique union of relevant documents from all generated queries -# """ -# queries = await self.agenerate_queries(query, run_manager) -# if self.include_original: -# queries.append(query) -# documents = await self.aretrieve_documents(queries, run_manager) -# return self.unique_union(documents) -# -# async def agenerate_queries( -# self, question: str, run_manager: AsyncCallbackManagerForRetrieverRun -# ) -> List[str]: -# """Generate queries based upon user input. -# -# Args: -# question: user query -# -# Returns: -# List of LLM generated queries that are similar to the user input -# """ -# response = await self.llm_chain.ainvoke( -# {"question": question}, config={"callbacks": run_manager.get_child()} -# ) -# if isinstance(self.llm_chain, LLMChain): -# lines = response["text"] -# else: -# lines = response -# if self.verbose: -# logger.info(f"Generated queries: {lines}") -# return lines -# -# async def aretrieve_documents( -# self, queries: List[str], run_manager: AsyncCallbackManagerForRetrieverRun -# ) -> List[Document]: -# """Run all LLM generated queries. -# -# Args: -# queries: query list -# -# Returns: -# List of retrieved Documents -# """ -# document_lists = await asyncio.gather( -# *( -# self.retriever.ainvoke( -# query, config={"callbacks": run_manager.get_child()} -# ) -# for query in queries -# ) -# ) -# return [doc for docs in document_lists for doc in docs] -# -# def _get_relevant_documents( -# self, -# query: str, -# *, -# run_manager: CallbackManagerForRetrieverRun, -# ) -> List[Document]: -# """Get relevant documents given a user query. -# -# Args: -# query: user query -# -# Returns: -# Unique union of relevant documents from all generated queries -# """ -# queries = self.generate_queries(query, run_manager) -# if self.include_original: -# queries.append(query) -# documents = self.retrieve_documents(queries, run_manager) -# return self.unique_union(documents) -# -# def generate_queries( -# self, question: str, run_manager: CallbackManagerForRetrieverRun -# ) -> List[str]: -# """Generate queries based upon user input. -# -# Args: -# question: user query -# -# Returns: -# List of LLM generated queries that are similar to the user input -# """ -# response = self.llm_chain.invoke( -# {"question": question}, config={"callbacks": run_manager.get_child()} -# ) -# if isinstance(self.llm_chain, LLMChain): -# lines = response["text"] -# else: -# lines = response -# if self.verbose: -# logger.info(f"Generated queries: {lines}") -# return lines -# -# def retrieve_documents( -# self, queries: List[str], run_manager: CallbackManagerForRetrieverRun -# ) -> List[Document]: -# """Run all LLM generated queries. -# -# Args: -# queries: query list -# -# Returns: -# List of retrieved Documents -# """ -# documents = [] -# for query in queries: -# docs = self.retriever.invoke( -# query, config={"callbacks": run_manager.get_child()} -# ) -# documents.extend(docs) -# return documents -# -# def unique_union(self, documents: List[Document]) -> List[Document]: -# """Get unique Documents. -# -# Args: -# documents: List of retrieved Documents -# -# Returns: -# List of unique retrieved Documents -# """ -# return _unique_documents(documents) -############################################################################################################ -``` \ No newline at end of file diff --git a/App_Function_Libraries/Web_Scraping/Search_Prompt.py b/App_Function_Libraries/Web_Scraping/Search_Prompt.py new file mode 100644 index 000000000..92ca6f169 --- /dev/null +++ b/App_Function_Libraries/Web_Scraping/Search_Prompt.py @@ -0,0 +1,342 @@ +# Taken from https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py +CHAT_PROMPT = """\ +Generate a comprehensive and informative answer for a given question solely based on the provided web Search Results (URL, Page Title, Summary). You must only use information from the provided search results. Use an unbiased and journalistic tone. + +You must cite the answer using [number] notation. You must cite sentences with their relevant citation number. Cite every part of the answer. +Place citations at the end of the sentence. You can do multiple citations in a row with the format [number1][number2]. + +Only cite the most relevant results that answer the question accurately. If different results refer to different entities with the same name, write separate answers for each entity. + +ONLY cite inline. +DO NOT include a reference section, DO NOT include URLs. +DO NOT repeat the question. + + +You can use markdown formatting. You should include bullets to list the information in your answer. + + +{my_context} + +--------------------- + +Make sure to match the language of the user's question. + +Question: {my_query} +Answer (in the language of the user's question): \ +""" + +RELATED_QUESTION_PROMPT = """\ +Given a question and search result context, generate 3 follow-up questions the user might ask. Use the original question and context. + +Instructions: +- Generate exactly 3 questions. +- These questions should be concise, and simple. +- Ensure the follow-up questions are relevant to the original question and context. +Make sure to match the language of the user's question. + +Original Question: {query} + +{context} + + +Output: +related_questions: A list of EXACTLY three concise, simple follow-up questions +""" + +HISTORY_QUERY_REPHRASE = """ +Given the following conversation and a follow up input, rephrase the follow up into a SHORT, \ +standalone query (which captures any relevant context from previous messages). +IMPORTANT: EDIT THE QUERY TO BE CONCISE. Respond with a short, compressed phrase. \ +If there is a clear change in topic, disregard the previous messages. +Strip out any information that is not relevant for the retrieval task. + +Chat History: +{chat_history} + +Make sure to match the language of the user's question. + +Follow Up Input: {question} +Standalone question (Respond with only the short combined query): +""".strip() + + +QUERY_PLAN_PROMPT = """\ +You are an expert at creating search task lists to answer queries. Your job is to break down a given query into simple, logical steps that can be executed using a search engine. + +Rules: +1. Use up to 4 steps maximum, but use fewer if possible. +2. Keep steps simple, concise, and easy to understand. +3. Ensure proper use of dependencies between steps. +4. Always include a final step to summarize/combine/compare information from previous steps. + +Instructions for creating the Query Plan: +1. Break down the query into logical search steps. +2. For each step, specify an "id" (starting from 0) and a "step" description. +3. List dependencies for each step as an array of previous step ids. +4. The first step should always have an empty dependencies array. +5. Subsequent steps should list all step ids they depend on. + +Example Query: +Given the query "Compare Perplexity and You.com in terms of revenue, number of employees, and valuation" + +Example Query Plan: +[ + {{ + "id": 0, + "step": "Research Perplexity's revenue, employee count, and valuation", + "dependencies": [] + }}, + {{ + "id": 1, + "step": "Research You.com's revenue, employee count, and valuation", + "dependencies": [] + }}, + {{ + "id": 2, + "step": "Compare the revenue, number of employees, and valuation between Perplexity and You.com", + "dependencies": [0, 1] + }} +] + +Query: {query} +Query Plan (with a final summarize/combine/compare step): +""" + +SEARCH_QUERY_PROMPT = """\ +Generate a concise list of search queries to gather information for executing the given step. + +You will be provided with: +1. A specific step to execute +2. The user's original query +3. Context from previous steps (if available) + +Use this information to create targeted search queries that will help complete the current step effectively. Aim for the minimum number of queries necessary while ensuring they cover all aspects of the step. + +IMPORTANT: Always incorporate relevant information from previous steps into your queries. This ensures continuity and builds upon already gathered information. + +Input: +--- +User's original query: {user_query} +--- +Context from previous steps: +{prev_steps_context} + +Your task: +1. Analyze the current step and its requirements +2. Consider the user's original query and any relevant previous context +3. Consider the user's original query +4. Generate a list of specific, focused search queries that: + - Incorporate relevant information from previous steps + - Address the requirements of the current step + - Build upon the information already gathered +--- +Current step to execute: {current_step} +--- + +Your search queries based: +""" + + +# License Reproduction: +# Apache License +# Version 2.0, January 2004 +# http://www.apache.org/licenses/ +# +# TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +# +# 1. Definitions. +# +# "License" shall mean the terms and conditions for use, reproduction, +# and distribution as defined by Sections 1 through 9 of this document. +# +# "Licensor" shall mean the copyright owner or entity authorized by +# the copyright owner that is granting the License. +# +# "Legal Entity" shall mean the union of the acting entity and all +# other entities that control, are controlled by, or are under common +# control with that entity. For the purposes of this definition, +# "control" means (i) the power, direct or indirect, to cause the +# direction or management of such entity, whether by contract or +# otherwise, or (ii) ownership of fifty percent (50%) or more of the +# outstanding shares, or (iii) beneficial ownership of such entity. +# +# "You" (or "Your") shall mean an individual or Legal Entity +# exercising permissions granted by this License. +# +# "Source" form shall mean the preferred form for making modifications, +# including but not limited to software source code, documentation +# source, and configuration files. +# +# "Object" form shall mean any form resulting from mechanical +# transformation or translation of a Source form, including but +# not limited to compiled object code, generated documentation, +# and conversions to other media types. +# +# "Work" shall mean the work of authorship, whether in Source or +# Object form, made available under the License, as indicated by a +# copyright notice that is included in or attached to the work +# (an example is provided in the Appendix below). +# +# "Derivative Works" shall mean any work, whether in Source or Object +# form, that is based on (or derived from) the Work and for which the +# editorial revisions, annotations, elaborations, or other modifications +# represent, as a whole, an original work of authorship. For the purposes +# of this License, Derivative Works shall not include works that remain +# separable from, or merely link (or bind by name) to the interfaces of, +# the Work and Derivative Works thereof. +# +# "Contribution" shall mean any work of authorship, including +# the original version of the Work and any modifications or additions +# to that Work or Derivative Works thereof, that is intentionally +# submitted to Licensor for inclusion in the Work by the copyright owner +# or by an individual or Legal Entity authorized to submit on behalf of +# the copyright owner. For the purposes of this definition, "submitted" +# means any form of electronic, verbal, or written communication sent +# to the Licensor or its representatives, including but not limited to +# communication on electronic mailing lists, source code control systems, +# and issue tracking systems that are managed by, or on behalf of, the +# Licensor for the purpose of discussing and improving the Work, but +# excluding communication that is conspicuously marked or otherwise +# designated in writing by the copyright owner as "Not a Contribution." +# +# "Contributor" shall mean Licensor and any individual or Legal Entity +# on behalf of whom a Contribution has been received by Licensor and +# subsequently incorporated within the Work. +# +# 2. Grant of Copyright License. Subject to the terms and conditions of +# this License, each Contributor hereby grants to You a perpetual, +# worldwide, non-exclusive, no-charge, royalty-free, irrevocable +# copyright license to reproduce, prepare Derivative Works of, +# publicly display, publicly perform, sublicense, and distribute the +# Work and such Derivative Works in Source or Object form. +# +# 3. Grant of Patent License. Subject to the terms and conditions of +# this License, each Contributor hereby grants to You a perpetual, +# worldwide, non-exclusive, no-charge, royalty-free, irrevocable +# (except as stated in this section) patent license to make, have made, +# use, offer to sell, sell, import, and otherwise transfer the Work, +# where such license applies only to those patent claims licensable +# by such Contributor that are necessarily infringed by their +# Contribution(s) alone or by combination of their Contribution(s) +# with the Work to which such Contribution(s) was submitted. If You +# institute patent litigation against any entity (including a +# cross-claim or counterclaim in a lawsuit) alleging that the Work +# or a Contribution incorporated within the Work constitutes direct +# or contributory patent infringement, then any patent licenses +# granted to You under this License for that Work shall terminate +# as of the date such litigation is filed. +# +# 4. Redistribution. You may reproduce and distribute copies of the +# Work or Derivative Works thereof in any medium, with or without +# modifications, and in Source or Object form, provided that You +# meet the following conditions: +# +# (a) You must give any other recipients of the Work or +# Derivative Works a copy of this License; and +# +# (b) You must cause any modified files to carry prominent notices +# stating that You changed the files; and +# +# (c) You must retain, in the Source form of any Derivative Works +# that You distribute, all copyright, patent, trademark, and +# attribution notices from the Source form of the Work, +# excluding those notices that do not pertain to any part of +# the Derivative Works; and +# +# (d) If the Work includes a "NOTICE" text file as part of its +# distribution, then any Derivative Works that You distribute must +# include a readable copy of the attribution notices contained +# within such NOTICE file, excluding those notices that do not +# pertain to any part of the Derivative Works, in at least one +# of the following places: within a NOTICE text file distributed +# as part of the Derivative Works; within the Source form or +# documentation, if provided along with the Derivative Works; or, +# within a display generated by the Derivative Works, if and +# wherever such third-party notices normally appear. The contents +# of the NOTICE file are for informational purposes only and +# do not modify the License. You may add Your own attribution +# notices within Derivative Works that You distribute, alongside +# or as an addendum to the NOTICE text from the Work, provided +# that such additional attribution notices cannot be construed +# as modifying the License. +# +# You may add Your own copyright statement to Your modifications and +# may provide additional or different license terms and conditions +# for use, reproduction, or distribution of Your modifications, or +# for any such Derivative Works as a whole, provided Your use, +# reproduction, and distribution of the Work otherwise complies with +# the conditions stated in this License. +# +# 5. Submission of Contributions. Unless You explicitly state otherwise, +# any Contribution intentionally submitted for inclusion in the Work +# by You to the Licensor shall be under the terms and conditions of +# this License, without any additional terms or conditions. +# Notwithstanding the above, nothing herein shall supersede or modify +# the terms of any separate license agreement you may have executed +# with Licensor regarding such Contributions. +# +# 6. Trademarks. This License does not grant permission to use the trade +# names, trademarks, service marks, or product names of the Licensor, +# except as required for reasonable and customary use in describing the +# origin of the Work and reproducing the content of the NOTICE file. +# +# 7. Disclaimer of Warranty. Unless required by applicable law or +# agreed to in writing, Licensor provides the Work (and each +# Contributor provides its Contributions) on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied, including, without limitation, any warranties or conditions +# of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +# PARTICULAR PURPOSE. You are solely responsible for determining the +# appropriateness of using or redistributing the Work and assume any +# risks associated with Your exercise of permissions under this License. +# +# 8. Limitation of Liability. In no event and under no legal theory, +# whether in tort (including negligence), contract, or otherwise, +# unless required by applicable law (such as deliberate and grossly +# negligent acts) or agreed to in writing, shall any Contributor be +# liable to You for damages, including any direct, indirect, special, +# incidental, or consequential damages of any character arising as a +# result of this License or out of the use or inability to use the +# Work (including but not limited to damages for loss of goodwill, +# work stoppage, computer failure or malfunction, or any and all +# other commercial damages or losses), even if such Contributor +# has been advised of the possibility of such damages. +# +# 9. Accepting Warranty or Additional Liability. While redistributing +# the Work or Derivative Works thereof, You may choose to offer, +# and charge a fee for, acceptance of support, warranty, indemnity, +# or other liability obligations and/or rights consistent with this +# License. However, in accepting such obligations, You may act only +# on Your own behalf and on Your sole responsibility, not on behalf +# of any other Contributor, and only if You agree to indemnify, +# defend, and hold each Contributor harmless for any liability +# incurred by, or claims asserted against, such Contributor by reason +# of your accepting any such warranty or additional liability. +# +# END OF TERMS AND CONDITIONS +# +# APPENDIX: How to apply the Apache License to your work. +# +# To apply the Apache License to your work, attach the following +# boilerplate notice, with the fields enclosed by brackets "[]" +# replaced with your own identifying information. (Don't include +# the brackets!) The text should be enclosed in the appropriate +# comment syntax for the file format. We also recommend that a +# file or class name and description of purpose be included on the +# same "printed page" as the copyright notice for easier +# identification within third-party archives. +# +# Copyright [yyyy] [name of copyright owner] +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/Docs/Design/Coding_Page.md b/Docs/Design/Coding_Page.md index cf5a36a61..ac772de7d 100644 --- a/Docs/Design/Coding_Page.md +++ b/Docs/Design/Coding_Page.md @@ -12,7 +12,7 @@ https://github.com/simonw/files-to-prompt https://github.com/yamadashy/repomix/tree/main https://github.com/chanhx/crabviz - +https://github.com/charmandercha/ArchiDoc https://pythontutor.com/c.html#mode=edit https://pythontutor.com/articles/c-cpp-visualizer.html diff --git a/Docs/Design/Creative_Writing.md b/Docs/Design/Creative_Writing.md index 0bdd46b48..11d8d7f2b 100644 --- a/Docs/Design/Creative_Writing.md +++ b/Docs/Design/Creative_Writing.md @@ -7,7 +7,7 @@ ### Link Dump: https://github.com/p-e-w/arrows - +https://huggingface.co/jukofyork/creative-writing-control-vectors-v3.0 diff --git a/Docs/Design/DB_Design.md b/Docs/Design/DB_Design.md index f075bb8c3..27d536ef5 100644 --- a/Docs/Design/DB_Design.md +++ b/Docs/Design/DB_Design.md @@ -7,6 +7,11 @@ - [Interesting/Relevant Later](#interesting-relevant-later) + +SQLite + https://highperformancesqlite.com/watch/dot-commands + https://www.youtube.com/watch?v=XP-h304N06I + Migrating to sqlite-vec https://www.youtube.com/live/xmdiwdom6Vk?t=1740s https://alexgarcia.xyz/blog/2024/sqlite-vec-metadata-release/index.html diff --git a/Docs/Design/ETL_Pipeline.md b/Docs/Design/ETL_Pipeline.md index d3a976f14..469a06232 100644 --- a/Docs/Design/ETL_Pipeline.md +++ b/Docs/Design/ETL_Pipeline.md @@ -8,39 +8,41 @@ This page serves as documentation regarding the ETL pipelines within tldw and pr ## ETL Pipelines ### Data Sources -- - **Audio** - - f + - faster_whisper + - pyaudio - **Ebooks (epub)** - - f + - ebooklib - **PDFs** - Docling - pymupdf4llm - **Plain Text(`.md`, `.txt`)** - - f -- **Podcasts** - - f + - stdlib +- **PowerPoint Presentations** - need to add + - docling - **Rich Text(`.rtf`, `.docx`)** - - f + - doc2txt + - pypandoc - **RSS Feeds**: - f - **Videos** - f - **Websites**: - - f + - playwright + - bs4 + - requests - **XML Files** - - f' - - - + - xml.etree.ElementTree +- **3rd-Party Services** + - Sharepoint + * https://llamahub.ai/l/readers/llama-index-readers-microsoft-sharepoint + * + +### Tools +https://github.com/ucbepic/docetl +https://ucbepic.github.io/docetl/concepts/optimization/ +### Links +https://arxiv.org/html/2410.21169 ### Link Dump: -https://arxiv.org/abs/2410.12189 -https://ucbepic.github.io/docetl/concepts/optimization/ -https://arxiv.org/abs/2410.21169 -https://towardsdatascience.com/etl-pipelines-in-python-best-practices-and-techniques-0c148452cc68 -https://arxiv.org/html/2410.21169v2 -https://github.com/whyhow-ai/knowledge-table -https://github.com/yobix-ai/extractous -https://llamahub.ai/l/readers/llama-index-readers-microsoft-sharepoint diff --git a/Docs/Design/Prompts.md b/Docs/Design/Prompts.md new file mode 100644 index 000000000..b990d2d4f --- /dev/null +++ b/Docs/Design/Prompts.md @@ -0,0 +1,6 @@ +# Prompts & Prompt Engineering + +### Link Dump: +https://github.com/PySpur-Dev/PySpur +https://github.com/itsPreto/tangent +https://github.com/LouisShark/chatgpt_system_prompt \ No newline at end of file diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 81c96aeda..fbee60611 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -10,6 +10,65 @@ https://arxiv.org/abs/2411.15114 https://journaliststudio.google.com/pinpoint/about/ https://blog.google/products/gemini/google-gemini-deep-research/ https://github.com/neuml/annotateai +https://pub.towardsai.net/learn-anything-with-ai-and-the-feynman-technique-00a33f6a02bc +https://help.openalex.org/hc/en-us/articles/24396686889751-About-us +https://www.ginkgonotes.com/ +https://www.reddit.com/r/Anki/comments/17u01ge/spaced_repetition_algorithm_a_threeday_journey/ +https://github.com/open-spaced-repetition/fsrs4anki/wiki/Spaced-Repetition-Algorithm:-A-Three%E2%80%90Day-Journey-from-Novice-to-Expert#day-3-the-latest-progress + + +### Ideas + +Follow gptresearchers method at first, planner LLM -> query LLM -> analyzer LLM -> summarizer LLM + + + +Researcher config section +``` +[researcher] +# Researcher settings +default_search_engine = google +# Options are: google, bing, yandex, baidu, searx, kagi, serper, tavily +default_search_type = web +# Options are: web, local, both +default_search_language = en +# Options are: FIXME +default_search_report_language = en +# Options are: FIXME +default_search_sort = relevance +# Options are: relevance, date +default_search_safe_search = moderate +# Options are: off, moderate, strict +default_search_planner = openai-o1-full +# Options are: FIXME +default_search_planner_max_tokens = 8192 +default_search_analyzer = openai-o1-full +# Options are: FIXME +default_search_analyzer_max_tokens = 8192 +default_search_summarization = openai-o1-full +# Options are: FIXME +default_search_summarization_max_tokens = 8192 +search_max_results = 100 +search_report_format = markdown +# Options are: markdown, html, pdf +search_max_iterations = 5 +search_max_subtopics = 4 +search_custom_user_agent = "CUSTOM_USER_AGENT_HERE" +``` + + + + + +### Researcher Workflow + + + +### Researcher Prompts + + + + diff --git a/Docs/Design/Structured_Outputs.md b/Docs/Design/Structured_Outputs.md index 2a5ca8f6c..7643ac475 100644 --- a/Docs/Design/Structured_Outputs.md +++ b/Docs/Design/Structured_Outputs.md @@ -16,27 +16,49 @@ This page serves as documentation regarding the structured outputs within tldw a - .toml file - 2. Data Extraction + - https://github.com/yobix-ai/extractous + - Can use structured outputs for data extraction from unstructured text. Though why isn't this talked about/even mentioned in any of the papers about RAG or writeups on RAG implementations? hmmmm...... +3. Data Generation + - Can use structured outputs for data generation from unstructured text. + - Could come in handy for RPGs/Text-based games reliant on world building/lore generation. +### Implementation +- Integration for file creation +- Look at using for ETL pipeline +- Support/integration for content creation pipelines for RPG campaigns, etc. + + +Process + https://python.plainenglish.io/generating-perfectly-structured-json-using-llms-all-the-time-13b7eb504240 + +Tools + https://python.useinstructor.com/ + https://github.com/mlc-ai/xgrammar + https://github.com/guidance-ai/guidance + https://github.com/boundaryml/baml + https://docs.pydantic.dev/latest/ + https://github.com/outlines-dev/outlines + https://github.com/Dan-wanna-M/formatron/tree/master + https://github.com/whyhow-ai/knowledge-table + +Examples + https://github.com/dottxt-ai/demos/tree/main/lore-generator + https://github.com/dottxt-ai/demos/tree/main/logs + https://github.com/dottxt-ai/demos/tree/main/earnings-reports + https://github.com/dottxt-ai/demos/tree/main/its-a-smol-world + https://github.com/dottxt-ai/cursed/tree/main/scp -### Link Dump: -https://github.com/yobix-ai/extractous -https://llamahub.ai/l/readers/llama-index-readers-microsoft-sharepoint -https://blog.dottxt.co/say-what-you-mean.html -https://github.com/dottxt-ai/demos/tree/main/lore-generator -https://github.com/dottxt-ai/cursed/tree/main/scp -https://python.useinstructor.com/ -https://github.com/mlc-ai/xgrammar -https://github.com/guidance-ai/guidance -https://blog.dottxt.co/coalescence.html -https://arxiv.org/html/2408.02442v1 -https://www.boundaryml.com/blog/sota-function-calling -https://arxiv.org/abs/2408.02442 -https://towardsdatascience.com/enforcing-json-outputs-in-commercial-llms-3db590b9b3c8 -https://python.plainenglish.io/generating-perfectly-structured-json-using-llms-all-the-time-13b7eb504240 -https://docs.pydantic.dev/latest/ -https://github.com/outlines-dev/outlines[ -https://github.com/Dan-wanna-M/formatron/tree/master -https://blog.dottxt.co/selective-multiplication.html -https://blog.dottxt.co/say-what-you-mean.html +Reliability/Quality of: + https://dylancastillo.co/posts/say-what-you-mean-sometimes.html + https://blog.dottxt.co/say-what-you-mean.html + +Papers + https://arxiv.org/html/2408.02442v1 - Structured Outputs harms reasoning capabilities + + +Gemini + https://ai.google.dev/gemini-api/docs/structured-output?lang=python + +### Link Dump: diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 7d97bcf08..c828f0dc8 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -26,6 +26,21 @@ Flow: ### Link Dump: https://github.com/albirrkarim/react-speech-highlight-demo +https://funaudiollm.github.io/cosyvoice2/ +https://funaudiollm.github.io/cosyvoice2/ +https://github.com/InternLM/InternLM-XComposer/tree/main/InternLM-XComposer-2.5-OmniLive + + + +Gemini + https://ai.google.dev/gemini-api/docs#rest + https://ai.google.dev/gemini-api/docs/models/gemini-v2 + +ElevenLabs + https://github.com/elevenlabs/elevenlabs-examples/blob/main/examples/text-to-speech/python/text_to_speech_file.py + https://elevenlabs.io/docs/api-reference/text-to-speech + https://elevenlabs.io/docs/developer-guides/how-to-use-tts-with-streaming + Models https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 011209b3e..5edf884bf 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -54,6 +54,11 @@ https://github.com/Vali-98/ChatterUI https://github.com/Rugz007/liha https://astro.new/latest/ https://github.com/lobehub/lobe-chat +https://www.scoutos.com/ +https://github.com/FishiaT/yawullm +https://ilikeinterfaces.com/2015/03/09/map-ui-ghost-in-the-shell/ +https://jdan.github.io/98.css +https://github.com/vercel/ai-chatbot https://www.nngroup.com/videos/the-danger-of-defaults/ https://writings.stephenwolfram.com/2024/12/useful-to-the-point-of-being-revolutionary-introducing-wolfram-notebook-assistant/ https://en.wikipedia.org/wiki/Template:Google_payment_apps diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 44c362a1a..4ffd3dbec 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -18,7 +18,7 @@ https://huggingface.co/Qwen/Qwen2-VL-2B https://www.reddit.com/r/StableDiffusion/comments/1h7hunp/how_to_run_hunyuanvideo_on_a_single_24gb_vram_card/ https://arxiv.org/abs/2412.05185 https://huggingface.co/collections/OpenGVLab/internvl-25-673e1019b66e2218f68d7c1c - +https://huggingface.co/Infinigence/Megrez-3B-Omni https://huggingface.co/papers/2412.07626 https://huggingface.co/AI-Safeguard/Ivy-VL-llava https://github.com/matatonic/openedai-vision @@ -32,3 +32,6 @@ https://arxiv.org/abs/2409.17146 +https://lyra-omni.github.io/ +https://apollo-lmms.github.io/ +https://huggingface.co/Apollo-LMMs diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 17807f572..ff265b8c9 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -3,32 +3,16 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. -## Web Search - - -### Link Dump: -https://github.com/appvoid/search -https://github.com/felladrin/MiniSearch -https://github.com/TheBlewish/Web-LLM-Assistant-Llamacpp-Ollama -https://github.com/pengfeng/ask.py -https://cookbook.openai.com/examples/third_party/web_search_with_google_api_bring_your_own_browser_tool -https://developers.google.com/custom-search/v1/overview -https://www.ignorance.ai/p/how-to-build-an-ai-search-engine-83b?publication_id=1407539 -https://www.ignorance.ai/p/how-to-build-an-ai-search-engine -https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/document_loaders/brave_search.py - Could instantiate a browser, perform a search with X engine, and then parse the results. -https://github.com/YassKhazzan/openperplex_backend_os -https://github.com/InternLM/MindSearch -https://github.com/developersdigest/llm-answer-engine - - ### Search Engines - **Google Search** - - [Google Search API]( FIXME ) - - + - [Google Search API](https://developers.google.com/custom-search/v1/overview) + - Setup: + - Setup a `Programmable Search Engine` + - Get the `API Key` + - 100 Search queries per day for free - **Bing Search** - - [Bing Search API]( FIXME ) + - [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) - - **Yandex Search** - [Yandex Search API](https://yandex.com/dev/search/) @@ -39,3 +23,32 @@ https://github.com/developersdigest/llm-answer-engine - **Searx Search** - [Searx Search API](https://searx.github.io/searx/) - +- **Kagi** + - [Kagi Search API](https://help.kagi.com/kagi/api/search.html) + + + +### Implementaiton +- **Text Search Workflow** + 1. User inputs a search query + 2. User selects a search engine (Option for default search engine in config file) + 3. The user presses 'Search' + 4. The search query is passed to the selected search engine + 5. The appropriate search engine is used to perform a search via API call + 6. The search results are returned from the search engine's API + 7. Search engine results are then _MODIFIED_ (if necessary/enabled) to fit the user's preferences + - This could include re-ranking, summarization/analysis, or other modifications + 8. The (modified) search results are displayed to the user + 9. Results are then displayed to the user, + - either as titles of pages with dropdown for all info, + - or as a list of links with a briefing/summary of each link + - or as a single briefing/summary of all results + 10. User may then select to save this resulting text to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information + 11. Search results are then saved to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information + 12. This is then searchable via the Media DB + + +### Link Dump: +https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/document_loaders/brave_search.py + Could instantiate a browser, perform a search with X engine, and then parse the results. + diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 5e2234ce5..7c308f19b 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -32,10 +32,11 @@ https://arxiv.org/abs/2412.05265 https://arxiv.org/abs/2411.19865 https://arxiv.org/abs/2412.06769 - https://arxiv.org/abs/2412.01113 - +https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute ### Test-Time Compute - https://github.com/GAIR-NLP/O1-Journey +- https://arxiv.org/abs/2408.03314 ### Personalization diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index 99acd5d2e..e1fc58391 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -8,6 +8,8 @@ https://arxiv.org/abs/2412.04235 +https://arxiv.org/abs/2412.11536 +https://github.com/sunnynexus/RetroLLM RAG https://www.lycee.ai/blog/rag-ragallucinations-and-how-to-fight-them diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 91b3bf542..11fa6e021 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -14,6 +14,27 @@ ---------------------------------------------------------------------------------------------------------------- + +LightEval + Argilla + distilabel +- Open source, (will) support litellm and can use distilabel for synth data gen + + +https://huggingface.co/papers/2412.06745 +https://huggingface.co/dranger003/c4ai-command-r-v01-iMat.GGUF +https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-DPO +https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-SFT-no-safety-data +https://huggingface.co/mradermacher/Llama-3.1-Tulu-3-8B-SFT-no-safety-data-GGUF +https://arxiv.org/html/2412.02611v1 +https://huggingface.co/blog/synthetic-data-generator +https://towardsdatascience.com/stop-guessing-and-measure-your-rag-system-to-drive-real-improvements-bfc03f29ede3 +https://huggingface.co/SultanR/SmolTulu-1.7b-Instruct +https://huggingface.co/DevQuasar/allenai.Llama-3.1-Tulu-3-8B-SFT-no-safety-data-GGUF +https://huggingface.co/CohereForAI/c4ai-command-r7b-12-2024 +https://arxiv.org/html/2412.09569v1 +https://huggingface.co/tiiuae +https://github.com/naver/bergen?tab=readme-ov-file + + Have LLMs play Social deception games Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 0691e2d5c..deac55892 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -24,6 +24,18 @@ Unsorted https://arxiv.org/abs/2409.14924 https://arxiv.org/abs/2412.02830 https://arxiv.org/abs/2407.04125 +https://cobusgreyling.medium.com/four-levels-of-rag-research-from-microsoft-fdc54388f0ff +https://arxiv.org/html/2412.00239v1 +https://arxiv.org/abs/2412.02830 +https://arxiv.org/html/2412.02035v1 +https://towardsdatascience.com/dragin-dynamic-retrieval-augmented-generation-based-on-the-information-needs-of-large-language-dbdb9aabc1ef +https://towardsdatascience.com/improve-your-rag-context-recall-by-40-with-an-adapted-embedding-model-5d4a8f583f32 + + + + + + GraphRAG @@ -43,9 +55,9 @@ GraphRAG https://arxiv.org/abs/2412.04119 https://aibyhand.substack.com/p/beginners-guide-to-graph-rag https://github.com/gusye1234/nano-graphrag - - - + https://arxiv.org/abs/2412.03589 + https://towardsdatascience.com/graph-rag-a-conceptual-introduction-41cd0d431375 + https://towardsdatascience.com/towards-named-entity-disambiguation-with-graph-embeddings-ef164aaad37c From f50e4f272b847fbdab665936fb10133bbb50250c Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 18 Dec 2024 19:43:26 -0800 Subject: [PATCH 13/88] uhh --- Docs/Design/Coding_Page.md | 2 +- Docs/Design/Creative_Writing.md | 1 + Docs/Design/Prompts.md | 1 + Docs/Design/Researcher.md | 6 +- Docs/Design/TTS_STT.md | 4 + Docs/Design/VLMs.md | 17 +++- Docs/Design/WebSearch.md | 103 +++++++++++++++++++- Docs/Handy_Dandy_Papers.md | 2 + Docs/Issues/Citations_and_Confabulations.md | 2 + Docs/Issues/Evaluation_Plans.md | 4 + Docs/Issues/ISSUES.md | 43 ++++++++ 11 files changed, 178 insertions(+), 7 deletions(-) diff --git a/Docs/Design/Coding_Page.md b/Docs/Design/Coding_Page.md index ac772de7d..8d00cb69c 100644 --- a/Docs/Design/Coding_Page.md +++ b/Docs/Design/Coding_Page.md @@ -11,7 +11,7 @@ https://github.com/cyclotruc/gitdigest https://github.com/simonw/files-to-prompt https://github.com/yamadashy/repomix/tree/main https://github.com/chanhx/crabviz - +https://github.com/abinthomasonline/repo2txt https://github.com/charmandercha/ArchiDoc https://pythontutor.com/c.html#mode=edit https://pythontutor.com/articles/c-cpp-visualizer.html diff --git a/Docs/Design/Creative_Writing.md b/Docs/Design/Creative_Writing.md index 11d8d7f2b..88b3ae2e6 100644 --- a/Docs/Design/Creative_Writing.md +++ b/Docs/Design/Creative_Writing.md @@ -8,6 +8,7 @@ ### Link Dump: https://github.com/p-e-w/arrows https://huggingface.co/jukofyork/creative-writing-control-vectors-v3.0 +https://github.com/FartyPants/StoryCrafter/tree/main diff --git a/Docs/Design/Prompts.md b/Docs/Design/Prompts.md index b990d2d4f..adb08609a 100644 --- a/Docs/Design/Prompts.md +++ b/Docs/Design/Prompts.md @@ -3,4 +3,5 @@ ### Link Dump: https://github.com/PySpur-Dev/PySpur https://github.com/itsPreto/tangent +https://arxiv.org/abs/2412.13171 https://github.com/LouisShark/chatgpt_system_prompt \ No newline at end of file diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index fbee60611..14be1618e 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -18,7 +18,8 @@ https://github.com/open-spaced-repetition/fsrs4anki/wiki/Spaced-Repetition-Algor ### Ideas - +Gated, checkpoints with 'retry, skip, continue' options +s Follow gptresearchers method at first, planner LLM -> query LLM -> analyzer LLM -> summarizer LLM @@ -54,9 +55,12 @@ search_report_format = markdown search_max_iterations = 5 search_max_subtopics = 4 search_custom_user_agent = "CUSTOM_USER_AGENT_HERE" +search_blacklist_URLs = "URL1,URL2,URL3" ``` +https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py + diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index c828f0dc8..495816e01 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -29,6 +29,10 @@ https://github.com/albirrkarim/react-speech-highlight-demo https://funaudiollm.github.io/cosyvoice2/ https://funaudiollm.github.io/cosyvoice2/ https://github.com/InternLM/InternLM-XComposer/tree/main/InternLM-XComposer-2.5-OmniLive +https://github.com/huggingface/transformers.js-examples/tree/main/moonshine-web +https://huggingface.co/onnx-community/moonshine-base-ONNX +https://github.com/usefulsensors/moonshine + diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 4ffd3dbec..8e4110fba 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -19,6 +19,7 @@ https://www.reddit.com/r/StableDiffusion/comments/1h7hunp/how_to_run_hunyuanvide https://arxiv.org/abs/2412.05185 https://huggingface.co/collections/OpenGVLab/internvl-25-673e1019b66e2218f68d7c1c https://huggingface.co/Infinigence/Megrez-3B-Omni +https://huggingface.co/FastVideo/FastHunyuan https://huggingface.co/papers/2412.07626 https://huggingface.co/AI-Safeguard/Ivy-VL-llava https://github.com/matatonic/openedai-vision @@ -29,9 +30,19 @@ https://huggingface.co/tencent/HunyuanVideo https://aivideo.hunyuan.tencent.com/ https://github.com/deepseek-ai/DeepSeek-VL2 https://arxiv.org/abs/2409.17146 +https://www.reddit.com/r/LocalLLaMA/comments/1hfkytk/answering_my_own_question_i_got_apollo_working/ +https://arxiv.org/abs/2412.09645 +https://github.com/tdrussell/diffusion-pipe +https://proptest.ai/?#/playground +https://lyra-omni.github.io/ +Apollo + https://arxiv.org/abs/2412.10360 + https://apollo-lmms.github.io/ + https://huggingface.co/Apollo-LMMs + https://huggingface.co/manysuch-cases/Apollo-Github-Files + https://huggingface.co/GoodiesHere/Apollo-LMMs-Apollo-7B-t32 + https://huggingface.co/GoodiesHere/Apollo-LMMs-Apollo-3B-t32 + https://www.reddit.com/r/LocalLLaMA/comments/1hgri8g/has_apollo_disappeared/ -https://lyra-omni.github.io/ -https://apollo-lmms.github.io/ -https://huggingface.co/Apollo-LMMs diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index ff265b8c9..3fa32a681 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -4,6 +4,49 @@ This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. + + +OpenPerplex + 1. Search Query + 2. Perform search via API + 3. Collect results, grab story titles + snippets + 4. Extract content from each URL + 5. Concat if exceeds X length, re-rank, then combine into a single document + 6. Present final doc to user + +llm-websearch + https://github.com/Jay4242/llm-websearch + 1. Performs query -> Fetches results + 2. Iterates over each page description and decides whether to 'investigate' the page's content + 3. If so, it fetches the page content and creates a summary of it + 4. Repeat, until all URLs are processed. + 5. Create summary of all summaries/page contents and present final doc to user + +mindsearch + Uses Knowledge Graph for results + Interesting, seems to combat repetition in search results + top_p=0.8, + top_k=1, + temperature=0, + max_new_tokens=8192, + repetition_penalty=1.02, + +Perplexity + 1. Search Query + 2. Perform search via API + 3. Collect results, grab story titles + snippets + 4. Extract content from each URL + 5. Concat if exceeds X length, re-rank, then combine into a single document + 6. Present final doc to user + 7. Display top 3 sources at top, with option to show/hide remaining sources - can click on sources to go that URL + 8. Option for Follow up questions + Potential other questions, if these are selected, the content is added to the 'ongoing' document + 9. Ongoing document is saved to the DB, and can be accessed later + - Option for ranking/share/rewrite/copy to clipboard + - Pro allows for seemingly 'planning'/sub-question breakdown of the user's original query, maybe just asking 'what would be questions likely asked?'? + - Performs a planning step and then sub-query breakdown searches into 3(?) sub-queries + - Non-pro simply performs the search, combines the results, and presents them to the user while showing the sources + - Spaces: Seems to be a way of sharing collections of research document chains, with the ability to add comments, etc. + ### Search Engines - **Google Search** - [Google Search API](https://developers.google.com/custom-search/v1/overview) @@ -25,8 +68,11 @@ This page serves as documentation regarding the web search functionality within - - **Kagi** - [Kagi Search API](https://help.kagi.com/kagi/api/search.html) - - +- **Brave** + - [Brave Search API](https://brave.com/search/api/) +- **Arxiv** +- **PubMedCentral** + - https://www.ncbi.nlm.nih.gov/home/develop/api/ ### Implementaiton - **Text Search Workflow** @@ -52,3 +98,56 @@ This page serves as documentation regarding the web search functionality within https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/document_loaders/brave_search.py Could instantiate a browser, perform a search with X engine, and then parse the results. +### Prompt dumps + +https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py +``` +search_prompt_system = """ +You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate +and unbiased answers the way a highly informed individual would. +Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. +You must answer in english. +Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. +you are known for your expertise in this field. + + +###Guidelines### +1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. +2- never mention the context or this prompt in your response, just answer the user question. + +###Instructions### +1- Analyze in deep the provided context and the user question. +2- extract relevant information's from the context about the user question. +3- Yassine must take into account the date and time to answer the user question. +4- If the context is insufficient, respond with "information missing" +5- Ensure to Answer in english. +6- Use the response format provided. +7- answer the user question in a way an expert would do. +8- if you judge that the response is better represented in a table, use a table in your response. + + +###Response Format### + +You must use Markdown to format your response. + +Think step by step. +""" + +relevant_prompt_system = """ + you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related + to the user query and contexts provided. + you must keep the questions related to the user query and contexts.don't lose the context in the questions. + + The JSON object must not include special characters. + The JSON schema should include an array of follow-up questions. + + use the schema: + { + "followUp": [ + "string", + "string", + "string" + ] + } +""" +``` diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 7c308f19b..e2b7967d7 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -37,6 +37,8 @@ https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute ### Test-Time Compute - https://github.com/GAIR-NLP/O1-Journey - https://arxiv.org/abs/2408.03314 +- https://github.com/huggingface/search-and-learn +- https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute ### Personalization diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index e1fc58391..fa7837bca 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -10,6 +10,8 @@ https://arxiv.org/abs/2412.04235 https://arxiv.org/abs/2412.11536 https://github.com/sunnynexus/RetroLLM +https://huggingface.co/PleIAs/Pleias-Nano +https://deepmind.google/discover/blog/facts-grounding-a-new-benchmark-for-evaluating-the-factuality-of-large-language-models/ RAG https://www.lycee.ai/blog/rag-ragallucinations-and-how-to-fight-them diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 11fa6e021..49e5db94f 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -25,6 +25,8 @@ https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-DPO https://huggingface.co/allenai/Llama-3.1-Tulu-3-8B-SFT-no-safety-data https://huggingface.co/mradermacher/Llama-3.1-Tulu-3-8B-SFT-no-safety-data-GGUF https://arxiv.org/html/2412.02611v1 +https://arxiv.org/abs/2412.05579 +https://huggingface.co/collections/ibm-granite/granite-31-language-models-6751dbbf2f3389bec5c6f02d https://huggingface.co/blog/synthetic-data-generator https://towardsdatascience.com/stop-guessing-and-measure-your-rag-system-to-drive-real-improvements-bfc03f29ede3 https://huggingface.co/SultanR/SmolTulu-1.7b-Instruct @@ -33,6 +35,8 @@ https://huggingface.co/CohereForAI/c4ai-command-r7b-12-2024 https://arxiv.org/html/2412.09569v1 https://huggingface.co/tiiuae https://github.com/naver/bergen?tab=readme-ov-file +https://arxiv.org/abs/2412.13147 +https://arxiv.org/abs/2412.13018 Have LLMs play Social deception games diff --git a/Docs/Issues/ISSUES.md b/Docs/Issues/ISSUES.md index 993229d6f..54a82e325 100644 --- a/Docs/Issues/ISSUES.md +++ b/Docs/Issues/ISSUES.md @@ -122,3 +122,46 @@ Jailbreaks https://arxiv.org/pdf/2409.11445 + + + + + +### Model Routing +https://github.com/pepijndevos/llama_multiserver/tree/main + + +I trained one router LoRA capable of routing into a list of LoRAs I had previously trained on my domain data. One of the LoRAs I used was created by another user on Hugging Face but performed exceptionally well for my use case. + +Given a question, the classifier LoRA first classifies the question into one of my domains. Then another LoRA, based on the domain classification and the list of available LoRA models (fetched from my database), selects the most appropriate one for the answer. The message, along with any relevant context, is then sent to the selected LoRA. The final answers are aggregated and presented to the user. + +I have implemented multiple approaches. In one, the domain selector and router LoRA were trained and integrated into a single unitary adapter instead of relying on a two-step process. + +Additionally, in another experiment, I used a COT LoRA. For each step, the classifier identified the domain and selected the adapter to solve each of the given steps independetly. The process involved generating content from each of the "agents," aggregating the content and feeding it to the next step, aggregating the final answer, and presenting it to the user. + +I usually trained these adapters with 500–5000 samples. Most samples were generated using GPT-4o with few-shot prompts and domain-specific information to build the synthetic dataset. In my case this approach was effective in producing a tailored synthetic dataset. + +Weak-Abbreviations15 +You definitely can. +My approach is this: + + High Context length 2048-4096 + + I get my team to produce some (50 max) high quality Chains on the domains we work on. + + Use GPT to expand and optimize these chains. The output is the OptimalChain. + + Use the OptimalChains and FewShot to generate the remaining chains in the domains we need, based on the given structure. + + Structurizing chains in json Like format does seem to help but i dont have any proof of this claim. + + Experiment with Lora + Unsloth tuning with different parameters. You can use LLamafactory to do this. + + This seemed to be enough for most use cases. Im experimenting now with adding a RLHF/OPRO post finetuning, but i've yet to try it. (Ive built the dataset by synthetically messing up the answers to create the prefered and non prefered answer for the process. ) + +EDIT: 8. Anecdotal advice: unquantized Lora finetuning seems to train faster, and loss also converges faster. Thus is my preferred approach. + +Ive had my fair share of experience in finetuning. My MSc Thesis was based on adding new copied layers to a LLM, and then finetune the layers specifically with a new language dataset which the LLM knew very little about, and essentially aggressively train those new layers without changing any of the other LLM structures, the result was that the LLM gained new linguistical abilities without downgrading its base skills in english. So i believe that for most usecases finetuning is good enough. + + +In my paper i used instruct models. Later i discovered that non instruction tuned models seem to perform much better for finetuning. They converge easier and in my experience seem less prone to forgetting. I used QLora but only bcs the models i used were bigger than my VRAM. I also wasnt aware of the faster training in Lora vs Qlora. Sort of my insights were developed once i was deep in the paper. I started the paper with llama 7b and ended up using the llama 3.1 8b until i finalized the project. Using LlamaPro i added a number of layers whoch increased its size to smthing between 9.5-10.2 billion parameters i cant recall exactly, which made it larger than what fit my GPU. I used fairly large r and alpha for lora on the fiest pass, and then lowered them as the tuning progressed. I wanted to use a very large dataset, but ended up using a much smaller one due to time and compute constraints. Everything was done locally hence i had to fidget around and over optimize. Probably with more compute itd be better managed. The result was pretty good considering the base model produced only jibberish. Some issues when speaking some very peculiar features of my native language. I used Bleu4 and a couple of the rouge metrics to rest the quality of the outputs. Differences in performance was tested for signficance. From d96dd8d53e06c3da234a8d41b2b5bba0c5d800af Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 18 Dec 2024 22:34:46 -0800 Subject: [PATCH 14/88] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 375ee1064..f77cc6fa1 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Screenshot of the Frontpage ![Screenshot](Docs/Screenshots/blank-front.png) - **Backup Management** - A way to back up the DBs, view backups, and restore from a backup. (4 SQLite DBs: Media, Character Chats, RAG Chats, Embeddings) - **Trashcan Support** - A way to 'soft' delete content, and restore it if needed. (Helps with accidental deletions) - Trashcan is only for the MediaDB. - **Support for 7 Local LLM APIs:** `Llama.cpp`, `Kobold.cpp`, `Oobabooga`, `TabbyAPI`, `vLLM`, `Ollama`, `Aphrodite`, `Custom OpenAI API`. -- **Support for 8 Commercial APIs:** `Claude Sonnet 3.5`, `Cohere Command R+`, `DeepSeek`, `Groq`, `HuggingFace`, `Mistral`, `OpenAI`, `OpenRouter`. +- **Support for 9 Commercial APIs:** `Claude Sonnet 3.5`, `Cohere Command R+`, `DeepSeek`, `Google`, `Groq`, `HuggingFace`, `Mistral`, `OpenAI`, `OpenRouter`. - **Local Audio Recording with Transcription** - Record audio locally and transcribe it. - **Structured Prompt Creation and Management** - Create prompts using a structured approach, and then edit and use them in your chats. Or delete them. - Also have the ability to import prompts individually or in bulk. As well as export them as markdown documents. From aae29a6d09bdc7e29be6195ae9705830039c6ee5 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 20 Dec 2024 23:15:00 -0800 Subject: [PATCH 15/88] sure do love the reliability of PGE --- Docs/Bitnet_and_Mamba_Papers.md | 2 + Docs/Design/Creative_Writing.md | 5 ++- Docs/Design/ETL_Pipeline.md | 2 +- Docs/Design/Education.md | 6 ++- Docs/Design/RSS_Ranking.md | 2 +- Docs/Design/TTS_STT.md | 12 +++++ Docs/Design/UX.md | 2 + Docs/Design/VLMs.md | 4 +- Docs/Design/WebSearch.md | 77 +++++++++++++++++++++++++++++++++ Docs/Handy_Dandy_Papers.md | 11 ++++- Docs/Issues/Evaluation_Plans.md | 7 ++- Docs/RAG_Notes.md | 8 ++-- README.md | 2 +- 13 files changed, 128 insertions(+), 12 deletions(-) diff --git a/Docs/Bitnet_and_Mamba_Papers.md b/Docs/Bitnet_and_Mamba_Papers.md index 4e028a9cb..93f0d4d22 100644 --- a/Docs/Bitnet_and_Mamba_Papers.md +++ b/Docs/Bitnet_and_Mamba_Papers.md @@ -1,6 +1,8 @@ # Bitnet and MAMBA Papers +Bamba +https://huggingface.co/blog/bamba ### Bitnet - Unsorted diff --git a/Docs/Design/Creative_Writing.md b/Docs/Design/Creative_Writing.md index 88b3ae2e6..fd2a9206f 100644 --- a/Docs/Design/Creative_Writing.md +++ b/Docs/Design/Creative_Writing.md @@ -9,8 +9,9 @@ https://github.com/p-e-w/arrows https://huggingface.co/jukofyork/creative-writing-control-vectors-v3.0 https://github.com/FartyPants/StoryCrafter/tree/main - - +https://github.com/datacrystals/AIStoryWriter +https://perchance.org/ai-story-generator +https://perchance.org/tutorial#repeating_things diff --git a/Docs/Design/ETL_Pipeline.md b/Docs/Design/ETL_Pipeline.md index 469a06232..10598d592 100644 --- a/Docs/Design/ETL_Pipeline.md +++ b/Docs/Design/ETL_Pipeline.md @@ -3,7 +3,7 @@ ## Introduction This page serves as documentation regarding the ETL pipelines within tldw and provides context/justification for the details of each. - +https://towardsdatascience.com/etl-pipelines-in-python-best-practices-and-techniques-0c148452cc68 ## ETL Pipelines diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index da599dcf5..7fa5ce41b 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -3,4 +3,8 @@ ### Link Dump: https://arxiv.org/abs/2412.02035 -https://github.com/andreamust/NEON-GPT \ No newline at end of file +https://github.com/andreamust/NEON-GPT +https://excalidraw.com/ + + + diff --git a/Docs/Design/RSS_Ranking.md b/Docs/Design/RSS_Ranking.md index 13706fb72..c633626ef 100644 --- a/Docs/Design/RSS_Ranking.md +++ b/Docs/Design/RSS_Ranking.md @@ -18,7 +18,7 @@ https://arxiv.org/abs/2411.19352 - +https://blog.badsectorlabs.com/files/blogs.txt diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 495816e01..607706c51 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -20,6 +20,11 @@ Flow: - https://cloud.google.com/text-to-speech/docs/ssml +### Benchmarks +https://huggingface.co/blog/big-bench-audio-release + https://huggingface.co/datasets/ArtificialAnalysis/big_bench_audio +https://artificialanalysis.ai/models/speech-to-speech + @@ -34,7 +39,14 @@ https://huggingface.co/onnx-community/moonshine-base-ONNX https://github.com/usefulsensors/moonshine +GLM-4-Voice + https://github.com/THUDM/GLM-4-Voice/blob/main/README_en.md + https://github.com/THUDM/GLM-4-Voice/tree/main +MoonShine + https://huggingface.co/onnx-community/moonshine-base-ONNX + https://huggingface.co/spaces/webml-community/moonshine-web + https://github.com/huggingface/transformers.js-examples/tree/main/moonshine-web Gemini https://ai.google.dev/gemini-api/docs#rest diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 5edf884bf..1acfaf3d7 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -65,6 +65,8 @@ https://en.wikipedia.org/wiki/Template:Google_payment_apps https://github.com/stackblitz-labs/bolt.diy https://github.com/Haervwe/open-webui-tools https://interactjs.io/ +https://perchance.org/ai-chat +https://lovable.dev/ https://markwhen.com/ Prompt Engineering page: diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 8e4110fba..563263e49 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -32,7 +32,9 @@ https://github.com/deepseek-ai/DeepSeek-VL2 https://arxiv.org/abs/2409.17146 https://www.reddit.com/r/LocalLLaMA/comments/1hfkytk/answering_my_own_question_i_got_apollo_working/ https://arxiv.org/abs/2412.09645 - +https://rentry.org/crhcqq54 +https://arxiv.org/abs/2412.13501 +https://huggingface.co/IamCreateAI/Ruyi-Mini-7B https://github.com/tdrussell/diffusion-pipe https://proptest.ai/?#/playground https://lyra-omni.github.io/ diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 3fa32a681..788d7490b 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -4,6 +4,53 @@ This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. +ask.py https://github.com/pengfeng/ask.py + 1. User inputs a search query + 2. Search query is then used to generate several sub-queries + 3. Each sub-query is then used to perform a search via API call + 4. Sub-queries are setup using a specific prompt, they do 'at least 3, but no more than 5' - makes me wonder about how many sub-queries should be the default and the range thereof + 5. Search results returned, passed to groq + +inference prompt +``` +system_prompt = ( + "You are an expert summarizing the answers based on the provided contents." +) + +user_promt_template = """ +Given the context as a sequence of references with a reference id in the +format of a leading [x], please answer the following question using {{ language }}: + +{{ query }} + +In the answer, use format [1], [2], ..., [n] in line where the reference is used. +For example, "According to the research from Google[3], ...". + +Please create the answer strictly related to the context. If the context has no +information about the query, please write "No related information found in the context." +using {{ language }}. + +{{ length_instructions }} + +Here is the context: +{{ context }} +""" +``` +extraction prompt +``` +system_prompt = ( + "You are an expert of extract structual information from the document." +) +user_promt_template = """ +Given the provided content, if it contains information about {{ query }}, please extract the +list of structured data items as defined in the following Pydantic schema: + +{{ extract_schema_str }} + +Below is the provided content: +{{ content }} +""" +``` OpenPerplex @@ -14,6 +61,8 @@ OpenPerplex 5. Concat if exceeds X length, re-rank, then combine into a single document 6. Present final doc to user + + llm-websearch https://github.com/Jay4242/llm-websearch 1. Performs query -> Fetches results @@ -22,6 +71,8 @@ llm-websearch 4. Repeat, until all URLs are processed. 5. Create summary of all summaries/page contents and present final doc to user + + mindsearch Uses Knowledge Graph for results Interesting, seems to combat repetition in search results @@ -31,6 +82,8 @@ mindsearch max_new_tokens=8192, repetition_penalty=1.02, + + Perplexity 1. Search Query 2. Perform search via API @@ -47,6 +100,8 @@ Perplexity - Non-pro simply performs the search, combines the results, and presents them to the user while showing the sources - Spaces: Seems to be a way of sharing collections of research document chains, with the ability to add comments, etc. + + ### Search Engines - **Google Search** - [Google Search API](https://developers.google.com/custom-search/v1/overview) @@ -75,6 +130,28 @@ Perplexity - https://www.ncbi.nlm.nih.gov/home/develop/api/ ### Implementaiton +- Configuration Options: + - Query count + - Whether to create Sub-Queries + - Sub-Query count + - Search Engine Selection + - Search Engine API Key + - Search Engine Customization Options + - Safe Search + - Language + - Date Range + - Search Result Options + - Number of Results + - Result Sorting - Auto rerank according to ? + - Result Filtering - Filter according to a blacklist of URLs? Maybe also content? + - Search Result Display Options + - Display Format (List, Briefing, Full) + - Display Metadata (URL, Date, etc) + - Search Result Default Saving Options + - Save to DB + - Save to File + - Save to Clipboard + - Save to Notes DB - Create a new note with the search results + metadata & query - **Text Search Workflow** 1. User inputs a search query 2. User selects a search engine (Option for default search engine in config file) diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index e2b7967d7..57f5646f8 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -16,7 +16,7 @@ https://arxiv.org/pdf/2410.21272 https://arxiv.org/abs/2412.04318 https://arxiv.org/abs/2412.02142 https://arxiv.org/abs/2412.05265 - +https://arxiv.org/abs/2412.05346 ### Context https://arxiv.org/pdf/2304.12102 @@ -26,6 +26,12 @@ https://arxiv.org/abs/2412.05265 https://arxiv.org/abs/2412.04645 +### Jailbreaking + https://arxiv.org/abs/2411.01084 + https://arxiv.org/abs/2408.04811 + https://arxiv.org/html/2412.05346v1 + https://github.com/centerforaisafety/HarmBench + https://github.com/dsbowen/strong_reject ### Reasoning - https://huggingface.co/papers/2411.15862 @@ -62,6 +68,9 @@ https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute ### Structured Generation https://arxiv.org/abs/2411.15100 +### Synthetic data creation +https://arxiv.org/abs/2412.14689 + ### Test-Time related https://mathcritique.github.io/ diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 49e5db94f..3c7fe9597 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -61,6 +61,7 @@ Chat arena Potential issues with creating a chatarena system https://arxiv.org/abs/2412.04363 LLM-as-judge + https://arxiv.org/html/2412.14140v1 Basics https://huggingface.co/learn/cookbook/en/llm_judge Evaluating LLMs as Judges @@ -198,7 +199,8 @@ Links: - https://github.com/RUC-NLPIR/FlashRAG - Olmes - https://github.com/allenai/olmes - +- **CheeseBench** + - https://gist.github.com/av/db14a1f040f46dfb75e48451f4f14847 - **Citations** - L-CiteEval - https://huggingface.co/papers/2410.02115 @@ -239,6 +241,7 @@ Links: - HelloBench - https://github.com/Quehry/HelloBench - https://arxiv.org/abs/2409.16191 + - https://longbench2.github.io/ - https://github.com/jonathan-roberts1/needle-threading/ - Michelangelo - https://arxiv.org/abs/2409.12640 @@ -376,6 +379,8 @@ Links: - https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/summarization_score/ - https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/noise_sensitivity/ - https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/agents/#topic_adherence +- **Temporal Bias** + - https://arxiv.org/abs/2412.13377 - **Text Comprehension** - **QA (General)** * https://direct.mit.edu/tacl/article/doi/10.1162/tacl_a_00276/43518/Natural-Questions-A-Benchmark-for-Question diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index deac55892..216a231c6 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -30,6 +30,8 @@ https://arxiv.org/abs/2412.02830 https://arxiv.org/html/2412.02035v1 https://towardsdatascience.com/dragin-dynamic-retrieval-augmented-generation-based-on-the-information-needs-of-large-language-dbdb9aabc1ef https://towardsdatascience.com/improve-your-rag-context-recall-by-40-with-an-adapted-embedding-model-5d4a8f583f32 +https://arxiv.org/abs/2412.06078 +https://www.zyphra.com/post/the-mixture-of-pageranks-retriever-for-long-context-pre-processing @@ -58,9 +60,9 @@ GraphRAG https://arxiv.org/abs/2412.03589 https://towardsdatascience.com/graph-rag-a-conceptual-introduction-41cd0d431375 https://towardsdatascience.com/towards-named-entity-disambiguation-with-graph-embeddings-ef164aaad37c - - - + https://generativeai.pub/advanced-rag-retrieval-strategies-using-knowledge-graphs-12c9ce54d2da + https://medium.com/@ianormy/microsoft-graphrag-with-an-rdf-knowledge-graph-part-2-d8d291a39ed1 + https://generativeai.pub/knowledge-graph-extraction-visualization-with-local-llm-from-unstructured-text-a-history-example-94c63b366fed?gi=b98ffc33dcbc ### Links - RAG 101 diff --git a/README.md b/README.md index f77cc6fa1..39d06ab79 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Screenshot of the Frontpage ![Screenshot](Docs/Screenshots/blank-front.png) - **Structured Prompt Creation and Management** - Create prompts using a structured approach, and then edit and use them in your chats. Or delete them. - Also have the ability to import prompts individually or in bulk. As well as export them as markdown documents. - See `./Docs/Prompts/` for examples of prompts. and `./Docs/Propmts/TEMPLATE.md` for the prompt template used in tldw. -- Features to come: Anki Flashcard Deck Editing (Creation is in), Mindmap creation from content(currently in under `Utilities`, uses PlantUML), better document handling, migration to a FastAPI backend(Gradio is a placeholder UI), and more. +- Features to come: Migration to a FastAPI backend(Gradio is a placeholder UI). Anki Flashcard Deck Editing (Creation is in), Mindmap creation from content(currently in under `Utilities`, uses PlantUML), better document handling, and more. #### The original scripts by `the-crypt-keeper` for transcribing and summarizing youtube videos are available here: [scripts here](https://github.com/the-crypt-keeper/tldw/tree/main/tldw-original-scripts) From 15d1c6505a02e9d1dd2c3c2bf40ead3c90890333 Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 21 Dec 2024 22:37:48 -0800 Subject: [PATCH 16/88] uhhh --- Docs/Design/Researcher.md | 5 +- Docs/Design/VLMs.md | 3 + Docs/Design/WebScraping.md | 11 ++ Docs/Design/WebSearch.md | 211 +++++++++++++++++++++++++++++++- Docs/Handy_Dandy_Papers.md | 3 + Docs/Issues/Evaluation_Plans.md | 4 + Docs/RAG_Notes.md | 10 +- LICENSE.txt | 2 +- 8 files changed, 240 insertions(+), 9 deletions(-) create mode 100644 Docs/Design/WebScraping.md diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 14be1618e..250886c84 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -60,7 +60,8 @@ search_blacklist_URLs = "URL1,URL2,URL3" https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py - +https://storm.genie.stanford.edu/ +https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama @@ -73,7 +74,7 @@ https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_ - +https://github.com/SakanaAI/AI-Scientist one2manny diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 563263e49..1e18aa7d9 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -4,6 +4,8 @@ ### Link Dump: +https://github.com/jabberjabberjabber/LLMOCR/blob/main/llm-ocr-gui.py#L178 + https://arxiv.org/abs/2411.18279 https://github.com/breezedeus/Pix2Text https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct @@ -26,6 +28,7 @@ https://github.com/matatonic/openedai-vision https://github.com/breezedeus/Pix2Text https://github.com/Tencent/HunyuanVideo https://github.com/huggingface/blog/blob/main/smolvlm.md +https://colivara.com/ https://huggingface.co/tencent/HunyuanVideo https://aivideo.hunyuan.tencent.com/ https://github.com/deepseek-ai/DeepSeek-VL2 diff --git a/Docs/Design/WebScraping.md b/Docs/Design/WebScraping.md new file mode 100644 index 000000000..45a4aab7f --- /dev/null +++ b/Docs/Design/WebScraping.md @@ -0,0 +1,11 @@ +# WebScraping Pipeline + +### Overview +Page describing design and implementation of the web scraping pipeline + + +### Flow + + +### Link Dump +https://github.com/scrapinghub/article-extraction-benchmark diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 788d7490b..6c3b0bfc7 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -3,6 +3,7 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. +https://github.com/scrapinghub/article-extraction-benchmark ask.py https://github.com/pengfeng/ask.py 1. User inputs a search query @@ -53,6 +54,197 @@ Below is the provided content: ``` +Web-LLM Assistant - https://github.com/TheBlewish/Web-LLM-Assistant-Llamacpp-Ollama +Starting prompt +``` +SYSTEM_PROMPT = """You are an AI assistant capable of web searching and providing informative responses. +When a user's query starts with '/', interpret it as a request to search the web and formulate an appropriate search query. + +ALWAYS follow the prompts provided throughout the searching process EXACTLY as indicated. + +NEVER assume new instructions for anywhere other than directly when prompted directly. DO NOT SELF PROMPT OR PROVIDE MULTIPLE ANSWERS OR ATTEMPT MULTIPLE RESPONSES FOR ONE PROMPT! +""" +``` +self-improving prompt +``` + def evaluate_scraped_content(self, user_query: str, scraped_content: Dict[str, str]) -> Tuple[str, str]: + user_query_short = user_query[:200] + prompt = f""" +Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: + +User's question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Your task: +1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. +2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. + +Respond using EXACTLY this format: +Evaluation: [Your evaluation of the scraped content] +Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] +""" +``` +Query Creation +``` +def formulate_query(self, user_query: str, attempt: int) -> Tuple[str, str]: + user_query_short = user_query[:200] + prompt = f""" +Based on the following user question, formulate a concise and effective search query: +"{user_query_short}" +Your task: +1. Create a search query of 2-5 words that will yield relevant results. +2. Determine if a specific time range is needed for the search. +Time range options: +- 'd': Limit results to the past day. Use for very recent events or rapidly changing information. +- 'w': Limit results to the past week. Use for recent events or topics with frequent updates. +- 'm': Limit results to the past month. Use for relatively recent information or ongoing events. +- 'y': Limit results to the past year. Use for annual events or information that changes yearly. +- 'none': No time limit. Use for historical information or topics not tied to a specific time frame. +Respond in the following format: +Search query: [Your 2-5 word query] +Time range: [d/w/m/y/none] +Do not provide any additional information or explanation. +""" +``` +Select relevant content +``` +def select_relevant_pages(self, search_results: List[Dict], user_query: str) -> List[str]: + prompt = f""" +Given the following search results for the user's question: "{user_query}" +Select the 2 most relevant results to scrape and analyze. Explain your reasoning for each selection. + +Search Results: +{self.format_results(search_results)} + +Instructions: +1. You MUST select exactly 2 result numbers from the search results. +2. Choose the results that are most likely to contain comprehensive and relevant information to answer the user's question. +3. Provide a brief reason for each selection. + +You MUST respond using EXACTLY this format and nothing else: + +Selected Results: [Two numbers corresponding to the selected results] +Reasoning: [Your reasoning for the selections] +""" +``` +Final answer generation +``` + def generate_final_answer(self, user_query: str, scraped_content: Dict[str, str]) -> str: + user_query_short = user_query[:200] + prompt = f""" +You are an AI assistant. Provide a comprehensive and detailed answer to the following question using ONLY the information provided in the scraped content. Do not include any references or mention any sources. Answer directly and thoroughly. + +Question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Important Instructions: +1. Do not use phrases like "Based on the absence of selected results" or similar. +2. If the scraped content does not contain enough information to answer the question, say so explicitly and explain what information is missing. +3. Provide as much relevant detail as possible from the scraped content. + +Answer: +""" +``` +Final Answer Synthesis +``` + def synthesize_final_answer(self, user_query: str) -> str: + prompt = f""" +After multiple search attempts, we couldn't find a fully satisfactory answer to the user's question: "{user_query}" + +Please provide the best possible answer you can, acknowledging any limitations or uncertainties. +If appropriate, suggest ways the user might refine their question or where they might find more information. + +Respond in a clear, concise, and informative manner. +""" +``` + + + +appvoid search - https://github.com/appvoid/search +Eval query type +``` +async def evaluate_query_type(self, session, query): + messages = [ + {"role": "system", "content": """You are an Web assistant that evaluates the type of query a user asks. + Categorize the query into one of the following types: + 1. simple: if it can be answered with general knowledge or information that is typically well-known on the internet, please provide a short answer as relevant as possible from the llm itself, but make sure you are completly sure you know the answer, don't make things up. + 2. realtime: if it requires up-to-date information like the current date, time, or recent events, or the user explicitly asks you to look on the internet you should state as: realtime + 3. math: if it involves ANY kind of mathematical calculations. Every math question be it counting letters or complex formulas. + + Remember to prioritize realtime over anything else if you are not sure about something. Realtime is like your default. + + Respond with the category as a single word ("simple", "realtime", or "math") without any additional text."""}, + {"role": "user", "content": f"Query: {query}"} + ] +``` +Generate Search Queries +``` +async def generate_search_queries(groq_api, session, original_query, max_retries=3, fixed_count=None, previous_queries=None, previous_answer=None): + system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. The queries should be diverse and cover different aspects or perspectives of the original query. Return the queries as a JSON array. + Important instructions: + + 1. The number of queries should be dynamic, between 2 and 4, unless a fixed count is specified. + 2. Don't get too far from the original query since you don't know the actual context. + 3. Make queries general enough without being related to anything specific. + 4. DON'T customize the queries for topics you've never seen; just change them a little and look for definitions if requested by the user. + 5. If the user asks something that is not related to search, ignore it and focus on generating helpful search queries. + 6. Just return the given format ["custom_query_1","custom_query_2",...]. + 7. If you need to use your knowledge first, do so. + 8. When asked about the difference between two things, generate search intents for each topic separately. + 9. ALWAYS at most queries just require one or two queries, only on those cases where the query is simple or you are unsure, generate more than one or two. + 10. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. + 11. ALWAYS split searches for each important part of the query in case you need to gather information but make sure to not get off the rails. In short, don't look for things together, make a search for each important part instead. DONT LOOK FOR THINGS TOGETHER.""" + + messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": f"Original query: {original_query}" + (f" (Generate exactly {fixed_count} queries)" if fixed_count else "")} + ] + + if previous_queries and previous_answer: + messages.append({ + "role": "user", + "content": f"Previous queries: {previous_queries}\nPrevious answer: {previous_answer}\nPlease generate new queries to address any shortcomings in the previous answer." + }) +``` +Evaluate Answer +``` +async def evaluate_answer(groq_api, session, query, answer): + messages = [ + {"role": "system", "content": """You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. + Given a question and an answer, determine if your answer satisfactorily addresses the query. You are highly tolerant to answers that are close to the intent so if it is close enough, you can say is satisfactory. Remember, if it's close enough, mark it as satisfactory. + Respond with a JSON object containing two fields: + 1. "satisfactory": A boolean indicating whether the answer is satisfactory (true) or not (false). + 2. "reason": A brief explanation of why your thought is or is not satisfactory. Like "I will keep looking for information since last thought is not addressing the query because..." or "Let look for something different. My last search didn't solve the query. The reason is..." or "I found the right answer so I can ignore this..."."""}, + {"role": "user", "content": f"Query: {query}\nAnswer: {answer}"} + ] +``` +Eval best answer +``` +async def evaluate_best_answer(groq_api, session, query, cached_answers): + print('Answers pool > ', cached_answers) + messages = [ + {"role": "system", "content": """You are an assistant that evaluates multiple answers to a query and selects the best one based on relevance and completeness. + Given a query and a list of answers, choose the answer that best addresses the query. Respond with the best answer. Don't need to mention the word answers at all just be natural. Don't "the best answer" or things like that. Just provide the best one."""}, + {"role": "user", "content": f"Query: {query}\nAnswers: {json.dumps(cached_answers)}"} + ] +``` +Summarization +``` +messages = [ + {"role": "system", "content": """You are a web assistant that helps users find information from web search results. +Given a question and a set of search results, provide a concise response based on the information +available in the search results. If the information is not available in the search results, +state that you don't have enough information to answer the question. You MUST not comment on anything, just follow the instruction. Don't add additional details about anything."""}, + {"role": "user", "content": f"Question: {query}\nSearch Results: {json.dumps(all_results)}"} +] +``` + + + OpenPerplex 1. Search Query 2. Perform search via API @@ -129,19 +321,25 @@ Perplexity - **PubMedCentral** - https://www.ncbi.nlm.nih.gov/home/develop/api/ + ### Implementaiton - Configuration Options: + - Language + - Language used for writing query + - language used for performing query (query submitted to search engine) + - language used for summary/analysis of search results - Query count - Whether to create Sub-Queries - - Sub-Query count + - Sub-Query count - Search Engine Selection - - Search Engine API Key + - Search Engine API Key - Search Engine Customization Options - Safe Search - Language - Date Range - Search Result Options - - Number of Results + - Total number of queries to perform + - Total number of results/pages to review per query - Result Sorting - Auto rerank according to ? - Result Filtering - Filter according to a blacklist of URLs? Maybe also content? - Search Result Display Options @@ -152,6 +350,13 @@ Perplexity - Save to File - Save to Clipboard - Save to Notes DB - Create a new note with the search results + metadata & query + - Output Options + - Default Output Format (Markdown, HTML, PDF) - ehhhh + - Default Output Location (Display/ephemeral, DB) + - Output style (Briefing, Full, List) + - Output Metadata (URL, Date, etc) + - Word count limit for output (per search result & total) + - Output dialect (US, UK, a NY Radio DJ, etc) - **Text Search Workflow** 1. User inputs a search query 2. User selects a search engine (Option for default search engine in config file) diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 57f5646f8..e70aa3338 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -39,6 +39,9 @@ https://arxiv.org/abs/2411.19865 https://arxiv.org/abs/2412.06769 - https://arxiv.org/abs/2412.01113 https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute +- https://machinelearning.apple.com/research/gsm-symbolic + +- ### Test-Time Compute - https://github.com/GAIR-NLP/O1-Journey diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 3c7fe9597..4e227a759 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -199,6 +199,8 @@ Links: - https://github.com/RUC-NLPIR/FlashRAG - Olmes - https://github.com/allenai/olmes +- **Books** + - https://novelqa.github.io/ - **CheeseBench** - https://gist.github.com/av/db14a1f040f46dfb75e48451f4f14847 - **Citations** @@ -211,6 +213,8 @@ Links: - CodeMMLU - https://arxiv.org/abs/2410.01999 - https://github.com/FSoft-AI4Code/CodeMMLU + - LiveBench + - https://github.com/LiveBench/LiveBench - StackUnseen - https://prollm.toqan.ai/leaderboard/stack-unseen - https://huggingface.co/papers/2412.05210 diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 216a231c6..9fa963a73 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -32,9 +32,9 @@ https://towardsdatascience.com/dragin-dynamic-retrieval-augmented-generation-bas https://towardsdatascience.com/improve-your-rag-context-recall-by-40-with-an-adapted-embedding-model-5d4a8f583f32 https://arxiv.org/abs/2412.06078 https://www.zyphra.com/post/the-mixture-of-pageranks-retriever-for-long-context-pre-processing - - - +https://www.timescale.com/blog/finding-the-best-open-source-embedding-model-for-rag +https://weaviate.io/blog/late-chunking +https://github.com/superlinear-ai/raglite @@ -63,6 +63,10 @@ GraphRAG https://generativeai.pub/advanced-rag-retrieval-strategies-using-knowledge-graphs-12c9ce54d2da https://medium.com/@ianormy/microsoft-graphrag-with-an-rdf-knowledge-graph-part-2-d8d291a39ed1 https://generativeai.pub/knowledge-graph-extraction-visualization-with-local-llm-from-unstructured-text-a-history-example-94c63b366fed?gi=b98ffc33dcbc + https://memgraph.com/webinars/optimizing-insulin-management-the-role-of-graphrag-in-patient-care + https://ai.plainenglish.io/metagraphs-and-hypergraphs-for-complex-ai-agent-memory-and-rag-717f6f3589f5 + + ### Links - RAG 101 diff --git a/LICENSE.txt b/LICENSE.txt index 5fbc12911..fd84aa9a3 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2024 rmusser01 + Copyright 2024 Robert Musser / rmusser01 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 4c5b7877204a99d85675582285a7dfaf17adb7ac Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 22 Dec 2024 18:42:50 -0800 Subject: [PATCH 17/88] Fuck PGE --- .../Local_LLM/Local_LLM_huggingface.py | 51 +++++++++++++++++-- .../Third_Party/PubMedCentral.py | 19 +++++++ .../Web_Scraping/WebSearch_APIs.py | 48 +++++++++++++++++ Docs/Design/Prompts.md | 5 +- Docs/Design/TTS_STT.md | 1 + Docs/Design/UX.md | 6 +++ Docs/Handy_Dandy_Papers.md | 5 ++ Docs/Issues/Citations_and_Confabulations.md | 4 ++ Docs/Issues/Evaluation_Plans.md | 4 +- Docs/RAG_Notes.md | 6 ++- 10 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 App_Function_Libraries/Third_Party/PubMedCentral.py create mode 100644 App_Function_Libraries/Web_Scraping/WebSearch_APIs.py diff --git a/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py b/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py index da40f48fd..2c17e2452 100644 --- a/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py +++ b/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py @@ -1,8 +1,49 @@ -# import gradio as gr -# from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline -# import os -# import torch -# +# Local_LLM_huggingface.py +# Description: This file contains the functions that are used for performing inference with and managing Hugging Face Transformers models +# +# Imports +import os +# 3rd-Party Imports +from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline +import torch +# Local Imports +# +####################################################################################################################### +# +# Functions: + +def chat_with_transformers(user_message, system_message, model_name=None, model_path=None, max_new_tokens=100): + pass + +# Prepare the input as before +chat = [ + {"role": "system", "content": "You are a sassy, wise-cracking robot as imagined by Hollywood circa 1986."}, + {"role": "user", "content": "Hey, can you tell me any fun things to do in New York?"} +] + +# 1: Load the model and tokenizer +model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", device_map="auto", torch_dtype=torch.bfloat16) +tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct") + +# 2: Apply the chat template +formatted_chat = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True) +print("Formatted chat:\n", formatted_chat) + +# 3: Tokenize the chat (This can be combined with the previous step using tokenize=True) +inputs = tokenizer(formatted_chat, return_tensors="pt", add_special_tokens=False) +# Move the tokenized inputs to the same device the model is on (GPU/CPU) +inputs = {key: tensor.to(model.device) for key, tensor in inputs.items()} +print("Tokenized inputs:\n", inputs) + +# 4: Generate text from the model +outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.1) +print("Generated tokens:\n", outputs) + +# 5: Decode the output back to a string +decoded_output = tokenizer.decode(outputs[0][inputs['input_ids'].size(1):], skip_special_tokens=True) +print("Decoded output:\n", decoded_output) + + # # Assuming models are stored in a 'models' directory # MODELS_DIR = "models" # diff --git a/App_Function_Libraries/Third_Party/PubMedCentral.py b/App_Function_Libraries/Third_Party/PubMedCentral.py new file mode 100644 index 000000000..c0e5f169d --- /dev/null +++ b/App_Function_Libraries/Third_Party/PubMedCentral.py @@ -0,0 +1,19 @@ +# PubMedCentral.py +# Description: This file contains the functions that are used for performing queries against the PubMedCentral API +# +# Imports +# +# 3rd-Party Imports +# +# Local Imports +# +######################################################################################################################## +# +# Functions: + +def search_pubmedcentral(): + pass + +# +# End of PubMedCentral.py +######################################################################################################################## diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py new file mode 100644 index 000000000..fd653c193 --- /dev/null +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -0,0 +1,48 @@ +# WebSearch_APIs.py +# Description: This file contains the functions that are used for performing queries against various Search Engine APIs +# +# Imports +import requests +# +# 3rd-Party Imports +# +# Local Imports +# +####################################################################################################################### +# +# Functions: + +def search_baidu(): + pass + + +def search_bing(): + pass + + +def search_brave(): + pass + + +def search_duckduckgo(): + pass + + +def search_google(): + pass + + +def search_kagi(): + pass + + +def search_searx(): + pass + + +def search_yandex(): + pass + +# +# End of WebSearch_APIs.py +####################################################################################################################### diff --git a/Docs/Design/Prompts.md b/Docs/Design/Prompts.md index adb08609a..9120c2656 100644 --- a/Docs/Design/Prompts.md +++ b/Docs/Design/Prompts.md @@ -4,4 +4,7 @@ https://github.com/PySpur-Dev/PySpur https://github.com/itsPreto/tangent https://arxiv.org/abs/2412.13171 -https://github.com/LouisShark/chatgpt_system_prompt \ No newline at end of file +https://github.com/LouisShark/chatgpt_system_prompt +https://github.com/microsoft/PromptWizard + + diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 607706c51..f02d343ea 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -37,6 +37,7 @@ https://github.com/InternLM/InternLM-XComposer/tree/main/InternLM-XComposer-2.5- https://github.com/huggingface/transformers.js-examples/tree/main/moonshine-web https://huggingface.co/onnx-community/moonshine-base-ONNX https://github.com/usefulsensors/moonshine +https://github.com/Azure-Samples/aisearch-openai-rag-audio GLM-4-Voice diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 1acfaf3d7..6626267c3 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -51,6 +51,8 @@ https://github.com/rmusser01/agentic_rag/tree/main https://github.com/astramind-ai/Pulsar https://github.com/woshixiaobai2019/mirau-chat-ui https://github.com/Vali-98/ChatterUI +https://www.nngroup.com/videos/efficient-error-messages/ +https://uxdesign.cc/a-good-design-is-a-good-essay-197f4bd31c92 https://github.com/Rugz007/liha https://astro.new/latest/ https://github.com/lobehub/lobe-chat @@ -64,8 +66,12 @@ https://writings.stephenwolfram.com/2024/12/useful-to-the-point-of-being-revolut https://en.wikipedia.org/wiki/Template:Google_payment_apps https://github.com/stackblitz-labs/bolt.diy https://github.com/Haervwe/open-webui-tools +https://uxdesign.cc/the-evolution-of-attention-c6154276f1b4 +https://otranscribe.com/ https://interactjs.io/ https://perchance.org/ai-chat +https://uxdesign.cc/youre-trying-to-turn-an-oil-tanker-288ed30876e4 +https://uxdesign.cc/simple-recall-testing-for-b2b-ux-134e18330da9 https://lovable.dev/ https://markwhen.com/ diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index e70aa3338..257efd12d 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -60,6 +60,11 @@ https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute ### RAG https://arxiv.org/html/2407.21059v1 + +### RWKV + https://arxiv.org/abs/2412.14847 + + ### Sampling https://arxiv.org/abs/2411.09661 diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index fa7837bca..01dfe404d 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -12,6 +12,10 @@ https://arxiv.org/abs/2412.11536 https://github.com/sunnynexus/RetroLLM https://huggingface.co/PleIAs/Pleias-Nano https://deepmind.google/discover/blog/facts-grounding-a-new-benchmark-for-evaluating-the-factuality-of-large-language-models/ +https://arxiv.org/abs/2412.14860 +https://arxiv.org/abs/2412.14686 + + RAG https://www.lycee.ai/blog/rag-ragallucinations-and-how-to-fight-them diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 4e227a759..cdb37913c 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -37,7 +37,9 @@ https://huggingface.co/tiiuae https://github.com/naver/bergen?tab=readme-ov-file https://arxiv.org/abs/2412.13147 https://arxiv.org/abs/2412.13018 - +https://huggingface.co/blog/big-bench-audio-release +https://github.com/chigkim/openai-api-gpqa +https://github.com/chigkim/Ollama-MMLU-Pro Have LLMs play Social deception games Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 9fa963a73..ca3aa28be 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -35,7 +35,8 @@ https://www.zyphra.com/post/the-mixture-of-pageranks-retriever-for-long-context- https://www.timescale.com/blog/finding-the-best-open-source-embedding-model-for-rag https://weaviate.io/blog/late-chunking https://github.com/superlinear-ai/raglite - +https://arxiv.org/abs/2412.15101 +https://arxiv.org/abs/2412.14510 @@ -65,7 +66,8 @@ GraphRAG https://generativeai.pub/knowledge-graph-extraction-visualization-with-local-llm-from-unstructured-text-a-history-example-94c63b366fed?gi=b98ffc33dcbc https://memgraph.com/webinars/optimizing-insulin-management-the-role-of-graphrag-in-patient-care https://ai.plainenglish.io/metagraphs-and-hypergraphs-for-complex-ai-agent-memory-and-rag-717f6f3589f5 - + https://arxiv.org/html/2411.15671v1 +https://arxiv.org/pdf/2410.04739v1 ### Links From 5c4ba51db19193f9e12297be7899f56f588d126c Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 22 Dec 2024 23:04:05 -0800 Subject: [PATCH 18/88] search --- App_Function_Libraries/TTS/TTS_Providers.py | 12 ++ .../Third_Party/Note_Services.py | 24 +++ App_Function_Libraries/Third_Party/Slack.py | 20 +++ .../Web_Scraping/WebSearch_APIs.py | 140 ++++++++++++++++-- 4 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 App_Function_Libraries/Third_Party/Note_Services.py create mode 100644 App_Function_Libraries/Third_Party/Slack.py diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index c1c8259b7..1e9ceff25 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -164,6 +164,17 @@ def test_generate_audio_openai(api_key, input_text, voice, model, response_forma ####################################################### +####################################################### +# +# MS Azure TTS Provider Functions +# +#https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-azure-speech/README.md + +# +# End of MS Edge TTS Provider Functions +####################################################### + + ####################################################### # # MS Edge TTS Provider Functions - NOPE @@ -214,6 +225,7 @@ def test_generate_audio_openai(api_key, input_text, voice, model, response_forma # ElvenLabs TTS Provider Functions # FIXME - all of this +# https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-elevenlabs/README.md #https://elevenlabs.io/docs/api-reference/text-to-speech def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): """Generate audio using ElevenLabs API.""" diff --git a/App_Function_Libraries/Third_Party/Note_Services.py b/App_Function_Libraries/Third_Party/Note_Services.py new file mode 100644 index 000000000..6ad759a77 --- /dev/null +++ b/App_Function_Libraries/Third_Party/Note_Services.py @@ -0,0 +1,24 @@ +# Note_Services.py +# Description: This file contains the functions that are used for interacting with various Note-Taking Services +# +# Imports +import requests +# +# 3rd-Party Imports +# +# Local Imports +# +####################################################################################################################### +# +# Functions: + +###################### Notion API ###################### +# https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-notion/README.md + +# +# End Notion API +###################### End Notion API ###################### + +# +# End of Note_Services.py +####################################################################################################################### diff --git a/App_Function_Libraries/Third_Party/Slack.py b/App_Function_Libraries/Third_Party/Slack.py new file mode 100644 index 000000000..ea8f3c580 --- /dev/null +++ b/App_Function_Libraries/Third_Party/Slack.py @@ -0,0 +1,20 @@ +# Slack.py +# Description: This file contains the functions that are used for interacting with Slack API +# +# Imports +import requests +# +# 3rd-Party Imports +# +# Local Imports +# +####################################################################################################################### +# +# Functions: + +# https://api.slack.com/methods/conversations.history +# https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-slack/README.md + +# +# End of Slack.py +####################################################################################################################### diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index fd653c193..076df0602 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -12,24 +12,146 @@ # # Functions: +# https://cloud.baidu.com/doc/APIGUIDE/s/Xk1myz05f def search_baidu(): pass - -def search_bing(): - pass - - -def search_brave(): - pass - - +# https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/quickstarts/rest/python +# https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/query-parameters +# Country/Language code: https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes#country-codes +def search_bing(search_term, bing_lang="en", bing_country="en", date_range=None, result_count=None, bing_api_key=None): + if not bing_api_key: + # load key from config file + if not bing_api_key: + raise ValueError("Please provide a valid Bing Search API subscription key") + search_url = "https://api.bing.microsoft.com/v7.0/search" + if not result_count: + # Perform check in config file for default search result count + answer_count = 10 + else: + answer_count = result_count + + # date_range = "day", "week", "month", or `YYYY-MM-DD..YYYY-MM-DD` + # if not date_range: + # date_range = "month" + + # Language settings + if not bing_lang: + # do config check for default search language + setlang = bing_lang + + # Country settings + if not bing_country: + # do config check for default search country + setcountry = bing_country + + headers = {"Ocp-Apim-Subscription-Key": subscription_key} + params = {"q": search_term, "textDecorations": True, "textFormat": "HTML", "count": answer_count, + "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} + response = requests.get(search_url, headers=headers, params=params) + response.raise_for_status() + bing_search_results = response.json() + return bing_search_results +# Llamaindex https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-bing-search/llama_index/tools/bing_search/base.py +# def _bing_request(self, endpoint: str, query: str, keys: List[str]): +# response = requests.get( +# ENDPOINT_BASE_URL + endpoint, +# headers={"Ocp-Apim-Subscription-Key": self.api_key}, +# params={"q": query, "mkt": self.lang, "count": self.results}, +# ) +# response_json = response.json() +# return [[result[key] for key in keys] for result in response_json["value"]] + + +# Display results bing search +# from IPython.display import HTML +# +# rows = "\n".join([""" +# {1} +# {2} +# """.format(v["url"], v["name"], v["snippet"]) +# for v in search_results["webPages"]["value"]]) +# HTML("{0}
".format(rows)) + + +# https://brave.com/search/api/ +# https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-brave-search/README.md +def search_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, + result_filter=None, brave_api_key=None): + search_url = "https://api.search.brave.com/res/v1/web/search" + + if not brave_api_key: + # load key from config file + if not brave_api_key: + raise ValueError("Please provide a valid Brave Search API subscription key") + if not country: + country = "US" + if not search_lang: + search_lang = "en" + if not ui_lang: + ui_lang = "en" + if not result_count: + result_count = 10 + # if not date_range: + # date_range = "month" + if not result_filter: + result_filter = "webpages" + + headers = {"Accept": "application/json", "Accept-Encoding": "gzip", "X-Subscription-Token": brave_api_key} + + # https://api.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters + params = {"q": search_term, "textDecorations": True, "textFormat": "HTML", "count": result_count, + "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} + + response = requests.get(search_url, headers=headers, params=params) + response.raise_for_status() + # Response: https://api.search.brave.com/app/documentation/web-search/responses#WebSearchApiResponse + brave_search_results = response.json() + return brave_search_results + +# https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py +# https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py#L204 def search_duckduckgo(): pass def search_google(): pass +# Llamaindex +https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/tools/tool_spec/base.py +class GoogleSearchToolSpec(BaseToolSpec): + """Google Search tool spec.""" + + spec_functions = ["google_search"] + + def __init__(self, key: str, engine: str, num: Optional[int] = None) -> None: + """Initialize with parameters.""" + self.key = key + self.engine = engine + self.num = num + + def google_search(self, query: str): + """ + Make a query to the Google search engine to receive a list of results. + + Args: + query (str): The query to be passed to Google search. + num (int, optional): The number of search results to return. Defaults to None. + + Raises: + ValueError: If the 'num' is not an integer between 1 and 10. + """ + url = QUERY_URL_TMPL.format( + key=self.key, engine=self.engine, query=urllib.parse.quote_plus(query) + ) + + if self.num is not None: + if not 1 <= self.num <= 10: + raise ValueError("num should be an integer between 1 and 10, inclusive") + url += f"&num={self.num}" + + response = requests.get(url) + return [Document(text=response.text)] def search_kagi(): From 6083cdc9bec53c7d41a6aeb0f7efb1c6f0e90ad0 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 22 Dec 2024 23:30:18 -0800 Subject: [PATCH 19/88] Update WebSearch_APIs.py --- .../Web_Scraping/WebSearch_APIs.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 076df0602..3486da8a5 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -45,7 +45,7 @@ def search_bing(search_term, bing_lang="en", bing_country="en", date_range=None, # do config check for default search country setcountry = bing_country - headers = {"Ocp-Apim-Subscription-Key": subscription_key} + headers = {"Ocp-Apim-Subscription-Key": bing_api_key} params = {"q": search_term, "textDecorations": True, "textFormat": "HTML", "count": answer_count, "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} response = requests.get(search_url, headers=headers, params=params) @@ -118,7 +118,7 @@ def search_duckduckgo(): def search_google(): pass # Llamaindex -https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/tools/tool_spec/base.py +#https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/tools/tool_spec/base.py class GoogleSearchToolSpec(BaseToolSpec): """Google Search tool spec.""" @@ -154,14 +154,54 @@ def google_search(self, query: str): return [Document(text=response.text)] -def search_kagi(): +# https://help.kagi.com/kagi/api/search.html +def search_kagi(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, + result_filter=None, kagi_api_key=None): + search_url = "https://api.search.brave.com/res/v1/web/search" + + if not kagi_api_key: + # load key from config file + if not kagi_api_key: + raise ValueError("Please provide a valid Kagi Search API subscription key") + if not country: + country = "US" + if not search_lang: + search_lang = "en" + if not ui_lang: + ui_lang = "en" + if not result_count: + result_count = 10 + # if not date_range: + # date_range = "month" + if not result_filter: + result_filter = "webpages" + + + headers = {"Authorization: Bot " + kagi_api_key} + + # https://api.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters + params = {"q": search_term, "textDecorations": True, "textFormat": "HTML", "count": result_count, + "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} + + response = requests.get(search_url, headers=headers, params=params) + response.raise_for_status() + # Response: https://api.search.brave.com/app/documentation/web-search/responses#WebSearchApiResponse + brave_search_results = response.json() + return brave_search_results + # curl - v \ + # - H + # "Authorization: Bot $TOKEN" \ + # https: // kagi.com / api / v0 / search\?q = steve + jobs pass def search_searx(): pass - +# https://yandex.cloud/en/docs/search-api/operations/web-search +# https://yandex.cloud/en/docs/search-api/quickstart/ +# https://yandex.cloud/en/docs/search-api/concepts/response +# https://github.com/yandex-cloud/cloudapi/blob/master/yandex/cloud/searchapi/v2/search_query.proto def search_yandex(): pass From dbb782a9bdb04f9ba64c9ad35caf04ab86e2af08 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 23 Dec 2024 22:21:05 -0800 Subject: [PATCH 20/88] Config settings + defaults for google search --- App_Function_Libraries/Utils/Utils.py | 99 ++++++- .../Web_Scraping/WebSearch_APIs.py | 263 +++++++++++++++--- Docs/Design/Finetuning.md | 12 + Docs/Design/Researcher.md | 6 + Docs/Design/Search.md | 5 + Docs/Design/TTS_STT.md | 12 + Docs/Design/VLMs.md | 4 + Docs/Issues/Evaluation_Plans.md | 4 + 8 files changed, 357 insertions(+), 48 deletions(-) create mode 100644 Docs/Design/Finetuning.md create mode 100644 Docs/Design/Search.md diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 0ccc0b897..8a7a74dc3 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -339,6 +339,57 @@ def load_and_log_configs(): default_eleven_tts_output_format = config.get('TTS-Settings', 'default_eleven_tts_output_format', fallback='mp3_44100_192') + # Search Engines + search_provider_default = config.get('Search-Engines', 'search_provider_default', fallback='google') + search_language_query = config.get('Search-Engines', 'search_language_query', fallback='en') + search_language_results = config.get('Search-Engines', 'search_language_results', fallback='en') + search_language_analysis = config.get('Search-Engines', 'search_language_analysis', fallback='en') + search_default_max_queries = 10 + search_enable_subquery = config.get('Search-Engines', 'search_enable_subquery', fallback='True') + search_enable_subquery_count_max = config.get('Search-Engines', 'search_enable_subquery_count_max', fallback=5) + search_result_rerank = config.get('Search-Engines', 'search_result_rerank', fallback='True') + search_result_max = config.get('Search-Engines', 'search_result_max', fallback=10) + search_result_max_per_query = config.get('Search-Engines', 'search_result_max_per_query', fallback=10) + search_result_blacklist = config.get('Search-Engines', 'search_result_blacklist', fallback='') + search_result_display_type = config.get('Search-Engines', 'search_result_display_type', fallback='list') + search_result_display_metadata = config.get('Search-Engines', 'search_result_display_metadata', fallback='False') + search_result_save_to_db = config.get('Search-Engines', 'search_result_save_to_db', fallback='True') + search_result_analysis_tone = config.get('Search-Engines', 'search_result_analysis_tone', fallback='') + # Search Engine Specifics + baidu_search_api_key = config.get('Search-Engines', 'search_engine_api_key_baidu', fallback='') + # Bing Search Settings + bing_search_api_key = config.get('Search-Engines', 'search_engine_api_key_bing', fallback='') + bing_country_code = config.get('Search-Engines', 'search_engine_country_code_bing', fallback='us') + # Brave Search Settings + brave_search_api_key = config.get('Search-Engines', 'search_engine_api_key_brave', fallback='') + brave_country_code = config.get('Search-Engines', 'search_engine_country_code_brave', fallback='us') + # DuckDuckGo Search Settings + duckduckgo_search_api_key = config.get('Search-Engines', 'search_engine_api_key_duckduckgo', fallback='') + # Google Search Settings + google_search_api_key = config.get('Search-Engines', 'search_engine_api_key_google', fallback='') + google_search_engine_id = config.get('Search-Engines', 'search_engine_id_google', fallback='') + google_simp_trad_chinese = config.get('Search-Engines', 'enable_traditional_chinese', fallback='0') + limit_google_search_to_country = config.get('Search-Engines', 'limit_google_search_to_country', fallback='0') + google_search_country_code = config.get('Search-Engines', 'google_search_country_code', fallback='us') + google_filter_setting = config.get('Search-Engines', 'google_filter_setting', fallback='1') + google_user_geolocation = config.get('Search-Engines', 'google_user_geolocation', fallback='') + google_ui_language = config.get('Search-Engines', 'google_ui_language', fallback='en') + google_limit_search_results_to_language = config.get('Search-Engines', 'google_limit_search_results_to_language', fallback='') + google_default_search_results = config.get('Search-Engines', 'google_default_search_results', fallback='10') + google_safe_search = config.get('Search-Engines', 'google_safe_search', fallback='active') + google_enable_site_search = config.get('Search-Engines', 'google_enable_site_search', fallback='0') + google_site_search_include = config.get('Search-Engines', 'google_site_search_include', fallback='') + google_site_search_exclude = config.get('Search-Engines', 'google_site_search_exclude', fallback='') + google_sort_results_by = config.get('Search-Engines', 'google_sort_results_by', fallback='relevance') + # Kagi Search Settings + kagi_search_api_key = config.get('Search-Engines', 'search_engine_api_key_kagi', fallback='') + # Searx Search Settings + searx_search_api_key = config.get('Search-Engines', 'search_engine_api_key_searx', fallback='') + # Yandex Search Settings + yandex_search_api_key = config.get('Search-Engines', 'search_engine_api_key_yandex', fallback='') + yandex_search_engine_id = config.get('Search-Engines', 'search_engine_id_yandex', fallback='') + + return { 'api_keys': { 'anthropic': anthropic_api_key, @@ -433,6 +484,50 @@ def load_and_log_configs(): 'default_eleven_tts_voice_use_speaker_boost': default_eleven_tts_voice_use_speaker_boost, 'default_eleven_tts_output_format': default_eleven_tts_output_format # GPT Sovi-TTS + }, + 'search_settings': { + 'default_search_provider': search_provider_default, + 'search_language_query': search_language_query, + 'search_language_results': search_language_results, + 'search_language_analysis': search_language_analysis, + 'search_default_max_queries': search_default_max_queries, + 'search_enable_subquery': search_enable_subquery, + 'search_enable_subquery_count_max': search_enable_subquery_count_max, + 'search_result_rerank': search_result_rerank, + 'search_result_max': search_result_max, + 'search_result_max_per_query': search_result_max_per_query, + 'search_result_blacklist': search_result_blacklist, + 'search_result_display_type': search_result_display_type, + 'search_result_display_metadata': search_result_display_metadata, + 'search_result_save_to_db': search_result_save_to_db, + 'search_result_analysis_tone': search_result_analysis_tone, + }, + 'search_engines': { + 'baidu_search_api_key': baidu_search_api_key, + 'bing_search_api_key': bing_search_api_key, + 'bing_country_code': bing_country_code, + 'brave_search_api_key': brave_search_api_key, + 'brave_country_code': brave_country_code, + 'duckduckgo_search_api_key': duckduckgo_search_api_key, + 'google_search_api_key': google_search_api_key, + 'google_search_engine_id': google_search_engine_id, + 'google_simp_trad_chinese': google_simp_trad_chinese, + 'limit_google_search_to_country': limit_google_search_to_country, + 'google_search_country_code': google_search_country_code, + 'google_search_filter_setting': google_filter_setting, + 'google_user_geolocation': google_user_geolocation, + 'google_ui_language': google_ui_language, + 'google_limit_search_results_to_language': google_limit_search_results_to_language, + 'google_site_search_include': google_site_search_include, + 'google_site_search_exclude': google_site_search_exclude, + 'google_sort_results_by': google_sort_results_by, + 'google_default_search_results': google_default_search_results, + 'google_safe_search': google_safe_search, + 'google_enable_site_search' : google_enable_site_search, + 'kagi_search_api_key': kagi_search_api_key, + 'searx_search_api_key': searx_search_api_key, + 'yandex_search_api_key': yandex_search_api_key, + 'yandex_search_engine_id': yandex_search_engine_id } } @@ -441,7 +536,7 @@ def load_and_log_configs(): return None -global_api_endpoints = ["anthropic", "cohere", "groq", "openai", "huggingface", "openrouter", "deepseek", "mistral", "google", "custom_openai_api", "llama", "ooba", "kobold", "tabby", "vllm", "ollama", "aphrodite"] +global_api_endpoints = ["anthropic", "cohere", "google", "groq", "openai", "huggingface", "openrouter", "deepseek", "mistral", "custom_openai_api", "llama", "ooba", "kobold", "tabby", "vllm", "ollama", "aphrodite"] openai_tts_voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] @@ -454,12 +549,12 @@ def format_api_name(api): "openai": "OpenAI", "anthropic": "Anthropic", "cohere": "Cohere", + "google": "Google", "groq": "Groq", "huggingface": "HuggingFace", "openrouter": "OpenRouter", "deepseek": "DeepSeek", "mistral": "Mistral", - "google": "Google", "custom_openai_api": "Custom-OpenAI-API", "llama": "Llama.cpp", "ooba": "Ooba", diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 3486da8a5..75436ac04 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -3,6 +3,10 @@ # # Imports import requests + +from App_Function_Libraries.Utils.Utils import loaded_config_data + + # # 3rd-Party Imports # @@ -11,29 +15,85 @@ ####################################################################################################################### # # Functions: +# 1. perform_websearch +# 1. search_baidu +# 2. test_baidu_search +# 3. search_bing +# 4. test_search_bing +# 5. search_brave +# 6. test_search_brave +# 7. search_duckduckgo +# 8. test_duckduckgo_search +# 9. search_google +# 10. test_search_google +# 11. search_kagi +# 12. test_search_kagi +# 13. search_searx +# 14. test_search_searx +# 15. search_yandex +# 16. test_search_yandex +# +####################################################################################################################### +# + +def perform_websearch(search_engine, search_term, country, search_lang, output_lang, result_count, date_range, + safesearch, site_blacklist, api_key): + if search_engine.lower() == "baidu": + return search_web_baidu() + elif search_engine.lower() == "bing": + return search_web_bing(search_term, search_lang, country, date_range, result_count, api_key) + elif search_engine.lower() == "brave": + return search_web_brave(search_term, country, search_lang, output_lang, result_count, safesearch, api_key, + site_blacklist, date_range) + elif search_engine.lower() == "duckduckgo": + return search_web_ddg() + elif search_engine.lower() == "google": + return search_web_google() + elif search_engine.lower() == "kagi": + return search_web_kagi(search_term, country, search_lang, output_lang, result_count, safesearch, date_range, + site_blacklist, api_key) + elif search_engine.lower() == "searx": + return search_web_searx() + elif search_engine.lower() == "yandex": + return search_web_yandex() + elif search_engine.lower() == "serper": + return search_web_serper() + elif search_engine.lower() == "tavily": + return search_web_tavily() + else: + return f"Error: Invalid Search Engine Name {search_engine}" + # https://cloud.baidu.com/doc/APIGUIDE/s/Xk1myz05f -def search_baidu(): +def search_web_baidu(): pass + +def test_baidu_search(arg1, arg2, arg3): + result = search_web_baidu(arg1, arg2, arg3) + return result + + # https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/quickstarts/rest/python # https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/query-parameters # Country/Language code: https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes#country-codes -def search_bing(search_term, bing_lang="en", bing_country="en", date_range=None, result_count=None, bing_api_key=None): +def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bing_api_key=None, + date_range=None): if not bing_api_key: # load key from config file + bing_api_key = loaded_config_data['search-engines']['bing_search_api_key'] if not bing_api_key: raise ValueError("Please provide a valid Bing Search API subscription key") search_url = "https://api.bing.microsoft.com/v7.0/search" if not result_count: # Perform check in config file for default search result count - answer_count = 10 + answer_count = loaded_config_data['search-engines']['search_result_max'] else: answer_count = result_count # date_range = "day", "week", "month", or `YYYY-MM-DD..YYYY-MM-DD` - # if not date_range: - # date_range = "month" + if not date_range: + date_range = None # Language settings if not bing_lang: @@ -43,48 +103,32 @@ def search_bing(search_term, bing_lang="en", bing_country="en", date_range=None, # Country settings if not bing_country: # do config check for default search country + bing_country = loaded_config_data['search-engines']['bing_country_code'] + else: setcountry = bing_country headers = {"Ocp-Apim-Subscription-Key": bing_api_key} - params = {"q": search_term, "textDecorations": True, "textFormat": "HTML", "count": answer_count, + params = {"q": search_query, "mkt": bing_country, "textDecorations": True, "textFormat": "HTML", "count": answer_count, "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} response = requests.get(search_url, headers=headers, params=params) response.raise_for_status() bing_search_results = response.json() return bing_search_results -# Llamaindex https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-bing-search/llama_index/tools/bing_search/base.py -# def _bing_request(self, endpoint: str, query: str, keys: List[str]): -# response = requests.get( -# ENDPOINT_BASE_URL + endpoint, -# headers={"Ocp-Apim-Subscription-Key": self.api_key}, -# params={"q": query, "mkt": self.lang, "count": self.results}, -# ) -# response_json = response.json() -# return [[result[key] for key in keys] for result in response_json["value"]] - - -# Display results bing search -# from IPython.display import HTML -# -# rows = "\n".join([""" -# {1} -# {2} -# """.format(v["url"], v["name"], v["snippet"]) -# for v in search_results["webPages"]["value"]]) -# HTML("{0}
".format(rows)) # https://brave.com/search/api/ # https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-brave-search/README.md -def search_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, - result_filter=None, brave_api_key=None): +def search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", + brave_api_key=None, result_filter=None, date_range=None, ): search_url = "https://api.search.brave.com/res/v1/web/search" - if not brave_api_key: # load key from config file + brave_api_key = loaded_config_data['search-engines']['brave_search_api_key'] if not brave_api_key: raise ValueError("Please provide a valid Brave Search API subscription key") if not country: + brave_country = loaded_config_data['search-engines']['search_engine_country_code_brave'] + else: country = "US" if not search_lang: search_lang = "en" @@ -109,27 +153,122 @@ def search_brave(search_term, country, search_lang, ui_lang, result_count, safes brave_search_results = response.json() return brave_search_results + +def test_search_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, + result_filter=None, brave_api_key=None): + result = search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch, date_range, + result_filter, brave_api_key) + return result + + # https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py # https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py#L204 -def search_duckduckgo(): +def search_web_ddg(): pass -def search_google(): +def test_duckduckgo_search(): pass -# Llamaindex -#https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/tools/tool_spec/base.py -class GoogleSearchToolSpec(BaseToolSpec): - """Google Search tool spec.""" - spec_functions = ["google_search"] - def __init__(self, key: str, engine: str, num: Optional[int] = None) -> None: - """Initialize with parameters.""" - self.key = key - self.engine = engine - self.num = num +# https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list +def search_web_google(search_query, google_search_api_key, google_search_engine_id, result_count, c2coff, + results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation,ui_language, + search_result_language, safesearch, site_blacklist, sort_results_by): + search_url = "https://www.googleapis.com/customsearch/v1?" + if not c2coff: + # check config file for default setting + enable_traditional_chinese = loaded_config_data['search-engines']['google_simp_trad_chinese'] + else: + enable_traditional_chinese = "0" + + if not results_origin_country: + # check config file for default setting + limit_country_search = loaded_config_data['search-engines']['limit_google_search_to_country'] + if limit_country_search: + results_origin_country = loaded_config_data['search-engines']['google_search_country'] + + if not google_search_engine_id: + # load key from config file + google_search_engine_id = loaded_config_data['search-engines']['google_search_engine_id'] + if not google_search_engine_id: + raise ValueError("Please set a valid Google Search Engine ID in the config file") + + if not google_search_api_key: + # load key from config file + google_search_api_key = loaded_config_data['search-engines']['google_search_api_key'] + if not google_search_api_key: + raise ValueError("Please provide a valid Bing Search API subscription key") + + # Logic for handling date range + if not date_range: + date_range = None + + # Logic for handling exact terms - FIXMe + # Identifies a phrase that all documents in the search results must contain. + if not exactTerms: + exactTerms = None + + # Logic for handling exclude terms - FIXME + # Identifies a word or phrase that should not appear in any documents in the search results. + if not excludeTerms: + excludeTerms = None + + # Logic for handling filter + # Controls turning on or off the duplicate content filter. + # Default is 1 (On). + if not filter: + filter = "1" + + # Logic for handling geolocation + # Country codes: https://developers.google.com/custom-search/docs/json_api_reference#countryCodes + # Country location of user + if not geolocation: + geolocation = None + + # Logic for handling UI language + # https://developers.google.com/custom-search/docs/json_api_reference#wsInterfaceLanguages + # https://developers.google.com/custom-search/docs/json_api_reference#interfaceLanguages + if not ui_language: + ui_language = None + + # Logic for handling search result language + if not search_result_language: + search_result_language = None + + + if not result_count: + # Perform check in config file for default search result count + answer_count = loaded_config_data['search-engines']['search_result_max'] + else: + answer_count = result_count + # Language settings + if not bing_lang: + # do config check for default search language + setlang = bing_lang + + # Country settings + if not bing_country: + # do config check for default search country + bing_country = loaded_config_data['search-engines']['bing_country_code'] + else: + setcountry = bing_country + + params = {"c2coff": enable_traditional_chinese, "key": google_search_api_key, "cx": google_search_engine_id, "q": search_query} + response = requests.get(search_url, params=params) + response.raise_for_status() + bing_search_results = response.json() + return bing_search_results + self.key = key + self.engine = engine + self.num = num + + pass + + +# Llamaindex +#https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/tools/tool_spec/base.py def google_search(self, query: str): """ Make a query to the Google search engine to receive a list of results. @@ -154,9 +293,13 @@ def google_search(self, query: str): return [Document(text=response.text)] +def test_search_google(): + pass + + # https://help.kagi.com/kagi/api/search.html -def search_kagi(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, - result_filter=None, kagi_api_key=None): +def search_web_kagi(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, + result_filter=None, kagi_api_key=None): search_url = "https://api.search.brave.com/res/v1/web/search" if not kagi_api_key: @@ -176,7 +319,6 @@ def search_kagi(search_term, country, search_lang, ui_lang, result_count, safese if not result_filter: result_filter = "webpages" - headers = {"Authorization: Bot " + kagi_api_key} # https://api.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters @@ -195,14 +337,43 @@ def search_kagi(search_term, country, search_lang, ui_lang, result_count, safese pass -def search_searx(): +def test_search_kagi(): pass + +def search_web_searx(): + pass + + +def test_search_searx(): + pass + + +def search_web_serper(): + pass + + +def test_search_serper(): + pass + + +def search_web_tavily(): + pass + + +def test_search_tavily(): + pass + + # https://yandex.cloud/en/docs/search-api/operations/web-search # https://yandex.cloud/en/docs/search-api/quickstart/ # https://yandex.cloud/en/docs/search-api/concepts/response # https://github.com/yandex-cloud/cloudapi/blob/master/yandex/cloud/searchapi/v2/search_query.proto -def search_yandex(): +def search_web_yandex(): + pass + + +def test_search_yandex(): pass # diff --git a/Docs/Design/Finetuning.md b/Docs/Design/Finetuning.md new file mode 100644 index 000000000..88ff85ec3 --- /dev/null +++ b/Docs/Design/Finetuning.md @@ -0,0 +1,12 @@ +# Finetuning + + + + +### Link Dump +https://colab.research.google.com/drive/1WVf8SUdZ8YllGyqL6fBSCcCI2AC1JGc4?usp=sharing&pli=1 +https://colab.research.google.com/drive/1hrE-MPvOBThQQA51f20m0qBUIgN_g7DG?usp=sharing +https://colab.research.google.com/drive/1hmOqYfamizWYe5xjX01z-IIROeaWRhA7?usp=sharing +https://colab.research.google.com/drive/1TT6NED5iFUGratZj4aHe13iOJkDTUUVT?usp=sharing +https://kaitchup.substack.com/p/the-recipe-for-extremely-accurate-quantization?triedRedirect=true +https://github.com/Kiln-AI/Kiln \ No newline at end of file diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 250886c84..118e693e5 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -15,6 +15,12 @@ https://help.openalex.org/hc/en-us/articles/24396686889751-About-us https://www.ginkgonotes.com/ https://www.reddit.com/r/Anki/comments/17u01ge/spaced_repetition_algorithm_a_threeday_journey/ https://github.com/open-spaced-repetition/fsrs4anki/wiki/Spaced-Repetition-Algorithm:-A-Three%E2%80%90Day-Journey-from-Novice-to-Expert#day-3-the-latest-progress +https://www.scrapingdog.com/blog/scrape-google-news/ + + + + + ### Ideas diff --git a/Docs/Design/Search.md b/Docs/Design/Search.md new file mode 100644 index 000000000..c4f068d20 --- /dev/null +++ b/Docs/Design/Search.md @@ -0,0 +1,5 @@ +# Search + + +### Link Dump +https://pub.towardsai.net/hnsw-small-world-yes-but-how-in-the-world-is-it-navigable-77701ed37e20 \ No newline at end of file diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index f02d343ea..cbdc9d674 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -38,6 +38,18 @@ https://github.com/huggingface/transformers.js-examples/tree/main/moonshine-web https://huggingface.co/onnx-community/moonshine-base-ONNX https://github.com/usefulsensors/moonshine https://github.com/Azure-Samples/aisearch-openai-rag-audio +https://www.reddit.com/r/LocalLLaMA/comments/1f0awd6/best_local_open_source_texttospeech_and/ +https://github.com/FanaHOVA/smol-podcaster +https://docs.inferless.com/cookbook/serverless-customer-service-bot +https://github.com/aedocw/epub2tts +https://github.com/microsoft/SpeechT5 + + + + + + + GLM-4-Voice diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 1e18aa7d9..b4f528798 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -5,6 +5,10 @@ ### Link Dump: https://github.com/jabberjabberjabber/LLMOCR/blob/main/llm-ocr-gui.py#L178 +https://colab.research.google.com/drive/1wkCIO6q8UDJQbPsu8jI_og1JAUibpp38?usp=sharing#scrollTo=5LH3vJS6SkHF +https://colab.research.google.com/drive/1bcSu_mLki11aXpbS6Fwo_WF5idJv8uId?usp=sharing +https://colab.research.google.com/drive/1E-ySj39oldXcvcsbjDYnvW-USi281Llj?usp=sharing + https://arxiv.org/abs/2411.18279 https://github.com/breezedeus/Pix2Text diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index cdb37913c..977cc4c9d 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -40,6 +40,10 @@ https://arxiv.org/abs/2412.13018 https://huggingface.co/blog/big-bench-audio-release https://github.com/chigkim/openai-api-gpqa https://github.com/chigkim/Ollama-MMLU-Pro +https://huggingface.co/ymcki/Llama-3_1-Nemotron-51B-Instruct-GGUF +https://pub.towardsai.net/streamline-your-llm-evaluation-a-step-by-step-guide-to-rag-metrics-with-streamlit-38ed9efbdc9a + + Have LLMs play Social deception games Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 From f8c299dce2f2e81df59fc56147cc8a97b35f6058 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 23 Dec 2024 22:46:29 -0800 Subject: [PATCH 21/88] DDG --- .../Web_Scraping/WebSearch_APIs.py | 220 +++++++++++++----- Tests/WebSearch/DDG_WebSearch.py | 123 ++++++++++ 2 files changed, 282 insertions(+), 61 deletions(-) create mode 100644 Tests/WebSearch/DDG_WebSearch.py diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 75436ac04..b51e5be24 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -2,6 +2,11 @@ # Description: This file contains the functions that are used for performing queries against various Search Engine APIs # # Imports +import json +from random import choice +from time import sleep +from typing import Optional, Dict, List + import requests from App_Function_Libraries.Utils.Utils import loaded_config_data @@ -36,30 +41,32 @@ ####################################################################################################################### # -def perform_websearch(search_engine, search_term, country, search_lang, output_lang, result_count, date_range, +def perform_websearch(search_engine, search_query, country, search_lang, output_lang, result_count, date_range, safesearch, site_blacklist, api_key): if search_engine.lower() == "baidu": return search_web_baidu() elif search_engine.lower() == "bing": - return search_web_bing(search_term, search_lang, country, date_range, result_count, api_key) + return search_web_bing(search_query, search_lang, country, date_range, result_count, api_key) elif search_engine.lower() == "brave": - return search_web_brave(search_term, country, search_lang, output_lang, result_count, safesearch, api_key, + return search_web_brave(search_query, country, search_lang, output_lang, result_count, safesearch, api_key, site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": return search_web_ddg() elif search_engine.lower() == "google": - return search_web_google() + return search_web_google(search_query, api_key, google_search_engine_id, result_count, c2coff, + results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, + search_result_language, safesearch, site_blacklist, sort_results_by) elif search_engine.lower() == "kagi": - return search_web_kagi(search_term, country, search_lang, output_lang, result_count, safesearch, date_range, + return search_web_kagi(search_query, country, search_lang, output_lang, result_count, safesearch, date_range, site_blacklist, api_key) - elif search_engine.lower() == "searx": - return search_web_searx() - elif search_engine.lower() == "yandex": - return search_web_yandex() elif search_engine.lower() == "serper": return search_web_serper() elif search_engine.lower() == "tavily": return search_web_tavily() + elif search_engine.lower() == "searx": + return search_web_searx() + elif search_engine.lower() == "yandex": + return search_web_yandex() else: return f"Error: Invalid Search Engine Name {search_engine}" @@ -162,10 +169,139 @@ def test_search_brave(search_term, country, search_lang, ui_lang, result_count, # https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py -# https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py#L204 -def search_web_ddg(): - pass +# FIXME - 1shot gen with sonnet 3.5, untested. +def search_web_ddg( + keywords: str, + region: str = "wt-wt", + safesearch: str = "moderate", + max_results: Optional[int] = None +) -> List[Dict[str, str]]: + """ + Perform a DuckDuckGo search and return results. + + Args: + keywords: Search query + region: Region code (e.g., "wt-wt", "us-en") + safesearch: "on", "moderate", or "off" + max_results: Maximum number of results to return + + Returns: + List of dictionaries containing search results with 'title', 'href', and 'body' keys + """ + + # User agent list to randomize requests + user_agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0" + ] + + # Helper function to get vqd parameter + def get_vqd(keywords: str) -> str: + headers = {'User-Agent': choice(user_agents)} + resp = requests.get( + "https://duckduckgo.com/", + params={'q': keywords}, + headers=headers + ) + # Extract vqd from response + try: + vqd = resp.text.split('vqd="')[1].split('"')[0] + return vqd + except IndexError: + raise Exception("Could not extract vqd parameter") + + results = [] + vqd = get_vqd(keywords) + + # Search parameters + params = { + 'q': keywords, + 'kl': region, + 'l': region, + 'p': '1' if safesearch == "on" else "", + 's': '0', + 'df': '', + 'vqd': vqd, + 'ex': '-1' if safesearch == "moderate" else '-2' if safesearch == "off" else "" + } + + headers = { + 'User-Agent': choice(user_agents), + 'Referer': 'https://duckduckgo.com/' + } + + # Keep track of seen URLs to avoid duplicates + seen_urls = set() + + try: + while True: + response = requests.get( + 'https://links.duckduckgo.com/d.js', + params=params, + headers=headers + ) + + if response.status_code != 200: + break + + # Parse JSON response + try: + data = json.loads(response.text.replace('DDG.pageLayout.load(', '').rstrip(');')) + except json.JSONDecodeError: + break + + # Extract results + for result in data['results']: + url = result.get('u') + if url and url not in seen_urls: + seen_urls.add(url) + results.append({ + 'title': result.get('t', ''), + 'href': url, + 'body': result.get('a', '') + }) + + if max_results and len(results) >= max_results: + return results + + # Check for next page + if 'next' not in data: + break + + params['s'] = data['next'] + sleep(0.5) # Be nice to the server + + except Exception as e: + print(f"Error during search: {e}") + + return results + # def example_usage(): + # """Example usage of the DuckDuckGo search function""" + # try: + # # Basic search + # results = search_web_ddg("Python programming") + # print(f"Found {len(results)} results for 'Python programming'") + # + # # Print first 3 results + # for i, result in enumerate(results[:3], 1): + # print(f"\nResult {i}:") + # print(f"Title: {result['title']}") + # print(f"URL: {result['href']}") + # print(f"Description: {result['body'][:150]}...") + # + # # Search with different parameters + # limited_results = duckduckgo_search( + # keywords="artificial intelligence news", + # region="us-en", + # safesearch="on", + # max_results=5 + # ) + # print(f"\nFound {len(limited_results)} limited results") + # + # except DuckDuckGoSearchException as e: + # print(f"Search failed: {e}") def test_duckduckgo_search(): pass @@ -176,7 +312,7 @@ def search_web_google(search_query, google_search_api_key, google_search_engine_ results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation,ui_language, search_result_language, safesearch, site_blacklist, sort_results_by): search_url = "https://www.googleapis.com/customsearch/v1?" - + # FIXME - build the query string dynamically if not c2coff: # check config file for default setting enable_traditional_chinese = loaded_config_data['search-engines']['google_simp_trad_chinese'] @@ -237,60 +373,22 @@ def search_web_google(search_query, google_search_api_key, google_search_engine_ if not search_result_language: search_result_language = None - - if not result_count: - # Perform check in config file for default search result count - answer_count = loaded_config_data['search-engines']['search_result_max'] + if not safesearch: + safe_search = loaded_config_data['search-engines']['google_safe_search'] else: - answer_count = result_count - # Language settings - if not bing_lang: - # do config check for default search language - setlang = bing_lang + safe_search = safesearch - # Country settings - if not bing_country: - # do config check for default search country - bing_country = loaded_config_data['search-engines']['bing_country_code'] - else: - setcountry = bing_country + if not site_blacklist: + site_blacklist = None + + if not sort_results_by: + sort_results_by = None params = {"c2coff": enable_traditional_chinese, "key": google_search_api_key, "cx": google_search_engine_id, "q": search_query} response = requests.get(search_url, params=params) response.raise_for_status() - bing_search_results = response.json() - return bing_search_results - self.key = key - self.engine = engine - self.num = num - - pass - - -# Llamaindex -#https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/tools/tool_spec/base.py - def google_search(self, query: str): - """ - Make a query to the Google search engine to receive a list of results. - - Args: - query (str): The query to be passed to Google search. - num (int, optional): The number of search results to return. Defaults to None. - - Raises: - ValueError: If the 'num' is not an integer between 1 and 10. - """ - url = QUERY_URL_TMPL.format( - key=self.key, engine=self.engine, query=urllib.parse.quote_plus(query) - ) - - if self.num is not None: - if not 1 <= self.num <= 10: - raise ValueError("num should be an integer between 1 and 10, inclusive") - url += f"&num={self.num}" - - response = requests.get(url) - return [Document(text=response.text)] + google_search_results = response.json() + return google_search_results def test_search_google(): diff --git a/Tests/WebSearch/DDG_WebSearch.py b/Tests/WebSearch/DDG_WebSearch.py new file mode 100644 index 000000000..3dce5895f --- /dev/null +++ b/Tests/WebSearch/DDG_WebSearch.py @@ -0,0 +1,123 @@ +# DDG_WebSearch.py +# Description: DuckDuckGo Web Search API Unit Test +# +# Imports +import unittest +from typing import List, Dict +# +# 3rd party imports +# +# Local imports +from App_Function_Libraries.Web_Scraping.WebSearch_APIs import search_web_ddg +# +####################################################################################################################### +# +# Functions: + +def example_usage(): + """Example usage of the DuckDuckGo search function""" + try: + # Basic search + results = search_web_ddg("Python programming") + print(f"Found {len(results)} results for 'Python programming'") + + # Print first 3 results + for i, result in enumerate(results[:3], 1): + print(f"\nResult {i}:") + print(f"Title: {result['title']}") + print(f"URL: {result['href']}") + print(f"Description: {result['body'][:150]}...") + + # Search with different parameters + limited_results = search_web_ddg( + keywords="artificial intelligence news", + region="us-en", + safesearch="on", + max_results=5 + ) + print(f"\nFound {len(limited_results)} limited results") + + except search_web_ddg as e: + print(f"Search failed: {e}") + +class TestDuckDuckGoSearch(unittest.TestCase): + """Test cases for DuckDuckGo search function""" + + def test_basic_search(self): + """Test basic search functionality""" + results = search_web_ddg("Python programming") + self.assertIsInstance(results, list) + self.assertTrue(len(results) > 0) + + # Check result structure + first_result = results[0] + self.assertIn('title', first_result) + self.assertIn('href', first_result) + self.assertIn('body', first_result) + + # Check data types + self.assertIsInstance(first_result['title'], str) + self.assertIsInstance(first_result['href'], str) + self.assertIsInstance(first_result['body'], str) + + # Check for non-empty values + self.assertTrue(first_result['title']) + self.assertTrue(first_result['href']) + self.assertTrue(first_result['body']) + + def test_max_results(self): + """Test max_results parameter""" + max_results = 5 + results = search_web_ddg("Python programming", max_results=max_results) + self.assertLessEqual(len(results), max_results) + + def test_invalid_region(self): + """Test invalid region handling""" + results = search_web_ddg("Python", region="invalid-region") + self.assertIsInstance(results, list) # Should still return results with default region + + def test_result_uniqueness(self): + """Test that results are unique""" + results = search_web_ddg("Python programming", max_results=10) + urls = [result['href'] for result in results] + unique_urls = set(urls) + self.assertEqual(len(urls), len(unique_urls)) + + def test_url_normalization(self): + """Test URL normalization""" + results = search_web_ddg("Python programming") + for result in results: + self.assertTrue(result['href'].startswith(('http://', 'https://'))) + + def validate_search_results(self, results: List[Dict[str, str]]) -> bool: + """Helper method to validate search results structure""" + if not results: + return False + + required_keys = {'title', 'href', 'body'} + for result in results: + if not all(key in result for key in required_keys): + return False + if not all(isinstance(result[key], str) for key in required_keys): + return False + if not all(result[key].strip() for key in required_keys): + return False + return True + + +def run_tests(): + """Run all tests""" + unittest.main(argv=[''], exit=False) + + +if __name__ == "__main__": + # Example usage + print("Running example search:") + example_usage() + + print("\nRunning tests:") + run_tests() + +# +# End of DDG_WebSearch.py +####################################################################################################################### From f407bce4d293af9471b585c423b6094cd4112841 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 23 Dec 2024 22:47:12 -0800 Subject: [PATCH 22/88] Create Versioning_Scheme.jpg --- Docs/Versioning_Scheme.jpg | Bin 0 -> 57466 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Docs/Versioning_Scheme.jpg diff --git a/Docs/Versioning_Scheme.jpg b/Docs/Versioning_Scheme.jpg new file mode 100644 index 0000000000000000000000000000000000000000..feb06318e75d56d6c562a1c99bc387129ca27688 GIT binary patch literal 57466 zcmeFZ1yEi|w>J7>!QF!ecXxujySqzpcMI+=L4&)y1a}Q?!5xCTUow-CZ)VPS{_osd zb?dJ>i_PxtXRUtL>fS9o)$F{?zpMce#D&F#0YE?ifavQF;AIIQ008|(z(GO5p}@hx zp+aN{vmk3%MTJDFfcF(a0obXaJYA9NNDf=V|nQUAVPo~ zfQ^6v5deS@fj|&}UU~r703ZMe&<_FpCU3wYz(7HOe`rzu1pidOECOIafB?V?Y00u69d_T}OOcGMDu~OUr!-E&alxo6dr3D!3#D3jmjcJeps7PD zyp>a*%MUk}Fs)Rp9(@1kLR4F;)T?RcPm;})$~`9fG?igx-!}SX<5KP$qj2_Q^#7JfK^lDvCTDou4)eo1*eu#1x5`%qc3c^1)`2}=Us0#k z)m~MJg$Ftb%TRL_F92{WXEdm>vkK{AcR+6|S~I`EXaS`E_ULV_&u@f(OAAfdMs7U- zH79=9tWiGQWyHK`EfjTFs}thOuY|t{z3^P2Wb32=OKtRM84;to7|i0NViRhcsres~ zf|;O?<)+Yq?i?_zQFE!qU+KQC<6{FCA0%-MVkhfe1DCqU4Du4xGrnlW+B-vr?UZVI{o=qzkk1ju*+2^SJsqhWr%sL0aw=a0mKXh z`%QGE7Ju*6*Gh;Z8g`SwcUg7hmcLOj8j(5)1uNYdi_0*5&_blQ`7K7a<&^l54B1Qb z6cc#>(9XaTR2IW2vr$31gDeQ%AvTTc*n-U-4{0d?B7<$km9F$jTdtMs4AbT};=iVk zUVT(|N49Z}ud5X%QcH4o;7ZT?(t|(Dz|re6hD3Uhv~r}Eb$`M@$Z=1+5aG=P`fKWL zHRJ~!0DydhY%gp%r3TsSrk|J1|SKcR^RX6b6NsxwF9{t7bU^hS@< zPcd1x4;}ab0QnN1UdQN$;cbz@%y`yC`yB_})$g_co&h@DF}4#B_4_h5D_Bzchn^Gn zo-^B}^Xn3dbi^O%-@HROsIX>aATTKs%H#R~05BI>g$A50-rb=$F2dzQJoX-pV*xy$ z{~-8F4(N<~=90jd;~T({NZn(7zT*!)uB4iWvEJdiCh;A?A1FZ3XaX4>cxvX8jPef@ zB5P%=vQh2nnM)*bE?HY-_?M0bIxqho6Kt?Vh?B|1m2A{E@h5_R%>sWR#!Ml{1^3w0 zswa+idGh%~4bPLEM16CRv-SrJ2p-NRp#du>VaWK=UGQ*@q@TJI!TR_dwsLY` z0S|Vs$G28IosI`=b#i0Bk; z^c~VU7LCi)1+3T~_8wd&jFjrFR%@}{-09?7b{$NWV6x0fg17HAfUjPMk4&OSb?7^i*a9HUqTzk+Q z0BbjaXj_M^^>$nEK^?$Iu?manfxa;WydGrUtrcLBWC_*dRn zCqW~xD?Z%q=#|qt22eBS8#_#_I{Q>~->RkTL3@3j zi=P53{LDpw$+ZvW)8{wdra=F7A^H6a0P8ec{r|}Ll}4cf;TI48aO2Eq7F^#eto^Y> z{2`!DLZg(>fsw7ee_%gz5Q1KFV|^ii!~X_gr>I=nrAPk@4zhmpofe0BajvXs#;T!*gf2Mu>=SRo*wB6~Cj`0`szZJb9i0tsPWh(!o_?Kb& zCmrUhpyGC=|ESNOuz$)bB}#ix{aAGWRsPFpadl+J)zJc)q<^ej|3<*+ib)fc{r+n4 zI>^Z$!vATSKx|t(?kWiJ{F{yM2N@9m;+3Ao4ewux$xCn%$%sGy6@IOty08iiuD2Tj z_jlc&5MU&wxn4EfKOw(n!R+@g&%)ULhQAUEYw!%EwRZf2>URicRnIdFO{-!h=a z$oezHi@z2BoSHU z%FeiGT)Pd?h#Dcb64AuYDL!?PK8zYAmX=}L7jE(XC>Z7#bt1vB_c`1x@-S$OxZ?Gd z^c%99ZH!eUw1bn zecBcgsm9w_X~K(qB8>{}CO4PE*%|xe0O@Ks&wgRs{_OP8$Wtm6USx%k zmffBviLT-??Ko>9OeSY55KONdt~~d%^}5!427h+M+*{CsCcCIkTUqozH&R4TKGB$E z$YiEZ8IHI@U!az@?=I72(CckHEjePB{_Nnt$-nMFV1Eh8^$X$XK7ey28ffzxK58BS zvUz^xgKI5%e(wWMdvg3YLQsAk(b%Oo2?=PxJa|-K)B{Ywkf=0@1Xd70Jc>`7Cyb*P zUDJY}1HNu667Q5+tl2~!kdKfKErn{DR+gmK1zJtNu`JLsEhKFH) zo`+zJ_KI64EvN~JW9?IX`S%#?HAZ$@0Sr?*iRB<8$t2}+wO39PjxvPB5S zb}vNx8?MG_bwx}g7{!nm%f~{}++(Zjp%|XCkAdT_9lV1yd=D?vgJEXhIUl7E-24m* zA^SsdfrzD&b4WAZAcw)q)u>{ld^4}qGuZ7(c0jEc+3T$72>Pr(WM@B(hy0&}P1oWiA$*iuhajk*) ztw6|(IS@XNectLVI78l?eYhG^2^A#61ZxEd7;DySUYx+eH)K;jZ$;Z^t?SIWAt&2B z)tso}jIZg|uGw9ncYb&=adu=2pRmfWWAM6h zukLOiQy05!m4;DXLugI~+csa$6bd&nJnKO{j3+EWA;6Rr8wiy8$xR)<6o0+FF`JxT zpFX*w^xHIX`*3kJed;O^j~5y`m@PpYboYeVR0}6;XTqHvGqd~jZhKQ@C4OOPZ9d^_ zEAycdRDHDANQ88*=Q?)YFy*ZVb3jA~F~Yu_@;e=HK|LtQ92084 zum$;`=9+R^Sg=hAWuaY3iy<$^WQ?@*HgMsd>INl9QxHo{Lx90TO7KaF$|K{6up=qQ zJEg?rKzj>0Wp!OJx%)anVJsU6Z@3Wz>F`4NL~k1v9tntli1jx}xvne$8Gjbleuo(C ziQE-ad_DLBlJB|8y3o@};FEMVBL(NT6Ty!0w{UfJA-d%3+qS6G2aDb^XwI}DRPU~H zt@xE|+%|L_Jn9obwG`t+T|6k;!K1JDH9PaiKmX?BKTN}vSZ@PuzgO|E;ASZ=@=3a! z(4&-E$LR2%$nmb=u4f!$VgS>ttr(1p=nz!xE_KD%y9sd5D+7u)(Qo}{7O@2ll#@#= zUI6p|q2J%kLGCuV#(&@cfN+RJAAPU|>Ols#IX?ep^Pi^N%nqwx6`d@-FGP zGzXgUkzhag+`rC#rcYGyGwlXbCiZ$Fy9+ppc-rVu;hM76NhnqKidUBk=v?f!F|^p~ zs|1?KFMz^IJ*3hD(@4$wLSmc`R@0>P$wZAQ|7k$K+Jl5Ctnk-uFLm=Q{fW)PNa%%?Oq5i-Q6l`4l?WYaV(J>j~whz>0Z&y6+NNz8*CImGsFel3K$FLXqMC zMQZLaB;G5v;yc3sEDU(s_cOdnah?f+K(-yZl=4}iR$X0jpxfPp}OKp}y_zD0)7KPKtV-AM1Ds?$IL6Mi^N1g%pz=qPe?@1$WX}_{jvyvcs&LM0(*T0+yL$@ zG-sA9V|E0hL&aUPQ#&{``h?R^f3}qFwuVq2Q;YMtnjgyYkv`3PgZTDJNgrw4@@fbN zp4l!j2KTa_KoF!4sh+N|>n(P`nLTx;>tN=Wb+@J!Q8+)6Ztw-@sZ!4v<^)AFhDL`= z=UALvn#NK|26VkxW!c48?41czxq3M&QS5JvBv!a^!gL-5p;vQIVG&!Es?Zv$)aYbZ z!f}R@w7R#mzWeWUL~Z)qdjTGw2I)8NfQ->D@QL(y2bLkBJ{rrHAGwDT-PH3i%g&r}ee6^uJ`=+zmR6`9Ho!A-|A|S{h0mT~Xn|jzN0Y+$`B#pRs z9ut%WCHzW=T!c*yJQNzC3N&G!e4!G*F<~3}z@mQk8-5M{;j4}|M1FBUzhN5o*#?)m z@WYr|B%D!w#a3Lu%`XDZTZPH9j#7q@_^pmQecR;__b?%{4KIMR?O4_I%F-&;`y>{E z=3SDynWmh#A7G~4rb#;R4)O*R`AC@v=Kk&VRSL*JM^VGELu(YG?w6~Uu&zLA17q3c zZ6ow1X9(s-rT9J zLfOVdHV9YI2Rx-(mDMRENWIBQVc)zw_E_rfd;#nqkYtb%nkTN3RJ85WECsvv`}5I- zAZzBv9!^bZ4^YEs5jc0(6~*k{QXKBAFE35dpC=!82#lseMYtE_|6iEn%4lpsmj}=Q&3lhYkSc?$&(Oe<@g*j)0#1xJ@k#du zaCy1Vu<`-JC_{)XJ?3D^vU;vf?ne-S4NDj?qBe~y%euC$4^+ym`VZ8xbHi+UE%XdG z0iNN}qetZV4n&Nz1-BuZt zXmUlcSs8T%wtmT=VG1di5pr+uJJ)$ca!{HJrcGXgG|ZAFW_?w(HKemS$+=Xr&L>N2 zi=?u((Ynr5b2Mc|aHk8bB6rK+@AoJk%$LjV5g$dS!w&7ya9Az6=dJ|Y`hV>np~O+q zsqb9|`;@(jqH7Vpj%>@YVB0b{iM@E;n5iF5OVpROD6l*X4#f%R!6{s!GI_&X^8`@+ zdJl2$ghvF%xX`pcAsCIZSrr?4sMdk2Ht*DqdgA8_)-w;)2qOuEN7&%Ck}LP}N7%iE zxXgd;qb#RcX8u*rbDgZu(M7!<=|xT%5`(DQQ*aMca1SJXN1EFtwpz4sIHbacxn`Fv z6CY{n^2npN+{b3t#XGV@!ug&IC;LG~P+jv(U5E&D`zqndP6!7=p?#h{39UR8@3 z%2hF3j7;0EQXdLbxp+05#9Q=mZuTasW<|`(;dA-Bd@rR$k?Y0_QcDq+ zpjWP=^7fmp%}?9b5&p3d%$Jc{(yA9K#khTM3X&xm5rkb^)xH^G(Z{vjVE8~Wkit$~ zno1RUXDt;vIVvAtkmXcAtLLq5A*#JKzSf7qM%OeTW~3FLXEo^|f5(h1*KJbXe4ruR zuN1PF$MGQ<4!eEs%daEI%1)JUdh4~Z=#W$wRQa>(7)G%Xn}&!;D>E^YzwmMr zimZ*AsG*HGgu-k##A-<_Tc~Vjf($Ok)>9rgOVpG0Ll|TUU}Jm%NlRqDk%(zZtTMFH zFSbY|L3c^#LUHm`tz3wzSU+qvVW%=bP9n6Y)*txRtGGZVNpqI*%b?>#i{N2sh*I6$lTl+oWmR(a-ne0kx$RTg z<*0T3E2uh1B)-`(^q5)|>dG1vO}7E#q?>LlpDJ5~>FEyZH7u>0 zfSW1d*FKGhgx;*K`fcA4As@g9KOZ?VWvtR#0|A4wFrE!E9J@NtRi0 zgRvfbPctbv%OI4-?rz)f(eSz76uq-LA+UOm^2@)!CQ&Sb&DL7_XfxDQ@WAvnJ+8#U zH90eIdFzH5UA2yzw2rItkf{B>zW=KQoOGTa(ytb;M*rrj3ZOzqp5JzE{%=8!N zn!&MxkRGA|!wLvq1S5BE)`rijcip?^6^Ub4P8FX%%+d_n@P@>jNc~VY{p*C)rQxU1 z0Zl%qch;JCeRvG(pY=-`^tg&Mmqwo^Wd^Ff!^PCMqKd>dj*MRb)(kh2mDJmAgbi1j zXoCyK_y#3xHn_7lr$)H@#EFBrJf1^okID~6ksZ`61g=M)WY(;Yn{;m?dHq#YUk^-& z=Cv%$<|a(J7gC%>ydbB`mVMC0)3C*KAE&S4kbh&) zTlw`2siI~BRJn<8$uckg`UdSuymF~o$-O01iJMP+^1y?Y#HcYKqVyZ!2BhrXudDn8 z0KI%p=e+IGw6r)LsO3mi``$d}%MI4ieOVXft>+s2AcIVxAqyVER!Lzko?Q=5y?3k8 z`fOz>O_1MlS+%PQ4z8t6sk}Y|JP46Pr|tBaoj7}W(#iXzhRt?A{F{%TE3+mkzlbi; z5I`Hk5J+voR0PGM#ev)rM85GCo)`!Z;iuw{t5 zHzxBc@yv#X)ZHA?9=URhVq5({xDqJn|04k(V0 z99T*@&QV&~j;hBKN?M5oE^~9!S*4VAkEKX_Dbl$tN})4Pe4A@ae`459;HVDgHt(*h&bPj4r zlUS$a`H>Fh9&G@@fzmPm8y{f^CZCVFC(4403WQ6Lx&Xkx^cW4=GiiZ1WBz$jCsr^* zQi*PW4@Use&kmGgXj=YMGLFQGkb(z|?`%Rl2#zbLf3`oeDuB-wi%bnu=BiqvFq6E` zvS_S^?<5OdAZ(EVx(8EoM7W&**{>;%EXI?F)5xYLIfNpP*a}&6f)hArne@$IwFyvV zNa0OLsD7JJfNNBp&{ONu{Ep^2rk>%Hrm*!3KtPtCLsI|TNC<;MAc3i>=W*TCNg08O zpb)B65V|=3U~^0O{qiTer6CW=CDR=UH!Bb{`&F(-U zrGyLs;jJVm{YapXhoi{yQ5otWIYe)Az*lhD51EErHqpO1E((%I-0^VV_Do>02^nwx+6<34_;}dKr47s>diMW;oK_rJ|^RRVZOg|3G zDz=^cl9>^=eVi9~CavxkYBgUh8Nmwx)j*s+Zl#8&Lu7d6%ES9zvAIvP9Xh_IR3C6& zT7TU89rqbKrzD8cye`VRd|nH&l*LKd0LEk!6gQc-km0@6iV6@R>TAQ}AI6vsR;CEh z=m#pZx;^&e#j(cZu;3H*dStzkK%MN7wo6G2mb{ zHhnafE_=Uumy1L?XxLB1>!MKL0EW*^Cc+PB34+?t~E|eF01hCW?+;Oq@|qAkh9%U|8x8T zik9$uU_xgh^LL8eeZ!5P>-Z*%R5_WLKQvnSQyebQzd;kQyCA2+F!je_XhUFydfe1Q zo@atTRvqfpt(99OuDv7ZFNpv6E<9PUAJ$K7N=|sHE5avcGeEg#m`$|6z%Ye>CJ+q2 zlfT1D+sHv58iMfQje&Fmxz&CA1JZs!9=;2&eR`4?G*%6`f8G=I19j$#Va+0=GBt3B z7d_Z%!Tj^lXbI0L4GGRFX5V8BczO+tM72P|rtTEUIV!hG@C%@WeAg)u1N}t}`xPWy1`>kU0ilvx5s)?#D1LtC zdDk;ApyF0ju{3%vzTqe=04ifNK*0wW0}>h)q0fo_EwTWzDFlezvAsax1xm+i-uU;O znje@82B0dF6eiL3-bqAJX?f(MWAHB}mNbXqK+Kk=y{Y!u(#2%o?P_$%tv8)x#L1Op z03=CHB$o~?dI>@x)7W82(VGA_sxYY_ie|l;xHhEGLn6MDCG=OEjZJ@lI$NSSVr)xi zN-@wq2t?zVyAAD6n38aH^V;<))H>pwoPs@jfP*)zTX8W66B>!-WwSaKiAk{)Xhvfo zW;w}F%^2s~b2^r@t7fhk1kzuUOE<6Bakz6E^OngyL*+$2zP@{%7b55;v_bA+)l|uL zA@cW~5)!!@o{FV9e*=;VJBes7IhR*QvS#^D&|**Etr{V_9+;rzSw zdn(zCVZDKcaaKmwgc8-9vY9AjlWmK5>q7>&L16r3x?n18;6T{9_)Radq*ZT4zL8*9 z7^1#h1AYa&1(P_^0Uxq5q-*x<>O_b<$oyb}RQRUdB1h}#0=SkunJQNNksugjFH~&>@Aa_W+Zmq+_oUyvRkv?K&$qV6!Ch;S z4C$sD(ZipBdw(M&9#aMWZFDN28sV-niP`LJXRc8kAHo||m)J&AHQh!=FPo`=C_<_< z)lM450OP^ESwCtM$l*`3NCM$Lu-3B2(G6Z+@h#~w4;Z2r0%iC!;mXpJmOp&>b^}Q) zQ6>_{f9Y&_NUT*sw7}y><097Yw;D`2j&9>FRTJ;Sh_9v#%w6WnIe}}upTrsFi98@6 z9t_JDo*m2QRq1^r1C2n-z?WR1{~i*|?ppct8Q8{Zyc!$>kc59U$&d^YdO>>q1QyY| zcNoe2qI_iR^%*R;IF(xtAok6u5MXJgpIPJF5z zY$k&}0lWbrAnIF;`$`OSrA`eYA@bEFo2pb9?vb>e*|dVH?ab-663VEcG4}(WL7&_SSVfTi`$So%5G;#qQAelIVuY z^2xA#+Xkbco0)>fUu_x!PJYhJt$Z_Xr$ZFGwAEvKr=P$dVvtI_7&ea2q)#GrtF^<0 z3T<_1vFj+@9oziK4*g+4$n*su={0P0(+qtT*zRhbW9`Km`aPkc%c$Wn3ZmsfyJtZu z0v{ITI>W2jAg<0ZarP!TxAuht*m3O*{4xv>)OC5Ug+Cq6_XNitZoNN?n#SCg{8nX?- ztP;S(LQ)<;WpA*!N*JP9d|nutS;K8$h)iZ8oP)?IAczG7-_T2AId2GWV$S+SB8ntM z6x`qRnJdwihC7%fY;u*N;DiBEkYH{cZDb&pPy-)adxHDc9l4rtmjx!`v(W20ssdv& z?uKpIWN8)`$rWm|N-;E~hZ72KLnH>tS>?pyW;Z()S-icUJSN`U?lfd5V?q~Xl}5|p zk@clJ@;E^RD?IJM;3Ut8Fwv@q5yO;ks=-LVifu)VIXG=`7T3hU>~w2XnDthL>ReNG ztw$O4n*<5>;9W#YU%PmVSHQ({)pTDRrL*iOAlHgTeo2&`O7F*QWFb2bVbxNVXjc|0 zMZQwi**8_+M}_x8mT->vvNc&4{EhwRTQLb;lUJ=_H8Do27Es^3v86;1B zXi1aB<}6uswuibhhp}M{7JO_J2WrIUg-J~-9gE6R-{+E*7EyxSspQyYbckB&s|aL} znj^w?f=-b+^#WK^KIbk?eE-BA{t-B<66PuAEq(2R=XJksPlu*XH)??tJG*ez02S&u z5?(OF>vmJoHhjQxxYjD~lZtLTYf0 z(UT|`;?AZv;diPc+H@;vq&wo+0x$w0dja@=!tCyXNdK;wSy=XDl;Y}+SWbtqKrhLF zi$&ZNwRove8A?gxu`?tRV;FxJNoHySkC=Fe!W}t?%b;LefNt5aJ^VQ^*Pu!^+=az2 zz?jRS&b`#&`0@qtxWfUpBtFEjXRyAdqlN;)6x0t&504V(lO6&ZFVO14b*6%LT#U%d z?A;{-ZI&IO`N@zhAsROo(8nU#Goaa}$%?>9WCWP^wK!ZKKpPsMTyxoxDh47mw)CM& ztUz7qgESd}3eaVtftfhxk5prHb)*wqj``TyXd;ZVNe+zd%T<)ffLgzX6=ou6KE69vxTDgVQ86+eTzdv00uhz z8j}Vx-)!zT<7ecy2ZQ=4zjlE{di7Y*jcO1-id+HyckGHER1eJ7aXW`m%F^VV=dz(O zw0LEE6%-BI_8r-?p$j2=!gxXoJ~HU7te1GrREz4M+lhDJ(tXP_o+Me0dmJKEy+TNR4i0IlSeDYN?07!1y=9BO>s2?W%5A9 ztIz|%|0dGCB)LXlPPFv7EECi=bQx+%L_Jcd?)3YX6=i{%R5c2@B;{6*7;NLBj5CA~HARu6n(2(FDpg_8iO#M8hf5Y>7NWTsw{6M5pMn>sxGtDlq`<|w&Qxr z24%~+vhjlvT1idV9&{?dCCXGj+GZgn8B+_$@(EZiFjS~ZFUt$yjghO$sW08;I_-QJ zbtHNTGP#h&eBDhQui%m%8&Yg4Ni=Jh1+z{nd47~8z8Z7&Mo7D0N%31_vsz+tS@Bx3 zLe1e;6sS%+wGvaA==iyOM0yF9Z*uv=SyXfs=e^b)5RyFy6bsY2Q^kl3b71-{?Icoa zqf6xIIuDf#N%^+D=JWXM!tVa^sC-Ur5Q;r1WQ(3CvnWYINgDm(mH}C!5JGvXwI3;f z{Mq7nQVG^_Pus`HgykWN97u*-21>1G<5n7D$(Nwu!e|=Ar$AIa@lz^eh4NU;q*ZOD z^T?}<0ls|x#AIk0+90}%y3!mO=T12y^{8w!L!!)BD4^uI%;`1}SmfO)$i(rq^%m>= zaj+iA3wARl>ckr4{v_IZ^^w<0#*=PJU&lDo`c zb@7^oDo!?}^CL}9sWq5)$(zZ+N`C25oq0q~*w*QKo1!s<)@vp@DiLlg*VAo?ARMG= z9exENDjfkgKP4+eWkQ*gf!^H|_aT6}fQlTjpJ!N7&loO0=%`qLA38hAPv#`@0*KBu zXDYUdH}#ws!0%}>ED^&nXr8`sRtG$7d4D9%!{i`-UE!L%mGJS~ps;1CZ=s`8B`isr znF-Pw_~+WxT@ptJh|+{^1?%F}HEsoSJ~$1URoR(U@)PCbjs|}Br`F*0qbH`f^apld zyfkGC=SbqMpJDRJ<13XU_*%#3=+eVw$*bgcho4t`Y22?Cq*+*5f5~>}FI^(nofv)h z)Hw&7S#Q1m_5vES^f86OeC*S}i%%tQ8kIS% zpI4v1Ue=V16MuIw!{b?LV#;D1`H~$c8^+OO4!lsg;X>itM#A)_6H+fmumEctw0>s&Zk~f`MxO~an@OHc3YGh znQ%;#N{(xx_>nNqKxip5a+M+ty}`0G6SnMP)iSQVv-St?2Y#dFCegJDL$eWRY_T@h z@U~D>k$SWpcXkT}Kbdftl8pQmCH?W7epU0fof@TcbM>jH&|h2dEIqD1t|C54E+`W*;elO3N+Md#z>Mq^k58|8;fV8DRD*hl4xXQbWS(o zEWxIhFlSZK8V}rF0JpAX&7b2VjQ4VcWbl^J9r|69#69(83~kh}ap$MZYL4(QUI6l4 zaR({aBKIczkt6Um`{G8srDY`kYRcRLXHkxxF`Kvt4}^VOGuI;F?j_%&Fm6;9aOsaL zsyPjobqrnlo7?_k#_ykUG~c(u@tkCAc}R57ni2f3G{X8yOXm4#iX8m`F96v+s7s+8 za;i?1QnRJE)U{k+syfW)-3{7@bMN!htBktxWlq+Q5G`xWNL5drb~GWdZN(glj(w*V zaJKzPXPLi8f@pUyy|t`$jk1u}*Njq}E2pYGO28v_#w!vEbU%DxhEr`|YSC1G-40kW zV2qG7Me?@c=HaR>@m2A1r96w9ABn~ZxOv2AT1z0`j-c5Y2R~)?Ckxf33LbLxe^c?J z9U&to|2UnXZfza!-eOmhKe3fOnsI&F``(okOsWeSli{fx&)-BxPCc?_*&jLGd_tv* zadK$;ZRh5F)E8nIZ)MU6oGOJi0qK>lvFL?Lef%+^RveBoG3vwXv7G%q~x8=_hm7z)0S`g;;Ctoipe zZDWaGlbEHUmgp^9q1v0z_$)f|XE@A=w;l&ApGY3fx)KUS*VMTQlf{!R;9U7rCeT}x zVlRF{c9TRSK{gVbX*PU4{MXJ@Ilq4&N8!rbm|d(@&h6@ncrm;em7yc(aehy@n>2yP z*%o}aeJL_doBgUpXi`SEAV^AYiQ^2ueZoPiKCDbaR*-ZLt*2@zGlBRtl3zldQo7l4 zgSyrxZQ`>E9ifxBr>{jUuBeg8K66j7b=~i31i2*|N)E|6kvqV*7=bqgJ#Z3!FQ7tm zKoV1eSZj`uBoCf=Qp&%f|UJ4w?ti+kXtG)uDxL5Xxl z=}6MpSV5pfZ+RtbjmLRq<_fh5Wi5i(F8$kKcPn!g<^q~9&Y)-3Y6>d0f$Fl+vXHV2 ze#;1)RLLN^ZMv+c)`8NlokZ=qdnVYq-BI*_bF%Jw`_Zmo;>8h{|K>h3^CPwisO!;rSl z=qY>F^FhZB`teDEjVkIxKim=Rp_wPRlU{le)V24xO+!?4Y=xn^@|V6mh3(?B#Wkko zL8IkUv;#SSCUr638KgYCCUW*>w z+s8@k`l>37Y3IdCZ`$R684Q*KBanelOfWN0@ALv@(4CQg4ToVb!BSd&LMcWoJ_^>T&NRY7C20RLqCM*#XJF2%_@{YLpSGE{Pknbd+1 zvZ?&V8sJg`BsYD*@C7ghzT`c<9}p#;8F)1N=@J3F(vzK?`E6~|>9XSenbX#N&c(gO zc%ie`Mq6O6*Nm7<|9bPoN+{qubGVy)R|+Xc0d6#t>e7lps|{%Y7-^AC6y(4>P6Q>a zajdYo>YKjKq-@2z)~DBRe@ni(A#L!H#B;~lUcD+`*rkR8xxN|`+3s7ovl^ATB%Zn z?h-_z`A1*pDdmeKVn7_66np5XGa}6#r13{Ys#P%HOzBToa3|kXzFM}pMS*dQV21d~ zyRmvHcxacTZY{0lHKCs#$F)+Kn0dN*vOaL#5VElI;sij?_4SjzX0%LezL=~R(QB)(P<|u)lq66-+IVv;hrs^DcImb zY0xbu@fI(jF5uFftkG{AKZD?0)se<4I89p7eUi*A4hPA+b9$1f>ULR+oFA7gZh4c{ zjFBXjN5L32>8e^{f^ctZYaAlA^ZLPOyw%n>Sb{6EuL5>O=R%Jr`xn0sz}~q8Bl4bP zhC9xAYxPt47GHV>Dv8UIxOxyNXmV+gZrNT9X`^v5kWyiNT{|2RXID;#1$5hj^ZOWu zHyR{w)2*DGI@L$TmQU2F!3BP={NJg|O}~>gqE{$tTN(I#50&x)ux0&x_UvBzrFbT8!0J9y=aW)= zFg5ea{H0@t)*=Klri>6z&8({~%h&1{A3T}b5}zOvk2K_ia8{bY?Z;B3Lp(akUj|?o zrmOVrO>;7>B*kK6O^d!3tGQf^; zr5g?RDN*i;E3|&t&5!R5xEPAl1!R{;hVm!oDuo?A&rU=Swl3hVMPMJ=j=y|8@AG@B zdEeUoYvVw*To%M_Nnq+_zM88)<~cb7>$`uc+q)C@nSz86o#sTE*av1+lj~H+U21*= z?Zo1}2Cy1=4dP95XR~~(8*-aX98!R{KrNk4|N8K>AJIHc^oBUbh_GOfL|fGaXReA~ z^C?q#uj#x(J&?6^V6@d{z5=Ep2w<8MZIwr{-jrYN4QI3+XuIr=j5@w>Wfhi5aaCyG zJ`%pv4;y;=^IWgO2a4-sP#8)4s;yjjlW8%T9dD~Oaa%ePhRBpBcp|4K$o(=?sc(&N$+bE1 z&X)Nbbb-c^lv0n4xS5cjM8TY7$Mn4(3v>Ki()Qhc~r#37|UWcw^vYGDOX8RKY>%(U8% z;I?1);@3YCc-@pY=r4vJg}Gh5gZsy6&oJ(`p9%3e+j|UllWxO*u?dXc43*6;Cuwel z&Zu^KNPIGKycMDb|B@qwy_7W*!CD(dQUSMHn$fo1uBPP$7Ckwgk|USj_;;c(3P(e? zey6Dqr7c0G9b5TR+Jzt^45HxVtCeq!rT! z{TXYJx0X@pHxo;^v?n?BoI0x_7B1~=k@IKXo;Xz-&-d>QdY>+x`okXfj{aiCO3yG8 z53R%!>u8GyTGX1^#ae{nP7qmH`!t1O7Ep4WEcxvP0AV8+!R|qvuT~<8A|2~GM*lly z*S8d@Bu&m*kUfgj!AC;3n4oI1Lui?0df{aCwtfH6@zWqMjH#qfrY7Md|NOvH=Cgka z;{PyKj;8wWU?d+;9nYM(|I>EmWw|GP03o}VIPM#y+ zhBgCn4emHry)R9jRIuqTD4aLGYn0KD!pW5D?0)x(U>}|JpuKy*%aOKs%!DU@96!1! zU45M?E#vL~KdMWH*tv?=ja1xN9Q#4C5Ign+Z8#9I;rvB@FMuKCFUTC{v*AEQg7Xvl zehpr8Ao#b1pgFb$p=KAQZS0I0o9dD;SR9-GT`lMf;34FY>~}oWxqjur|JD!w)fkJM zRR*q(YxZ)OQY(yEJ&*8z_Jc|^mEg!+Y&zea@}E`w&6rSYD@f};(WJ&+bHphClf@{Dw>9bLe&2!Az z1?Q0WK5a3!bia)3MO*~jTZgG3LiZB?-c^V7D_2Ys?W>lqt!efV0#%zTxy1^`=7>T? z?uan(oH3(TOz$GCpo4=Qt+mg}IK%et4zAw#J7Mqj&2Y00FH%etx(rtjGRNK~b5h7@ z<{WIQA;~}Vmg@{5QjCLvj53D5{((6jdsz@0B`AhBAx~}mdE~^ErM>t}Czd2FmE=kT zWX)6ms6yFO!BrFoeWKcuH`x(FQ0iJKF#B^EA<=Ng^}-~lm)wqj^~|8?=dBCHE>B9& zHU}q7n<^QyE>xm6WRerbCpGy=_t*boZnz^>4dzr1J@B@6=XbNZ8aP+W#2(fSL5W7wI3Zma%i792+rmViRNH z%)aa*(O4^0IViR>KrcH+C&?OzNII`lDce~P;qnejPfxrI_|}{?l6Ev?It@p7_oS#e z+&~Omxuc%aNh*?bj%$^qaza^*r9B<>p1?tKYA(Y)Vd+Y)HH6$%4z0-%KAb?Q6@jZj zCpAU6Uqmj%$kGf)+lhuWurGRR4x$3K49s3{;!4b3;@oDF!=F7dze0Wx@_F<0!qbVG zp2~Ya>b;sW!m)%7P`BV!LC!&(D}n0XyM4Pu&aa7>P2?!J*6zUt_x=)@8uwA@3qz%v z`J<5gVu3p`+xhK^D5vWsLLiZFvfjIR-7}Am>vTNMn-Ewi)b_;tAoNpICo+{z3BQL!BRUl8YTE<+F{`OS8zsn3x-*ogK!@)}UAAf7UT=3B!~)^!GN4+0x@ z#}?382QM+&BtNF}>i=HOgQa{eRec=io}8CvNn_wzaWs+b70}ZQIzK*tRyd?QCq@8*_t= zc7x5{e1C7fRkv>4s`=;4Gd0ssJ#(gexk1bMvdq`0iZic?NQi+l$7x-Bz zS8~q5R!_~$!Ri!IGxzzr8@(zzXS)H$j-92<2nZdJN>iK`u3L^niD}xgaIIdKUO|o? zJ%%9THrY9-M^dAd;*jbWCrEQxM@_M0U*nu}ntEX^aCIulnu=Xo{zgud=|}^mD^(>w zQYO>k2umGj_Ky0$gHLKBXNC&h(=(;W?5m-QrFB@6jN>6fWaGBoQPiMu^X zUXO=uU6$#{o|ElEtZw__{CCOUSPyoQy}HMKB1?CpKD+2C~l=? z5V+}c73S;KnBISkc=FmqJhhBY{??ow39gp;N$<*A|9ij$dBtKHirT-K4kn%Lk}8GCy6 z6J?}=x)~O^DsVe3Bl#p|eQy(cJH3DQQ_k8cZ~X)O^?abnPeG*bg(=mRO~9N3IU*|K z)-IG&cj4THoshNRITXos+NzT{rk1DE)6s$ko`yr2X5h5XTjif~%u#MRYM0V2^u{K) zk*J!`6Dv=_1OhpmCkwHlAxJ=4Hc*kSUFe1*tfEib)Agt-&*?bs#Fd59JQx}zO_d`ksb_L|E#VHw&;vE} zi%e})#|Sa`@L&L!5{Uoqw03B7`N^5vCCIV$+nW56YUEY9@4{8fJTbonBtAZ>bI! zxJ=K`ETK}gUVfUuncI*?2D)Mc=1_qhHgTx1oU^>3IK+5U#r|A*Eebr-?rj*})ZlOe zE}f3}GH zjuH9&ETj1k@O(3J+;EM;e}SrSLcrA<6o8m$6Zhs;+d9PH2s?0!xZPcz9uf@qjpo?1 zlhZnl^aXs{kp`L#bG~2jOt0Qcs#bD^#T-+Z{25NVgZM{Mp)tL>s2MiKg->Ev0`j^% zb65k2%|g?tKgDs~JF65^{w4$rqoh_iwO6Py`bzhMPN~I)CIVJXU5sLOuT~K#u;HLn zHonwNV)y7`RM3-|6>Sau?tiPNJ_3mhTOjL<2!f{V$^Pxl}$uCP@v<7t*zdoeb{fyIS z80{FuF-Mk#0y&L}VG(dFLAdTXsqkdal?TStkffjj;k9WTJ=5v*Z=KntRr^tdK=c0A zB`2-zrigNFS3P?VrRGC1%&TVl`iD&{_2!fdclHJoBIhvW4i-*ZE?2$i!%Zl)(r%w(&20`Y7t{0oDGd zoO0cr@e#1EnPRp;a$4Ol8#X5d9__!g-x0D*xHXm}?Bg8ECi1N|E zCT0~0j%<89S-ld*pj^}2){g@ezxV(OgN_JCVYM`^ACT)x%0I(yf9^qT2MuAQ@tjcp z6#a$ep0gKdG=Ddy`eqy*&H9Porc0&+<#U!*5>=Yo`1ukv!-Ldeww@#i6h7m;Ybk7F zjb~d07mB3Kf4s3?gs+lz4OM90)rkwdBRQH&KK#w>`G=U#-x_Ol(rM;d-^Gy*ta2M# zrRqejzEyssM80*pEVt5ASmUEttxt2Vup-K|m1`5@)_7k>{EX1RC%+j@0Ilq$2xIWm z>A4`R{zhujv&@qfh8ki-He*haNDfk6Q2aW@&HM+@-ZwMU*OS%GIy1;_lO{x>pw7)$ zQ0%Tt>4tRaM72qsVU}w@LE(-qHXR|PXRZt#m%~D3%4FhmwC~K40sXLR4qwV#qxzAZ z#fX(w9gwVNsO*#GByWN9A`o3Uh9k1_x^~HxRzRDjMU=DVV^HEvKi%6bY0E-U5yIZH zjirCR%5P;Llq=KP-FUT(#HhYd0y6gC>U%CnXFyN8&vTBBCOER4v{YZ)@h9gel%83I zslclz%>DuVYG~Kk+Mp@)z(A+kA|Y&wGF$I!qbt^&tf!Ii*QIo3?+Dneyt_2hGa624 z8_GcKf=p~BxmiD{qS0~?k+k0Dq!FHc9Dy|fC5S)J^RZ!VSXc|!Tx@jL&?rfCbG zYgskg+CGcjq*whgj`Tx=A-fHzx4MkQEZ>#GAlBM#l0yq`&d*U5K?+K>Yi!TawFTnQ z*+uh^{g?LGFsEAZ-W(}>CAWuyiJWkbyw!GK!bIsiE z=!m!C7{&gI*!7jRqt?!S8X4fU%Txds>nn2w)3dcHPBR4@tbC2xnRbT>=b@ux{Dm%} zn(t)_HTe&aV(z*cTbYTab;vUd)oI&Zz$~|!tI+|cG}V``?#?Xf%VhA7ye$ud}UKOebn=ir7kwmC0xOp zGaXdDlmkvoZ~3N_i~VQa(;d}CmBAKyRzBAq4IK<|g}&46A|Vzw!_ho_K4^ydp_R=E z(Tkx_q$oJ3QxrEun~@zPWnj8eHf=hDid}wrLNchht@bsgKh0HSFQ^fjs628JXC?fFZs-5*g~Rvm)jN znnO!qr|Jp9KZn}KL+LXxce(Gy>gJP@9mAt6rDy08Hg-xTfw}p1iKjsx7Dj4+o5)`O z&~Xc;qaPKmR4<|A5Uw@uerKsZ6>Q-D2e_mL`u+n9;s)|VmwYRf$bPiEn3u~q7PjK3 zX!ynURh`TBF_xi@6@V5-Q|3gkLyZWddcCqAq#AA9B>NbZ_hxiNcmM89qccy*ZxYKB zZa<0n6HhJgyUaqpAh|p0k1F~tFR5}JUv7y``A6~hLcUPV>R)Iak*eC7=i!KV5tSLH zEQMV;axFGm$3Nq8uL7hL-Ow)4v%UTRO*As`rFyp>lsTA^WF~U6iy-8?dl&@c3L{t0 zHLGKG>q#9;Z8*e`>KwIbMMi=wws`R2r6=*7+INO(tCpqWD|%)X`=lL;gw?)f>gXxYohW#NA!t)zRVS~@-*JM( zM9`Zb4j7dwBA|W}$!WpOB;OK$_1m-G$jz29gy)+A|>ybA_Tw|rueAQ}h zeQhK8_~Rv;)x%k3IQ%_={s+Cys;fHF^v>9m52ptYi@!wjZ9%Ov-kh@n%iDe(lE!K_!^%YM;l+Ux3f8HpL{by&C;2KQzO8tJlVlh#y8`!4aZoHzfr5;mOj@k6m6(ZpHHaw?I zNAA!AVrC#uQmqt%ud5}(fabSOjRZ~*-O2@HI~F_3YI+5ft!(YEJ1b2+_aHZ~cxD!E zQmqWS?&MI6gw2MKUU8QZFjex)GwbunoOYIsgYY1nwQGKsOvZvOLA0ihB8xA4Ne#Ao zwaY&MG-W{Xn!_q@vg9nH>7pi+YSgbF-@wMY(^%Z5-fx{=rfItRYA^~CMg+Q-#Ir;%tCfZ$YN%m!TBTLC^~lH0pH3 zI7N4iwq+ruE?_<%S{nA*(wa?#T!bdO4mZOc!Bq7SB+sQMZ{D`nGxiq!k5Go?;;Py8 zO`GA(d2*iADGEb{v033{F3>T>t(lZyy2NNEmV(f5GtI8w{R9NgDnj<)*cwy5tD{c! zV1&r5uM!|*H=PI2Z^&{p+-|!(pz)fct~8PL zL;vUkb?QEy>^JHs39V$mm0~mBj2k6JWmg;pqT|+JOXAg9YEjy{w)Pr%ko_H4JDzn{ zb}OL-WujSvNw>@=%*tme6(L;)sOI#kO^S)_EMfWtN*PU~3tBW4?ny7%nI>?6JH3wO z^;w#klXmdx4o8tv_tw+BF)C2Ggqv+1LC|MJFtv08HLVo;A=o$M zjpO2$wx3bEx$$_Rhi})AQosiL8RzBZmB2rmJB&t zF~yav*HhYGF>dMf()t7x%T5e}u?FX~Cu!@L4V5@T^s82O4O>&#oN3k^ODdPFK zl%)Si`s&jdJ(z{V>4Y`5I`h1j!PvTsDbFs`O#>R!VW>fX+&-J8G|Po;%>$~K^wn%x@3nT#)kob`AlE-OD&`N?18eJ$w2W641{6^9WP>rCJwrOV1Xgzr;v z+g1r=TjEFYSAAe3f~`a+whUvJ?o)>U>~pDHi0569M2*~wga-@^)vDMIU;D7L#X2If z&DyLD&NQs}re-US01FRF83C_>b<-;wlX8di4>TnytcZEMrmFE zaVnJg{9`A|v0~j)g@}$w(PFi005w1Zm3|K(-$2EZmCiEX(vBz#&*$hN73CZO^Pc(P zUmxaD%aGR^t1Dv5OtH8~U-vOFKd;1dG$qXxo31RE^M>&^IbGP(Q}(`}oi%r8Ub!=y zd7o(MWHB-IUQ#JKJf?0GKK(`6lqIDPyHq{P)yYz&_2WdcB_1lzKOFgQ=Z5oPt-lP0%2!qS6e{vTidFu&T)RXE>nxDze-S`Vgkfd&FFF8Gdrpv=q-F zB@*!cy&HY`78cLGFSQ+Mo^NMZZiMLOs1x7GH0^Y$NuN?v^tWiGRKrHetCYnL{X+`# zo6jI97%RcVLz3V-HJ6YKt=8;C%jS-ysPUocq?b0RZ5VHtKnXSAz)+T2pib7Z90dJnBjNh0lYps?~Cx7+#$h1DKGo2TrKckTFtXg3Ed@AHjv zVA99z{7C?%&(`fqn|16t%89;A9nEZkv-3(Btqlc@+E_ti)>5q2r%$)Wg2J6{h9I1G z@G$;UKK6}IWW{`y;?HtnwrnqH%-Apnd+Heiw=)ER!r2)<<3Jqigw|AYpQyhCzs?@+L=J~LOVxtGS5EX5uJY(#jsJyjz|i(G6f zq`L!2dxQI3M?9-@v7uZuJ#T(%|D4S-`yeKMTD#G1W8hUSbdn)5MJA#q{13T+Mw}tL z4dV5Yosk@y0bHyPcW=aRlegKPUBgG=NV=n)c5V4Cw3Z|sw~2K2VRI+^p&YF)GDSNq z{AYho|uv$OW&^IV5j-^w|crjdvZ$DSoP>&j%{7g}%v8~&E%MZ5R60V^`-V~{FY9440#jAO z5CZCR;~CW?V7H6PrqyfIV2ef6xaLk{{#27+KUqu6($UA8uB&cIROeoNWou{?mf-Dy zuR)T@sa?yw4zHl)&Xecn5(m96RB*&F@!ASD>t1@}qVW%4V=Q<6h0|Y~S8eX!n7P)!`1GYjeB|VvwkYVkz`wgs>SPYUPb zHj4nZBS2oQCj~tw=1h8L3_sa(8n$#d5q>xYO;Dz(y-qtZj$W6-f)5&+FGLSp;s^?e4pDhyW zKTK5$UJTNY^%w!`UH<^B#Isj?l}8W@waX<&+$Z3!^}14Ddi;k02^P6d*{QBuCYI6s zJkK4?m6om=t5OZ4MTA*Vi{Mg+wZ-i?XUiO8RWE#_7r)gNrR3l(fAYorxik9XrL?o~ zzJ7;oH4u~I+oIv`PsWE&HCevqFpU_84?m>9T8o7IL zPyHvHz49`ck*^|N9S%%; z$X+=$acq6nX6v9Hp?5*mV^-P!y9=hkXw{&Rg5 z!F5XP;8}!6_=@8L_a`J-) zxo0=q%rZSB-MPwL$(A@0U+Qt5jp6ODWz`}rQ*6pggCUje#HE?n<@r3-NpdIGQuDM@ z&{E8dKL^~n#g^JqTjgDT@_0y3=@uRJa|GlT@N}>q^}-wUL@%@`%OPkdHN(-cr*7+1 z%YWa-Y>84;HU$ZpwUsKs81Dt$$2B8LuR^5)IT@&6^p#|Iv-Wf`L6e1{e$U@Hpq0*u zW~$_>WQNtc(!OpV)=df}eMfsRppKAEtAO|RC*Q2o&z?ol$T$cYrXEs})EdlF9Kc3DoY_nGL)_ zJ45aWiE6C}8NXw>fqaONLnDvp`Yoa zuEJC`iuCWSlq^de$JA0gmVWdqcgs_y^7$d&jAsQ)JqEY$c@!X;){$$rmD49_9RYz) z3>%^vR9QTu+ z)F7mWGqjnx3plLdEhnMIf|O0klBrgg3-pJ$DYDE}n=RFUfGPBfQCxLqln9#pru~G+ zA1@!~UH9$sxNAkyg;!y`LJOIh;#Gt*0@D&tbF+Lk(CE~c+CBaOV3=zcP#PVCLd7*o*|y>Q1v2T+bH-d( zp-b&~rL(=04!%U!GKU-BB03$#eIszSo9kw@`AL7h)BZXHyGkhg>aG6i{X;!<4&b$4 zBTCQmWL{t5idDa%9uUSdoS8CqXs|5ml8;mkeH(#qA6ja9EB`*^$#!0??TXQhnZto) zQ(peoW42Rb$x$thIw4iAs;gK-Q9X9=OJ3C*RfE=E3s9{*1J`wo0pFZ&WDXb8Go*ST zqm_Ptv&qPvs3~fb#VUnR5pwpGF6)t1tiHzAz`f>!L*R+7-%VovVJc0TLL*{~Sn=kw zl((?*N%2QUdxoLm6Q)q6#d%qr{#Jsop?-N=B*0jFWuHLkxxK=WI%8)456~$-r{9^g zT+DkT6G$CGcjQU?e)fS?_hbUr^exg=kDbNV9nj*P>Ey9jqLb0m*ImQTWiixq9kRUV zsy6gAN@0OSx;4&K0$E4dMe}YVA!HhsX`)mOZzAmtO7a4?%KLo`-j{VJy~nufQcJ^{U56eNnBRo#USb$1z5mX5P&8X@du$JIE`)tBhU?fRL(&oTqH;@npe-zH{p zw;a5O!RzpThCFLSb-G&=brZk4jPKUT*PmT$G;qYf_-+u|EtpBk#Lqfqr z{ExHwzk;+77~}wUaWzwP*Wl!$rolT^GBz=j#a{<5jY)<7m#tZp{6Ds4^ez}U$48KJ zKa8G@NWMrS9#}9uMr+(*1!BY@$NCR?@c>FG^g~qx*F8$mcRlHm{Ijqy(O-p@R^;T7 zICte=;9I>3s?wk_&AP?~a`=eEbef@<9Ls1Hr6_PUnT%^VLpc};kJkabso1x+Y12$9 ztGc_+?CNB^1NAbU8v28ohsY%FHIT>meE3`POg2>c^Y~{PYh;ume#yY6rSn=>HMrFJNMs>fi_D#Z;69LQEhP+V5%nK@o72;MbNS<(?)T@nU&yXtEIl6~e8 ztfg=rmr<8rJ>|$*&qyNBgd%Td)I*~ZAUT9=d=u*q;<3vhdZJ1~Y|2?N2pVF5K(f>o zvUY}vci#o4oKwUMtdqbv$3eUV?PWqu_EbJt!iYKY?{>2o~WTOEYdoA z6kdfyJ^QA529+&_oz?n2>s>cP_7Wo|!VrxtF&QYr$*c%em^UH}C(GX>8)y z=?NqIA0W*dDpg09r}MFj>3st8$WYZKgU)*(PMRqcY`rqBCsrS7`xo(GLW+jrj^Amd zt;(!ectqWvAdMp=UT}Ja1_o@xRW1@#HE{zvhWXPm6~RA*l+=Wc(0bd&y_P6W9u?mC znHZ@9Th)w20E@KWE{mb2cOZaZXp@HGcE-+Q)wroO#ma5!I8^V}-Jl6dW{zT&v|&fx zToZ-d%gMhQ%*%w5c1GslN7qw+HB>VRlaoVvE{x@oQB)~--}i=(HsK3Ffde2wr*_T% z1AH-O9AR+hGI=AmU0knINJoWw9&4Xo@J%bLL@Hg5O5IU8vGk;hC=xkto@V;5O>Mg1 zyW0(BoU zp)1-u8#uM6VUNKbe+zx(caT0c()lc6V)4VcSrzq}KOjyTF-Z>E>@%fj=k)Dcs^~lvN{tQ-rxg4sD#OSG;`cN|u6?<@fHOOqKg{nfD~6=FJPv z_kahC9(p0z+d`Fhf8@EF^eRlEKU^eUTc1*rDaCHNc<#1OMnF~ZqbNa==A26v_0;b|JXJcA8G^`eO7r332BwSgAqJppP%v3`fd*7QPenkXPLQ;&f3rKCrt> zrw(s}&FuW^Fe1(E9aa`fW}~u^6ON>|!bU{vw-*E?UJG+ZBX73TW3Aqx;E(0%?bY<0R%HDq74zgH~tS2rH3}?`eACTuUw4 zA4URBF?;%8CJPeYy)Jdb8h5Zro$=)T^AXDpu;DF9c++(K;_edw|JrB2^}~WLGf|n$ z5hG&BjbBYvkCV`cwtD;HRzf5^IL-y@FnVH=bKHthjkul#(nc8|1`7~-9e|F^_1tnd zVVNf_O38WrLw?(-UF1`hAQFj_jr0?8_d4t^--nkG%73pa-4(Jjx#^wai~Q(WGy(&W z(s9(4-=J?{z6AyYz^{M*lfcM!keAqgq4#r=%KqL$RV0^yuZ>Py%A&>5#60-LZGLz$ zkF3~&$yq?iFX9hxP$I(T1J~rWfv}%hgyHtj!{hc8Xq0egqD}KezRIif-?8Rf_Juzu zj*R;z=e1YM{x&(`;R5-TKL;xgaDi~RfnTgN1MHA}%}vib9S0E-r2UM?f1-qV8PCym z8IC8ZvL1hN%um5L^Fb)w3AP~%fUEP@znew2PD=owRL4)8bpKtG1Yps+bRDQR4SZ&s z+3?pNT)<*81xHNE4t1 zcwk$OG#)DEheA~uPp`IRz0UmuSU_OJq%ra5Hx6T>p*g4#bVyleZ@?#|%KY#k{R^?~ zd0~SIF!`=1NX#YLimHfJZOIL_~~eQI;40%`CS>AyEXrx_G}pUb;1j_+o5nYO8c^k^h* z5uyvg2XMXVEP6Bm8f?6 zK4$*(u?w{4Z^GU-TY2E|Yq!2M*gmMaZNsTFlCOn&|9x)zo&s4IukHX)Ehc5<-b z2$-usC@Rr_c6$He*$?agQ*vtCLfzRp@9vMvf2TM$Mn1R#ywTJ-x$eE-89TfX(Q&JV zm?eDn{C!cCf!u+u@_H!0z#Qdgt{pJmC3uI79py zx$vmcAprbD%^$E3UO0z$fGYjzAsYVQht!578O&a}(^u4hId*8{UWIfL?^^l0r27YK zEKUTe95NPsk7J4S(D|CsP1hM@cb|3W4>e6e?5MScHSk?V9Crdhbgc&b^W8tU{1xIW zBHV;fu&gZhBwx=p5jlO*px$?I@IGlLW-@2reS)Bx_p0ilU*IEJk}qUJ0)2|B_ZE+Y z26;$Z5CqhKXy||Pgd>V_g3bK2wKxoof`7-*zOVcqj@UsqgnKlx;TB<@iFyo#kmCx2 zBkT%e=hNCBa1~Uvc|DUoKbM)rIsazAD70Se+aUE|6n)x>(=x0H3} zZ)U7^>VPHDq@<^!d% ziz5DSqW?30QL|WnaKYLP*O{00|IcdwXHao7!bD5|tGOyNaWq_TMZ_n@N+!$^j(DTA zTa){N?38>9ER}Yvs{9Xt8#r*Gn>9eh3++hER{E)cQtV+5i1J51TJzybj@T`@e2NWD|A#RMC)DYMP^HM$UBC zZ#^-cDu$y9H-DT=fQw0Q%Bbb*>}Wm6Y{UO+Lxq=)?ila?cdZC)B{zt!+z{!+H89w4 z2LXFSi89|C*e2QKSNAAdLE)u};Tc%!PEo%S@p(4nMt;~|2wBm+qS>YWR>r2bL4tUt zZQ121`#{^g$yNU(U7q{8A&46EnSK}!1ONYSk$*QiLiT8{ZvFS3+-zc8W-5J>`hWZH z{qFm(M>3`vDMcx9SM+8w4fmI*^M9Sc`%Vd=wFnn%af`w~wVD0@`l^XRNd_qnU;hCp zzB=r%Bp!qRM@!^v;tD?r@$!Fu0NiO$ug7M-Mqki9>6K`JEe9??h@w!X2p6(fUtY*e z-&YS8t;Mj|EWY;gg`^MzQYCoKx4c-^gNCnU{f#aXCv9q;3?+%}T&-?}#?jX6uR|!= z&%C%xyK@)a_igW9#rZ=OKkw?IN5>EHk~HJi-dANPTz7m~MR$w8-las(MzGU^-Ahwn zRgPDTPlYk_Xt$0&6_){Bu@TOuuEYJ{2%8eTVd}y?CZ1h#fCC#^Z&{{r#c=SZQ`P`~9_9Igqog#}gV# z-1OJi^VAmurr9l@Z`k|fH>VhX4fF3r%0oXuRg^+O5UcOjRSsVY81fWD*-RdbV_AKE z1!?i@QU6Z9bfpiPC;6#?OV%$lmEAqar=K<;@L2@cmLYA&IkP(e%ZtfZYVk~caXqiivB0x%Sh+Yk6Cm7!YX8^KYyS5(73zBj#)Fg z0kDH##8!)-vx#M)NGmao8b3AP!ta$IL^nJAk}*bN_7bAr1)0s&ixA75Q`Hzdfyu=PB+2d=& zHHntbDW=b1;rgksCS2hjIH&GVFZ#Y@z1kC`9PDcQ9*ha}>v(fzg~#Hy{jKzbe=IYY z2|_LYVS%KJ5NyT)Sqky?{f;0-y^njFbEfF*{>!?F?@R7;%h7vn^c;E=;HZ2-cALy` zJs>N^N|FuP#v$C)HGb^LrPpj2-Q7>7H3bQ}nHd^RF+{!aD2rEJzAM6kWgEp$7?_z3Y#7oB4~$!<=6 zHMN2jAxVY_N9AZ~Dt(pX7DFLScajf8@oo^%LbodJ;u(u>wF=4zIJ5WiD@c+z4!g#_{>z_Vm7n36jRyN6smnKo_#A?q`+z7!)ahiL+U z4`fUkhVH3VV+u19;K@@n{+R=r#fNJ4x-XKZ0K^Cxlc$>+vYT6WWE0>VNCaZ_MMpt~ z)!?p>{SK7`XJ8XsZ2XXc#TWpCF|2+`R2Vr)nTIR;$b>=mtI6hakwwdL@)v1f4~eHj z;A*Nue~;$>A}&jTY64k!zq6IIto_r3phDWyM+Y7r(pel4WFeQdzcz4XjQ7_46(w}6 zmNRnFGJTv#BbJt)G69J#U-Cx|Py=2X9{M2K1Lbh7o(2|za8JN~B#mXQ+{6`tg9trh z1r0Y!^AamE4(yKkEDyYje{wkOkm_|M4WcSCnJ~SbcWivW*j2AL3O^}*^H%*9(WK~$ zxu`f{Y)JROsSY0^V-N5YYzxij+VhCCNlXhw3X&zZ%LWW07i(-&Cp0Ly+vCjj^v34$XI{Avp6rKCq z91IiX4cxiEt7_BlJ(b>-VnvxbS%t|gcqU;v_!2D1X5wpgu0AtftkqpVW64GdRq|=+ z(8Z^W_94J-#M30{3lC9&DzmT?bXF?5pyj9uEt|D7m_r0(RBiawAv&WEWJmpoV}(Ij z9k>?ZrNOn<57g@^G*M+M+lF$7BCJ0+g5aRZu+Ic zQv`(#E%R<(5En~>Zacb1RJ zTzd%+vMnX&vTSC-kTdSJ>Z-V}04RYWV4)I(2__T+xfueJ5jH#|w#gMzECx;l1pZA) zh7LKsn>C;i!y=@Ig;!0riJl+(O4h@Qyfyt2yHI8WGxoWUy~fp@CtUCNe?a}(&dk|F`d&Pccw4m zr!$^f(l0hfsStFj4i6;v4@}5+vXh*|iL?TdV2^Ce=@28xy>ey|KE7BMFrY-Fj5P}m z6b2Y~rJTS8O5Q;ZEMn-SL&D>Pl7H!T^>cmwxwB}gxhLS?gfKNqZR&-|qh{l-_ zYG+wCLOSs~@fv&oF7pjyL@g}6LsHdB40~^d{^+#gg~O~1uHziArUINZtnXArxtI*s z7)fMMJzT zP0r)0^ZGF0VCD{p3u#oCmL98BHIecC{0Au4zH}?sEdZ;b*!s6^*JMx1R&SNe4^nwD zXM_jPY5tZ?OOwWlu!h<5nI4T ziTP-a3tAPJL`oI!V8YaU(~)Z-GWg&^0+=91P-};+N#%wZvh{W$HzR&d+$2^L;EoHj zQ79bd1jO$#T~<*s&_eZD2O-^l<5U!m<`{g4Y8K{jB#$Xopry&x&nyom8$eSJ$kd~P z-QTD1kf>dR8&S-&!U&i0TIU0Egh^@2(6JB|h2-bW}l zP7YuWwX>9uc}1CUaB=A?GlbwnP*J7N#i)3LpN4J4Fp!D}n(%&TK_~ILCb~<9Yhakb zj9Mt2%lI?`!V4ax>RsuwcNURk{?JByc^=MA$KewV7DXRn;vDZYF(?n<0+0dsy^SRz zT=+>T@@olMh0|b!sPZPPLlbsLI~-pcHjG*9ieS)VJ8PiIK;?h=E!NWI9)(?rFZ2*jf>Ae~M zV*xM{%*?>*p#0MeD6LaW_t!&{coLx)d$AFyfX#v;&;@71?yh_|Ng+*!k(QlTwaW%j zIwW9}i)}^LS9IkxOidvFRstAOFCEN5X?sm%a*$oH} znhx{OUENHy8qh0s;m^P@VVfhdd+EV@Y~Q08AjH2l(#HSN+y_mSLLa4@w~9sfwX-Jw zv7Q-jCCx?EE*kN$>C0AvraPHQ&_Y?~)$5+(8iPoCh&AJSTVug{dlzh)CDN#|6$||; zOuCl+JfMKEE!~#3^Gsb{(LDOR615HG@$M)XqO%)~eJY&%0YEs?_p9{|BZiHv(RZO& z3=ue|%8_%490y_LRtRbZ_(F*zK>)=HDPs;+IO6Z52hDgjhpkbvQ<3WEt�+ybFSz z<{yQYaWl#KQ3-tpDxoXwBfy8~G;jmuWTastDEQxhLL*`XSTmP3)XsJ28>}v++mqpj z!Iv^)!mlKl2p6J#;oo3XzPeb@^6Of;YE{2Qzl2sstHS;e#zRxA6rf~|RnZPwnLjNA zDlL#Mu&3Ca$z3v;Z@(_(&SkOJMALA`Nbe(LaWEXLLF+;uXiCaVLUoM^D%%F%Jpcf% z!18D4WHU)7mMhq412D4FE7c|ip)3hft?-UKGoiK719JgEELjp@Pf6w-RniSvV`h@@ zw{@F)>_JvsG@RL&)|?YXJY+KH{MDO>1?qdmkq|YL*(EMV<&-d*pJ4HjV)ysqU^%11 z712ci2Y|elz=;Sz7EyAgGu)ZNTV!A(Nc=@&|NLwE3?ePrw=}C$y2dCb@`1n!ZwUA< z80}uLS^BwsNBZ5$+G$;Y4w#t_wm<~N4m z`~jH%A~NB0aRVbomk_eCwH|=8a8hMC{GDjcr8|Y=W)tm1QZTwn$XI=FHKz_b|B7C~ zXtJ3EZ(&GJ(j;Y!V$OVF8My;&A|&%O{XX>3qe+LG^z*ndG66?RS(#*skf8_4sTF- zp&J$Oz^;ep1Y@sc77)F|!K`rNJes@6hR4=E8w{#%ACAV*tM(dIexOEuY=Fck_fx;V ztiV4utsSCiBPRAMrwoKa28a>)(AdOLV75>rB^wgy!P0|krA_#-@=^bv_P#r+sjOT4 zrT_r~gx(=^q)4dJO&}DJDqR8T(!_v(AgBqwcacu$Rir4=)XC~^h}eD^ zXXcx6X1@2m-}~>a_0Gz=XSciWIXnCA+?#XuInKlw#*;AXbR)c>;m5fAt$-5> zmn9Zh*RgsTr3i>)BIb^!!C9@3f^#)0(z$Wnk9Y2V2mBR1F1*paN(VZ?Z?k!)be4I5 z3)zds&7*F0LF{6M!!L9vZR;;EY+h$5j(o>(hs&m|ShAdbn?v>zKLc^d=M0Ec7!Sj6U_8XtXs6fOj-&4beBWCe@S*rLGN^*-^(Z( z7p0w~P@WDAJT+<|GNY|D?V`u?7@5WoPDaWt_}+xG;nUYviq&Nu2A@)-SB_oe zqgIcUJR~)RZkY@)08n0XBIx)c)7LZj&?v^IvQ zn4RS<-l0yUr6bX)y{4SmaElm`Zt@^MSNuwFNkgiP(q&w+$Mq(G2$E;@`7=2b{kJmJJ+iMY&%tvP2eK6tzTwbZ%x+Aw1rHe8vbZlYKJtQ?9+bIgWDGGI1zTKb5n$eI@; zT04gTE;w~7$ZXjpg_xEmq>h-4QT1tn35ZO2dh5`K4=_Hj{6+&Ef6N|NM;i}6A<5`q z3hzwZGFHx&!kkML)q1J3%Zy`Y_Y`8R9Zqx~K|t4a2&uTrBOlXmd1^@~p*GeO%=kE~ zIjAq!M5}&Dh>ez4kUvNuwA~5lE}fjhgE{jJ{9Y0YFpdI2rdh0FOAxOY={_q{H(BrX zQd7fpAzuK)RGU>^8jA@-N5o1JGc7e_qL0!@W~)%Y96jh9+v}vBMr~d zB3oU4OtaRb4wx;(B{;%hEr{9^!xs5o-;1n_U@e&tl?;*2tVb4Dd?TyhH6nXBa~f$jal-C#nA9xGrNKxS0YbU|Xy6$C|svjmq@p3>%rQuVEuJQUmO9^ey5W;qx6!U?sZn?@a+%5ZUOPGZ8D zMm;O`RDJbBQyO|!0Z`X_OO^XC(&&s3Fk@!iTYWEi4tsGDb$4;l+dAcsmag}TU9;4~ zEFwV6Y}lnEmAes)##)`NRq3#7EiiAER|ArVDW2r&!d_C@;soW$i0zsoE$u)<#vfl# zqzUfKmvZ*3p7oqXGN~0TUavTw6TeE8-5Dmvwg3|*u|VS3%kI?OaFS&w z;DoBg$d}a{=WC0idL^tC#1YZL?{tLMQf&nBSnHDE!jGa4@hf~n3me;?w}HJX+C@Hp zon_9@sy*6na6jn%lUWajs4m%jx~!NdBc-|?~KDuqd1Y?4~ zv$hx%aDgDGD<1b|H*$kl@C1!k4~^^wWH;B|p{{LDFhZMcLZ!XeF&jvT@>L8}b;ef# zi(Uf7g>{eM*=ObKy}ps|`3@|)wH&%JFrU1&B))SyoOJil`s-Qn&B^1kU&1&uz13S} z(-y5R^USpt8F4l^tG#4+!4UT$o9@NTq@rtRlF5-&yKR-FQVQ#!2MRgrI}bO~UmHG7 z(djSkygHR7cAY!GGHRy~?(Ls#DrxUf_VD5d!<%p;z~)Mzdh(?k7pzB(csUFOA39jtd${G4NgIf2mOnolA(l$7CSOu_mT}l=f!u)cu>8mIz+)R?~QDQU~5w`N!( z+(?$+f!OnLf@^a+8{wE(htqwZOlP@~QLwfk=EDqM3+>iI8oH6DmiKf~*fpFWnf3DYN|0YAHsZ~+xjMlKRwSJTP zRgVeg=hlOea`5R?>o?zCN;^6s@FMALiXwJT1FuufmrzSlDhw&c*V#fi`7)pG2)nUdmjbgRi9GD>{FPtGAxQapR+*O%#v#*WPC5V&Hit9&6UPhdG0N^<0k$yU(RRHy};=Et0F zFDt;qh%HiQBjbcX(vy4#*AJnQQS9~dm?-rj?gIUH4Wlc@uK~m<5{wchG^0y>p8*%& zlF;?=)GZxt&snYw&SKR*LeyG##4#-=!31{4(3>{*>H+70_<-jmE-U4doyBj`TmI8p zGcH1{PmAO=g5RE`w=9DW=|QZYX8Vemg_Ma?H!xahth#WtthujIVy;uA`oq)9z>lPY zOecF7xB;m(YjazXx6qq@Ry%9p2j|SNElju4*IOgj=nrqZXi<>7%1Z(K|WY+95Q?)naUXYuKKX-#;MKP@^UVX zle^cttHKD9q7eA4y%V3L1C?G46wsUttuBs2JD_i)yhv12{^GmUhR_qfCo|r*4DBPE z(p4MICUY|MdEE^VK=Q2NfA7I?ew_jlKPArU8-YNNm3+GKZ2K% zH;4xYm}187`9(VmGfK-^rk_Ewv}5Nb$4BX|4mt-Ai=Ti(()cV=XW*DrZ!OsP^Vpd$ zRS2H4h0{?ol$__RhOO#^z8c_GHfFkeCSj`Kl3+UHCcT|Rez^O!iHfWzWyg^%YM!nrJkTPQ!# zkZ6gIxN{dQd{c}$Vc&t?A?K`zLZnByQ(5W=MT-h#d2D8{+{xI=;A@x#XKlwSDGRIY z=d^J<^iuLdSj!rX^>K zic#ej!n;O8vjgM$-NU~EYB0Z)CBRu1aGvY0$XqzQS<(PG9gyUSpvl|glob?Cx?En&c}egBL!StnsrQFk6B=B)a%TM9Z}ytdD4cqg+f)3> z%@%M!Np^jvE&-YB+R*VG=%XQVoj@w*4i-T%{<=@c{V6V|ZI#qUO3;fY`r@gYQi?^n z!zG#*PX?iV5Y*eCk?~W0bz%~C5Y0y40WV)2FvzO7ZQ{`gqEh0HiXvdMb8mMJ$5*FT z?g)OWKJd-B_^FoT77;t%29i7Bm7Xt4sy%TDoF0O5D|CtFqAI|ukQB{4tIm%PIDz`T z>wTCNNT*}+PZqLNgGuND@+P~w$ACoXXwX18QV`(cM~p>Pf&FLku(geD-5^cD6RthM zv&|;G(S)2?4%Q2%JWykyh$@WWHm&{<^pLqBo0}wAppipGB11?^K`Zm@_4oR#%x8YSmX$GfRmn z&M@<+YcuILRYimt)!vu#qcXS%rI{{@BYsCdyPesW&Tx`{&W+Aer4!FQ9szEsDDZQ4bUZ95ynjT0ca=XgY5t7-x8UDgnDj63p8@<|$z&h>TcQ3N?4PuN;7B}x1Oq~V_%FXBe<>8=?;*e{R!V>tPW+kmvjtEXKzIepgQJy# zbp6wJ_w%`bU_ZMLIh2BaIs2yu{w7QzjOu|Qv|uO#97zt6T7SOE&su)O^dBnVKjJ&| z&wenag9LP-TEhr%xVRQp0r^K$fS>L=kT}yI(i{xn@IXV4LJ0}wWWeH#{!JRu1;dd5 zfp`EF0U)$_5aB;J;18WpZ5RqqA&!9o1Q-jE{6h^#fVGgwUj^vZwO~jv1rnWr6fgkL zRKnxskY!N&TM%{`(d<{$K2i{L9YCi2t~A@~;v{$vY=Qz)&c~FRAc9E)N12 z1%63|_ah+Einoy{)5b1xD*W<$aw`14GT{;AAsYDkPgC2Orfo9R61~pLeN8Hz6iN11 zsDg>`BU;?t8-T8;mv3DY75C^8LNU_mhRfBz~c5P33qi@h-*X!dvU)bX<@jr z(AqvT&A;};F*LfK{Y!}Gk}e7#NVB+fpFbX@0c|e8wEIt*nD7?VYZDw_v~W9p60~Tn z?wu53mHOtC>a08`#lfAqfIKNRaa7m*79oX@sl%BYq}RaA_3&(NY3^$fZ*=ON2Io@P z?nsxWXO4#>HI>1=@V1A3(8T@o+t>*KlZbc7Sj@8L-A|is7VWcY<*URB;znyHZC!tj z`fe1McWGX>9?22nUP1?Ycsk@Ni9JN>obOVX%1lv1ipO_=`Q3q}>uJ_dsD{2tgUfEX z8PlOfutv7wMyMHx-iE!@?PU0rvtI$sh;M0->8mHu+;a%_18p49FpjmoT9Ft%l&lj+ zXCG_`hFyo*=_|HcDHyMNmhH#OUy7$Fc|F)Fc1?%+M!{hifCa1T$BETKpRTy5bE+S2 z-G$JEw%s{;Ifo{(FmfbNR<+6$wAn0k3L52QeyCETNuK}z1TLpdeXe83GGHE3x!s`~ z_Yrf4r%KpzJ8oS!)h-qC65|$w?PB^Q4SyT%nk;IlW7c|hh2Xd?pdXa^I#!FfK9f3O zd4VZ+^tklos|F*MGtm)Ty_y^aA58*os`f|ThG;DTt|K_-pUu zgjeR<2)fPu^xQsfN@Zl4+I5vJ0oId=y$6}9=Wt6D)z(cyaYxF#tHYJ4D)n|n=)!Gg z(`z5=Q|og-DvL)@f~FI7J?`_?sL;b?FyN8VIl4qf-Z~gm4qmz=k2!VR)_F@PU-$wbe(83tYU8TCW80d>n0wpDE0ioXe4>r&SvDUYDy4(kTkC$;ev% zR>EL%JddmQ{1ui4Ia0;&H8r!jbW|YRgWNfT2Qh~VQW#asi#;Y^N{JLTj>WEkh~}_nD%ICZ2jEl2K*ZxebpvrpIpAE`u5dy zf=al18XxXOv^DNYf8y75#+lT!35TkDoO_91Cug{uwkc`1aXqi!xZ1rxv+Akv+%6P3 zyEAi}$L^&jDauY5KCzUKp`(jcjj{T|@0?v*UR4?Jesm*xV57`i(UB5xTE6}YRZqOt zOUozbXju@wONlQ!AHXwQ+SppjFmteH>z-`@NS@OenkI6jY0pstA@zq?|C4+ZZOl3M zs*RQTt6r^GXrzh|-`00s0Bx3q;;R_L-R({HXMJV*m#Pb2Z{usgB)BVw4PqHY~e`TB^tWrG`4^TO~X{gdn{d`iB(%@19{P{GAfxF3wz&*Hpw+nN)v&k!l zNu+;=h>^O-$>+Yj$5$0_eUg+7!yRxXF5d!BKFb;bsg zx3~VHkZ0}-l`)X^axB_08do|lenq3J`kJ`7Sq%Ztc1RT#>?2flXW=vp*P|m3HCB;L z+AsP;yISuZ?zU5IDTrIcsjK>SnCjXeEUSAllNW#Yh}Mz5GxYEc(PU09;90BRes!CD zeTw;|S^5jba>-|1EMDoOyzQY%CnM_hjZ`M6buZyHA92Z&#xH2Ez1oAODT)ycIwGDv z+MC9UnG`!s&rBOc#zPDOCJcDkwZxUj~1&BHG|+sJ64+@YO^w?MT^_Bt-QDlqhB0v|~n zaN2hX>4|KBC_z1yS`%a?e(){=R>OGiMme7eGjItfa_7Nw6!f+pEuuwFI`-l zE#pEue`|-A!h#BtrFSZ?oJ3zDz28%X6+O0kP6+`k7e0#}Xk=efUP9FY#1O5z$AgbU z;{)846X?^+kP+$e*2k4Wc0qPmGQYIC6|M&Kou9++*y+2p+*MX$zF0|m=>Hk=h9H&F}a%=r5#~2F<@Yj|mTL_+-1cI){D@ z^4ECwDDSCaS;W(;i7>It1s0_@!1C8&Sv`*J|GUZ5$q%Pab}n);dRMvP`}le_uH zcMTpB@tpFRkE}~WR`lK14}*?=dfUFwS3SBrocwNMo6`3>^K-|n?*JXclO4h;J><=; zpdF@Ib$dMT+go!H3b6D|^f#)h(yfzB!O1My@@}pQ<^_>i$F;$*%r7yPbK>F+Z;F#8 zSTy2Ux#om!Fa-1SEA1l@E26=FB}Wm##XMW)^}laV(j65rp244RkK|@ zt)ZlYuP*m9H&#!5v0;Ty68i~YNEFIHlyD=rPw*;ZG`e>jS#ebzzoTv5yYcptH)Viy z;@+ivZa3%NF8JNK1OsLD7lnOro3kZ7l(G8xG3nIXdJEUJf@5q-;`GClq=Z24Wa(A1 zCqDTi_v_DZ${Z`g3L9$=K@Wts42spY>&&oa7~L#J{KwhiMa9~#clB5U;v28TPks3w z<%gNcOhCNyD}(M&H3?279r-{j-~=m5&uT#qPw=^C;t% zdtw-?SgaN^*eJAPqNQjm8RXIp6dCTHPZk_DhFvHMCspCL`v?}xYkg1iTMrz`@>ab* zvPH}sBjsb=WCbg4jd{XwZbojML^AIel`Aeo8=9Q=bKdiHM|JEgJL|9Rct1Y!j2dC_ zVh%&^&v^%9dFNRh=wa+x`mpPuT+S-WfrXe&Grh^Sv|J(kw&XQECa%8QYiGj38ld%e z9hRI|veFp5C}=9+z}+Iw62kpXYqlrwO`OySLH-J z^7vIZGMwGqOqOc=HUdL_6DNNtd7h)ZJR%0l_}u!C(&zf+XdKV8_}pB+_mb6Djvk&) zrz|Ms&8=|!(bgB-^-*M@2OAXNVAvjOqIv?VzNC>k`3Tu9c^4x%>4$Dl?z@R666O1Q8gI|CPULIC5jFKqgu^3XZYM%<0Mcw$%bRYK|V>maZayU!+5xX4!C zSE~8++e3Puie`m`ICn97)7{L^)1QhQo-8ZyL$7jvZ8_thrF8Zsv8rBX=u(3@dpW41 zP%Zv)sTb{gJz=6*W6Ni6Dz=wR2nGD)lLZC?+8K((giSl%^!hI+YM-`#cGW5-kJ-`( z<6l01fZT1lf5jTEq_N%6+x+%M=CQ@Bn?A^FFbPjp-Z5iNxFK;)UsBWl{g~s`$6KVj zzy&=zJKfs3cZTZcYBFiGZ=vUO#R?htstPn`QI=C>6z(9AXANT&DKpum!&wx+aR(Ej zx+i-4<$g5-$R(mNoT1*)@388!NB3^xFr0Jwbsf!j0JlI;AqYtteKu8d9b-s6r&V;^ zv&{M76k!`ddT_aRrlH7f%&8R(>v$`hBs=qB39a@zbK&UAN=i<8+L`E!Q=e)*2GXz|M#^8U z*~NL_=DTOQGPfCm{2aJ#Ga-BcIZG5W4SI&y*A>)oM&H94uctbf(EuP2GNn2C;|>ph z;s@7~=TwqA>#Nh~d(f=`slB+6aFfh00(4*Gtahe0$3u-Z0Sl}|$d~usM$FTM!#uIL zJ@Wsd=&Q45YEHC%%~cxmQ{x23E{@=31m@&3wz^SFSbO8pJc+qS9C9)O7p(l0wlv3u zaP$(qsZcawOX?!kj`jyxGY)Z`l264l>hFF9yT!)OSE+UNhU7%=&ru`wSba4{|aMnDEOMj>T=Xn98Kknzc{R zwZk(UDQRfUbXj>)b^2MRwd|q9{=QdA0&%YpdpSK{X<7--gJXGcrR6;IVobtec{%2T z;}mg2UJS#CCKVu7L7!Vzbex?Ut!wH6`#LAbEHWwW#3W+=m_rOrEW(X^QI93;V|4v61&W!pspDf0-Ol;Q|X8Ib}K%h#+zpskjtq(M{&DJ4?ZBtzL z&`V`;pe$UroJn2Vu9D+ifjS#nM4rYONo|;mi}eRayn`83H9#Nnrsei#`r9T6NcHf< z8kW!qLgs4io->@tr&+X2kS*Ud;!?%8L%&jr3k?_;8vsPTo&8P*BllwrpZseOX_fKJ z5`87<)8%=5B2r&zKdw=(U0HvD?Mj%>G4-WjRu(JNf%Wi)O)#`{J#-|3>7^uyll;Cm z0(P4)%_L6b3hb30^GkVO-Yu5oaY1w)bQtcX9?w;lrR||MK`&WUX^*VRIWu-$Jn{=k zC}&gq<9gvm8(Z@JkXO;O(5JC|C4wz;AL6Dygk9hgg{Q;vShI<&n-Hy-jRXnID+Sz` z!8WS~3LbJ}d>lU#$(@FDL_?;>wTHVAXb9m7fG-SBrJ8d^sVGyqhVdVNh`nI+yq2@{%(`oUEhEWq*XDav>x$wp*&X0%L%Wjob1#f^W5P%U3hQy z991KCery}JgW(cF#OL9oR_pYtYlys1*3!QBPm1t*Zy&sB(!bbjBLe1FsR0gcp=OiF zEk?{8b;BHlk5&hhaDbVF^)g$`Jyo0(7ayGffU4HLAa}Q7dR;7w9^geh z%I6I*2-?I`97rNR#B|6~T_3Z%?T%6XfO>Hh?QuAIfrH0ZOsK-v`#u=DqJ1#3Ys|^e zYeD=1M5>p7ir5jLUNNshB0j-CmCkajmMf$<%!6KNuGBMGW)1URp*O zvou#nx3nzdHf8C^ie$sFu9IZ+sAVUyW+le3wMHl*HX}bo!ba%oc)~zekC1j@mfrYC zycc|BPzT=U{rL7u=yxFa%zNH5=#B@gTIVw(4Uw+LAd9lv?g-U%zQ_~! zh|X|w))7R0Q`OfGkc0JWKD9_SCZs375xzH?!->4X#Y5#5%X(4s(fug+Ty8e8ENYxI z6T;GA>gsqzZ%&PO$WQpLB|Odq+bqOFK;4t7LBkLk#NOAE&37K? zUX1g`MKg^T;)?hKTNRZxg+T)3y>yf+0zDphWc)5Yl3#aiwtZaGo%^We*_N5qC%kdR z4AO(@yfeM^C%*%#Lwe;YvN4KTXSaKk<=Jrvm#`y12lyjJnm)n&k*yDb{B9i1Ldhx} z&S7D+tmJ})JVBTlXLqLQ(>rQo&iFrKjU95n+M0y%+>nlMW7s{vD@zmpvQ8Q-S}ZM+ z$x{E3K|#~54)7q@#Z8B$h;O-a6u{k;;eBa1OFE+w+FeQNWi5|XJ%{H40u&4ss;J&n z&)4DtV_F<=vGj92Z$}^^TfX3Q&TJ<2(8?9dF*&|&&Nj`K`lsvt=qpi7ZD|b*1jUZ+ z=M1>KK~QUf&QLIzZ#7=sz^PkRRCj>s$QP>KmetOb^~xX{jW^?9w?GMm{|WMrK^@1u z^c8#rAmh}G6kS|RiIbc_FmFkLBc?r3rR-HWP`zN6$m_m=t8{wc;cBV$n2nP??0lfN zW>xBkl&eG?fdsv{kA>}m;9~w*i@R_FgxDSQ{7suKJ8(i3(sD;UAspzbo~NYy@q={86@=aUC6t^V-Mw2)-!%r>Xwdke^whU za@;*Nm1u)V0(Y|jU{dil&;V+c;ogYgJJO^R^b}J2LV5~s&9iZP>XOhsIF>kr;1P;7 z3xk_-P{SgLusB2+kR?jPBu^1{frqaBoL6yX+L3$?=A7nOH$!MV;2&|C`hGk{nYfQa zM(kt2=Z-T3ARitf^d^7QH+g!R!f7&kL$bc1QAu|lB8I37vrep(DEkmAVaS;Fj=$uwQ^c&-mu;`aehHcTkN7#?hbFb%>g z+lzXP!v5kqEVpBkHyZrV7bNRu+a)bq#p|Jsm|2kazBFGCYK~Sp0+o7WJUs&J*jO+v z9M)g%;-s2o6d{{jQ<770+3U64V%z?FtL8PjuuwcHngjT+!au+;}4(ucjK?Lo-hoTVz@ zaZ}EjSWo-tEye6E_NFw2i@E|^_?EL7v9BJ1r$TcsgCNPrbYRAgEoTOK6G2T5vF5U@ z+Hb-)^37>GT9%gru zjKE_rP{)&!l+nxadN)l$dw86%MUen1b2bq_pGb2m_MRi{hfeadpLo8k>Di)d3^1@L zzTONa1~!Zs(ganI<$3Is_iX`i7x4;X)fxNNZ}Q_0twY-#%TTIB$A>WV1KQ8- z^$e}I*o!94b~y4vJ~HF);o(=OJ18V1OlfeYbn}gsw+VhIN2nhTX{zhIdwN6SO0D^e zE1$suMzfI2h@+I=nkX|WsGsCxbBOZ@tCJKt2i-049+f!0Iz2yqoNi=1hfC*uqm|T?h)?+FSEUb5Qdx}fo%<2574HJgbWQfy$?8g!uu6- znzO*elZYJbwa)vv=B_U5cjUGDE?n0s;t-DrS57|3y0Q~sMj859=#eCJ`CNK)fE{Bp zr|`_oi^1DRsy}jNx^qD8iux282)R#D1=Z1aTt90g>;JWxt)EE#Ug%1ki0&|Q|O z@QpO-NMKH*ujF1168TiF_O8>vu~mx zGmK8SigS#u-l-FC4MW}0Rr;Xw`56~YV_zg&&pWygqo2EKLaa3QcP#G;U4mV}qy%wgthF$ldKFf3>*>TKM2_ciu-( z!LtQ)=b9OYFwcfggI@D)`k$$QnTtZ(ebtJWwpd@8$<{HZIPT~TH%=u)Kt6hT6lWV! zsJgdg`mY|vLRb_)l6N9<33V5&QriN27{k-tPJ#%X@Dy_Di65bj`hF8SiO}wQrMvm; zNv8s(E|yQ|bIyiA)LpEkb1Y4br4HkuT*Q#oRfcdvBgO4yul5YC<%?YYyf^sa$5JK@ z(MZGWW1q{2&knR9Y3PV}pVQYrh}d$}>0+czlQJiV=r5(4yB~}Qh%hs2s0h8ByTNS7@Mw=g;yQgig`c%W2<|0#N5oApZD zRPhb#@^fuS5l((>ra=!K9xMo%6_#9$B=V}{s>IGhig{D|)N;kg8y4+1gL6m7C$79 zIP%TNs{$%{vP($WrfluV4LReqMZG|+7c!K~52?A2rt32K8w<}-xI204a+JCd!?9VmSr@yHym$9AQpn|1jBU+` zo`J=pkkcf%D~=?IJiehn{hE3uW+wGkx-mVj!%{Gu>s}g?v!;8gZ~z;D)-j1oRf2_K z_+VXu*N@){;EfWQcyMPQ|D7mkr6w3gYBUp64Rh(u5k1;?nuE4=WvG z3WoY|$=$b_YWT{tIR!W=nMi^wn~wtoteG|rH<4-_ns|QP@v-QUJ6cYnp#1Grc%zi` zhZ!wZ9@U=KMp1D*Uekw>lU)c@57z&Lb`^LouU&(gp%R3l9L^e#;c_i~%;avnQuMxV zuM}39z*u)n=3fE?oHT9D-k2vWapxWM+@%F!jXoFJ7Ni2BT=$E?+km3I*W!L+#M5To zO{MLI;0`Gqa!;=>9Y3#lwW9{Bc##?JHGp7Dt17+a={$sf05@3WmXD3IuaKp4?#4t! zxg*&W^_SLt-`(zwS)v2wHymrPYZ6+IOK?e-XIQhH`2m^PgYN)_z3Y7`h1KoXEHiZGO$y?B57J$@A1YFoNfb9RuAp(W z#gD*}ZmLiT#-X}{Kx*yGrEQYC}1KIYxbV!Pf`J!f#6)qT}9GGQze6Vn;rUI0fDjj2RD8zvHYk(vQr}&wUX4 zsx%KTIKkaaYb+M`a=k`{Brzq(K;tenikY@VrGCq$tN*%mrg-(?OQNS*^`@?nbBl#1QubW*FDh>I4cT0J7pgO8ou`6hj~K0uTEfWBt6D723n-?-T?d9YS(HSxM6?EFcPD?nexpDP;O{M53#3R8E{(?UduZ{ zIG_+VBZ^0t65i89Bj?na+xfc{?$gNx0vsf1kI^>)V{HH|X?YJOL(-~=xFj@=?hs_? zbdF$pA&#XRyDRY>U4mD&bT>6kvSdD4rTX6c=cL9ZKP<>j@6oKvbBYrYS3PVh>KfBNYd#kS1cux^ z(9j!Fi;a`_M@QDb&)Cw_*koV z#k-QKZqtwXZ*%#1n7jfnMBz_e^x}&9$RVmS7=6`)et>WgKMJC4623^DKcz5$q&&!p zA-~fH@`*e<{w2R*)EoJ@qY#&BHN-3osxfz(`$iqj;uaFjuYAGvtON{jej?PK-FL7S ztLY;Mq;je92y#oOUVJ_MC$e>n%%1GS=c~Bh)_25sklx4bY-DN5#ToEGsuSkVWI89cr@kJ z^WgBAi#KyV=)997zPlIwNtQ8h`xjk$P4Zpo`wg^;@B(Z z=$)#0HgCmHp&|?;NeYgfRU9cJx4~S%DUL4Qwqm&=29XBl*TM28Zd^o)e7dv&7=_lT z&m4f(JLNx%g`w^Ea@i!C!Rlu#>C#e5yjTJ}7|dLx^fl1r{etgg?0|)p6cJPL*H55J zR=a3G&coL?RFF(eBMmeHI+mzMw=h040?J1*5PYuat5sR$b7T4c@Q9MH91E%%GH?7U z&gbcVFSW|l@B%JU`^lh8wn(2muh)Lb+BoxG3O&j0jz^DcB;vx(Qh`c&p8UFHCQJTE zdI()-+VX+oC%kZGXo}#VNY!_M#cr|UMI5?q>JEt~ax9W6BE`92oo)sxmNvTNJgUdt z(NF=O@!+V4itmUF>8Nx5vd(`yyyIm02%SAfz0(KxHmqAv4wi zWR2s~Aq1m|%ri6w8X0}(<9JCDEe5Err(BW4gP+T) Date: Mon, 23 Dec 2024 23:04:40 -0800 Subject: [PATCH 23/88] searx --- App_Function_Libraries/Utils/Utils.py | 4 +- .../Web_Scraping/WebSearch_APIs.py | 54 ++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 8a7a74dc3..a5929cace 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -384,7 +384,7 @@ def load_and_log_configs(): # Kagi Search Settings kagi_search_api_key = config.get('Search-Engines', 'search_engine_api_key_kagi', fallback='') # Searx Search Settings - searx_search_api_key = config.get('Search-Engines', 'search_engine_api_key_searx', fallback='') + search_engine_searx_api = config.get('Search-Engines', 'search_engine_searx_api', fallback='') # Yandex Search Settings yandex_search_api_key = config.get('Search-Engines', 'search_engine_api_key_yandex', fallback='') yandex_search_engine_id = config.get('Search-Engines', 'search_engine_id_yandex', fallback='') @@ -525,7 +525,7 @@ def load_and_log_configs(): 'google_safe_search': google_safe_search, 'google_enable_site_search' : google_enable_site_search, 'kagi_search_api_key': kagi_search_api_key, - 'searx_search_api_key': searx_search_api_key, + 'searx_search_api_url': search_engine_searx_api, 'yandex_search_api_key': yandex_search_api_key, 'yandex_search_engine_id': yandex_search_engine_id } diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index b51e5be24..009b2a0a3 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -3,9 +3,11 @@ # # Imports import json +import os from random import choice from time import sleep from typing import Optional, Dict, List +from urllib.parse import urlparse, urlencode import requests @@ -439,11 +441,57 @@ def test_search_kagi(): pass -def search_web_searx(): - pass +# https://searx.space +# https://searx.github.io/searx/dev/search_api.html +def search_web_searx(search_query): + # Check if API URL is configured + searx_url = loaded_config_data['search-engines']['searx_search_api_url'] + if not searx_url: + return "Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet." + + # Validate and construct URL + try: + parsed_url = urlparse(searx_url) + params = { + 'q': search_query, + 'format': 'json' + } + search_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}?{urlencode(params)}" + except Exception as e: + return f"Search is disabled and no content was found. Invalid URL configuration: {str(e)}" + + # Perform the search request + try: + headers = { + 'Content-Type': 'application/json', + 'User-Agent': 'anything-llm' + } + + response = requests.get(search_url, headers=headers) + response.raise_for_status() + search_data = response.json() + + # Process results + data = [] + for result in search_data.get('results', []): + data.append({ + 'title': result.get('title'), + 'link': result.get('url'), + 'snippet': result.get('content'), + 'publishedDate': result.get('publishedDate') + }) + + if not data: + return "No information was found online for the search query." + + return json.dumps(data) + + except requests.exceptions.RequestException as e: + return f"There was an error searching for content. {str(e)}" def test_search_searx(): + search_web_searx("How can I bake a cherry cake?") pass @@ -455,7 +503,9 @@ def test_search_serper(): pass +# https://docs.tavily.com/docs/rest-api/api-reference def search_web_tavily(): + tavily_url = "https://api.tavily.com/search" pass From 274ac288b89edfb61f302bae6c4dc848f40fd650 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 23 Dec 2024 23:31:42 -0800 Subject: [PATCH 24/88] why does the test fail? --- .github/workflows/python-app.yml | 5 +++++ App_Function_Libraries/Web_Scraping/WebSearch_APIs.py | 1 + Docs/Design/VLMs.md | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 2ec85b147..7e4f1ad98 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -59,3 +59,8 @@ jobs: pwd cd ./Tests/Utils pytest test_utils.py + + - name: Test tldw runs + run: | + pwd + python summarize.py -h diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 009b2a0a3..49cb7a9b8 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -495,6 +495,7 @@ def test_search_searx(): pass +# https://github.com/YassKhazzan/openperplex_backend_os/blob/main/sources_searcher.py def search_web_serper(): pass diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index b4f528798..da9e9df02 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -16,7 +16,7 @@ https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct https://huggingface.co/tencent/HunyuanVideo https://huggingface.co/OpenGVLab/InternVL2_5-38B https://ivy-lvlm.github.io/Video-MA2MBA/ - +https://internvl.github.io/blog/2024-12-20-InternVL-2.5-MPO/ https://huggingface.co/Qwen/Qwen2-VL-72B https://huggingface.co/Qwen/Qwen2-VL-7B https://huggingface.co/Qwen/Qwen2-VL-2B From 30b9a4c4e73e5bac3b01c920478d0d358419a156 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 25 Dec 2024 10:06:02 -0800 Subject: [PATCH 25/88] search --- App_Function_Libraries/Utils/Utils.py | 3 + .../Web_Scraping/WebSearch_APIs.py | 502 +++++++++--------- Docs/Design/Education.md | 2 +- Docs/Design/Researcher.md | 16 +- Docs/Design/TTS_STT.md | 189 ++++++- Docs/Design/UX.md | 5 + Docs/Design/VLMs.md | 3 + Docs/Design/WebSearch.md | 16 + Docs/Issues/Evaluation_Plans.md | 2 +- Docs/RAG_Notes.md | 10 +- 10 files changed, 486 insertions(+), 262 deletions(-) diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index a5929cace..9a9f6f3af 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -360,12 +360,14 @@ def load_and_log_configs(): # Bing Search Settings bing_search_api_key = config.get('Search-Engines', 'search_engine_api_key_bing', fallback='') bing_country_code = config.get('Search-Engines', 'search_engine_country_code_bing', fallback='us') + bing_search_api_url = config.get('Search-Engines', 'search_engine_api_url_bing', fallback='') # Brave Search Settings brave_search_api_key = config.get('Search-Engines', 'search_engine_api_key_brave', fallback='') brave_country_code = config.get('Search-Engines', 'search_engine_country_code_brave', fallback='us') # DuckDuckGo Search Settings duckduckgo_search_api_key = config.get('Search-Engines', 'search_engine_api_key_duckduckgo', fallback='') # Google Search Settings + google_search_api_url = config.get('Search-Engines', 'search_engine_api_url_google', fallback='') google_search_api_key = config.get('Search-Engines', 'search_engine_api_key_google', fallback='') google_search_engine_id = config.get('Search-Engines', 'search_engine_id_google', fallback='') google_simp_trad_chinese = config.get('Search-Engines', 'enable_traditional_chinese', fallback='0') @@ -506,6 +508,7 @@ def load_and_log_configs(): 'baidu_search_api_key': baidu_search_api_key, 'bing_search_api_key': bing_search_api_key, 'bing_country_code': bing_country_code, + 'bing_search_api_url': bing_search_api_url, 'brave_search_api_key': brave_search_api_key, 'brave_country_code': brave_country_code, 'duckduckgo_search_api_key': duckduckgo_search_api_key, diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 49cb7a9b8..f196298c5 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -3,64 +3,44 @@ # # Imports import json -import os -from random import choice -from time import sleep -from typing import Optional, Dict, List +import logging +from typing import Optional, Dict, List, Any from urllib.parse import urlparse, urlencode - -import requests - -from App_Function_Libraries.Utils.Utils import loaded_config_data - - # # 3rd-Party Imports +import requests +from requests import RequestException # # Local Imports +from App_Function_Libraries.Utils.Utils import loaded_config_data # ####################################################################################################################### # # Functions: # 1. perform_websearch -# 1. search_baidu -# 2. test_baidu_search -# 3. search_bing -# 4. test_search_bing -# 5. search_brave -# 6. test_search_brave -# 7. search_duckduckgo -# 8. test_duckduckgo_search -# 9. search_google -# 10. test_search_google -# 11. search_kagi -# 12. test_search_kagi -# 13. search_searx -# 14. test_search_searx -# 15. search_yandex -# 16. test_search_yandex +# 2. search_web_baidu # ####################################################################################################################### # def perform_websearch(search_engine, search_query, country, search_lang, output_lang, result_count, date_range, - safesearch, site_blacklist, api_key): + safesearch, site_blacklist): if search_engine.lower() == "baidu": return search_web_baidu() elif search_engine.lower() == "bing": - return search_web_bing(search_query, search_lang, country, date_range, result_count, api_key) + return search_web_bing(search_query, search_lang, country, date_range, result_count) elif search_engine.lower() == "brave": - return search_web_brave(search_query, country, search_lang, output_lang, result_count, safesearch, api_key, - site_blacklist, date_range) + return search_web_brave(search_query, country, search_lang, output_lang, result_count, safesearch, + site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": return search_web_ddg() elif search_engine.lower() == "google": - return search_web_google(search_query, api_key, google_search_engine_id, result_count, c2coff, - results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, + return search_web_google(search_query, result_count, results_origin_country, date_range, exactTerms, + excludeTerms, filter, geolocation, output_lang, search_result_language, safesearch, site_blacklist, sort_results_by) elif search_engine.lower() == "kagi": return search_web_kagi(search_query, country, search_lang, output_lang, result_count, safesearch, date_range, - site_blacklist, api_key) + site_blacklist) elif search_engine.lower() == "serper": return search_web_serper() elif search_engine.lower() == "tavily": @@ -73,7 +53,10 @@ def perform_websearch(search_engine, search_query, country, search_lang, output_ return f"Error: Invalid Search Engine Name {search_engine}" +######################### Baidu Search ######################### +# # https://cloud.baidu.com/doc/APIGUIDE/s/Xk1myz05f +# https://oxylabs.io/blog/how-to-scrape-baidu-search-results def search_web_baidu(): pass @@ -83,20 +66,28 @@ def test_baidu_search(arg1, arg2, arg3): return result +######################### Bing Search ######################### +# +# https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/overview0 +# https://learn.microsoft.com/en-us/bing/search-apis/bing-news-search/overview # https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/quickstarts/rest/python # https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/query-parameters # Country/Language code: https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes#country-codes +# https://github.com/Azure-Samples/cognitive-services-REST-api-samples/tree/master/python/Search def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bing_api_key=None, date_range=None): + # Load Search API URL from config file + search_url = loaded_config_data['search_engines']['bing_search_api_url'] + if not bing_api_key: # load key from config file - bing_api_key = loaded_config_data['search-engines']['bing_search_api_key'] + bing_api_key = loaded_config_data['search_engines']['bing_search_api_key'] if not bing_api_key: - raise ValueError("Please provide a valid Bing Search API subscription key") - search_url = "https://api.bing.microsoft.com/v7.0/search" + raise ValueError("Please Configure a valid Bing Search API key") + if not result_count: # Perform check in config file for default search result count - answer_count = loaded_config_data['search-engines']['search_result_max'] + answer_count = loaded_config_data['search_engines']['search_result_max'] else: answer_count = result_count @@ -112,19 +103,45 @@ def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bi # Country settings if not bing_country: # do config check for default search country - bing_country = loaded_config_data['search-engines']['bing_country_code'] + bing_country = loaded_config_data['search_engines']['bing_country_code'] else: setcountry = bing_country + # Construct a request + mkt = 'en-US' + params = {'q': search_query, 'mkt': mkt} +# params = {"q": search_query, "mkt": bing_country, "textDecorations": True, "textFormat": "HTML", "count": answer_count, +# "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} + headers = {'Ocp-Apim-Subscription-Key': bing_api_key} + + # Call the API + try: + response = requests.get(search_url, headers=headers, params=params) + response.raise_for_status() + + print("Headers: ") + print(response.headers) + + print("JSON Response: ") + print(response.json()) + bing_search_results = response.json() + return bing_search_results + except Exception as ex: + raise ex - headers = {"Ocp-Apim-Subscription-Key": bing_api_key} - params = {"q": search_query, "mkt": bing_country, "textDecorations": True, "textFormat": "HTML", "count": answer_count, - "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} - response = requests.get(search_url, headers=headers, params=params) - response.raise_for_status() - bing_search_results = response.json() - return bing_search_results +def test_search_web_bing(): + search_query = "How can I bake a cherry cake" + bing_lang = "en" + bing_country = "US" + result_count = 10 + bing_api_key = None + date_range = None + result = search_web_bing(search_query, bing_lang, bing_country, result_count, bing_api_key, date_range) + print(result) + +######################### Brave Search ######################### +# # https://brave.com/search/api/ # https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-brave-search/README.md def search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", @@ -132,11 +149,11 @@ def search_web_brave(search_term, country, search_lang, ui_lang, result_count, s search_url = "https://api.search.brave.com/res/v1/web/search" if not brave_api_key: # load key from config file - brave_api_key = loaded_config_data['search-engines']['brave_search_api_key'] + brave_api_key = loaded_config_data['search_engines']['brave_search_api_key'] if not brave_api_key: raise ValueError("Please provide a valid Brave Search API subscription key") if not country: - brave_country = loaded_config_data['search-engines']['search_engine_country_code_brave'] + brave_country = loaded_config_data['search_engines']['search_engine_country_code_brave'] else: country = "US" if not search_lang: @@ -170,233 +187,208 @@ def test_search_brave(search_term, country, search_lang, ui_lang, result_count, return result +######################### DuckDuckGo Search ######################### +# # https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py # FIXME - 1shot gen with sonnet 3.5, untested. -def search_web_ddg( - keywords: str, - region: str = "wt-wt", - safesearch: str = "moderate", - max_results: Optional[int] = None -) -> List[Dict[str, str]]: - """ - Perform a DuckDuckGo search and return results. - - Args: - keywords: Search query - region: Region code (e.g., "wt-wt", "us-en") - safesearch: "on", "moderate", or "off" - max_results: Maximum number of results to return - - Returns: - List of dictionaries containing search results with 'title', 'href', and 'body' keys - """ - - # User agent list to randomize requests - user_agents = [ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0" - ] - - # Helper function to get vqd parameter - def get_vqd(keywords: str) -> str: - headers = {'User-Agent': choice(user_agents)} - resp = requests.get( - "https://duckduckgo.com/", - params={'q': keywords}, - headers=headers - ) +def search_web_ddg(): + # fuck it https://github.com/deedy5/duckduckgo_search/tree/main?tab=readme-ov-file + #return results + pass - # Extract vqd from response - try: - vqd = resp.text.split('vqd="')[1].split('"')[0] - return vqd - except IndexError: - raise Exception("Could not extract vqd parameter") - - results = [] - vqd = get_vqd(keywords) - - # Search parameters - params = { - 'q': keywords, - 'kl': region, - 'l': region, - 'p': '1' if safesearch == "on" else "", - 's': '0', - 'df': '', - 'vqd': vqd, - 'ex': '-1' if safesearch == "moderate" else '-2' if safesearch == "off" else "" - } - - headers = { - 'User-Agent': choice(user_agents), - 'Referer': 'https://duckduckgo.com/' - } - - # Keep track of seen URLs to avoid duplicates - seen_urls = set() +def test_search_web_ddg(): + """Example usage of the DuckDuckGo search function""" try: - while True: - response = requests.get( - 'https://links.duckduckgo.com/d.js', - params=params, - headers=headers - ) - - if response.status_code != 200: - break - - # Parse JSON response - try: - data = json.loads(response.text.replace('DDG.pageLayout.load(', '').rstrip(');')) - except json.JSONDecodeError: - break - - # Extract results - for result in data['results']: - url = result.get('u') - if url and url not in seen_urls: - seen_urls.add(url) - results.append({ - 'title': result.get('t', ''), - 'href': url, - 'body': result.get('a', '') - }) - - if max_results and len(results) >= max_results: - return results - - # Check for next page - if 'next' not in data: - break - - params['s'] = data['next'] - sleep(0.5) # Be nice to the server + # Basic search + results = search_web_ddg("How to bake a cherry cake") + print(f"Found {len(results)} results for 'How to bake a cherry cake'") + + # Print first 3 results + for i, result in enumerate(results[:3], 1): + print(f"\nResult {i}:") + print(f"Title: {result['title']}") + print(f"URL: {result['href']}") + print(f"Description: {result['body'][:150]}...") + + # Search with different parameters + limited_results = search_web_ddg( + keywords="artificial intelligence news", + region="us-en", + safesearch="on", + max_results=5 + ) + print(f"\nFound {len(limited_results)} limited results") except Exception as e: - print(f"Error during search: {e}") - - return results - # def example_usage(): - # """Example usage of the DuckDuckGo search function""" - # try: - # # Basic search - # results = search_web_ddg("Python programming") - # print(f"Found {len(results)} results for 'Python programming'") - # - # # Print first 3 results - # for i, result in enumerate(results[:3], 1): - # print(f"\nResult {i}:") - # print(f"Title: {result['title']}") - # print(f"URL: {result['href']}") - # print(f"Description: {result['body'][:150]}...") - # - # # Search with different parameters - # limited_results = duckduckgo_search( - # keywords="artificial intelligence news", - # region="us-en", - # safesearch="on", - # max_results=5 - # ) - # print(f"\nFound {len(limited_results)} limited results") - # - # except DuckDuckGoSearchException as e: - # print(f"Search failed: {e}") - -def test_duckduckgo_search(): - pass + print(f"Search failed: {e}") +######################### Google Search ######################### +# # https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list -def search_web_google(search_query, google_search_api_key, google_search_engine_id, result_count, c2coff, - results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation,ui_language, - search_result_language, safesearch, site_blacklist, sort_results_by): - search_url = "https://www.googleapis.com/customsearch/v1?" - # FIXME - build the query string dynamically - if not c2coff: - # check config file for default setting - enable_traditional_chinese = loaded_config_data['search-engines']['google_simp_trad_chinese'] - else: - enable_traditional_chinese = "0" - - if not results_origin_country: - # check config file for default setting - limit_country_search = loaded_config_data['search-engines']['limit_google_search_to_country'] - if limit_country_search: - results_origin_country = loaded_config_data['search-engines']['google_search_country'] - - if not google_search_engine_id: - # load key from config file - google_search_engine_id = loaded_config_data['search-engines']['google_search_engine_id'] +def search_web_google( + search_query: str, + google_search_api_key: Optional[str] = None, + google_search_engine_id: Optional[str] = None, + result_count: Optional[int] = None, + c2coff: Optional[str] = None, + results_origin_country: Optional[str] = None, + date_range: Optional[str] = None, + exactTerms: Optional[str] = None, + excludeTerms: Optional[str] = None, + filter: Optional[str] = None, + geolocation: Optional[str] = None, + ui_language: Optional[str] = None, + search_result_language: Optional[str] = None, + safesearch: Optional[str] = None, + site_blacklist: Optional[str] = None, + sort_results_by: Optional[str] = None +) -> Dict[str, Any]: + """ + Perform a Google web search with the given parameters. + + :param search_query: The search query string + :param google_search_api_key: Google Search API key + :param google_search_engine_id: Google Search Engine ID + :param result_count: Number of results to return + :param c2coff: Enable/disable traditional Chinese search + :param results_origin_country: Limit results to a specific country + :param date_range: Limit results to a specific date range + :param exactTerms: Exact terms that must appear in results + :param excludeTerms: Terms that must not appear in results + :param filter: Control duplicate content filter + :param geolocation: Geolocation of the user + :param ui_language: Language of the user interface + :param search_result_language: Language of search results + :param safesearch: Safe search setting + :param site_blacklist: Sites to exclude from search + :param sort_results_by: Sorting criteria for results + :return: JSON response from Google Search API + """ + try: + # Load Search API URL from config file + search_url = loaded_config_data['search_engines']['google_search_api_url'] + logging.info(f"Using search URL: {search_url}") + + # Initialize params dictionary + params: Dict[str, Any] = {"q": search_query} + + # Handle c2coff + if c2coff is None: + c2coff = loaded_config_data['search_engines']['google_simp_trad_chinese'] + if c2coff is not None: + params["c2coff"] = c2coff + + # Handle results_origin_country + if results_origin_country is None: + limit_country_search = loaded_config_data['search_engines']['limit_google_search_to_country'] + if limit_country_search: + results_origin_country = loaded_config_data['search_engines']['google_search_country'] + if results_origin_country: + params["cr"] = results_origin_country + + # Handle google_search_engine_id + if google_search_engine_id is None: + google_search_engine_id = loaded_config_data['search_engines']['google_search_engine_id'] if not google_search_engine_id: raise ValueError("Please set a valid Google Search Engine ID in the config file") + params["cx"] = google_search_engine_id - if not google_search_api_key: - # load key from config file - google_search_api_key = loaded_config_data['search-engines']['google_search_api_key'] + # Handle google_search_api_key + if google_search_api_key is None: + google_search_api_key = loaded_config_data['search_engines']['google_search_api_key'] if not google_search_api_key: - raise ValueError("Please provide a valid Bing Search API subscription key") + raise ValueError("Please provide a valid Google Search API subscription key") + params["key"] = google_search_api_key + + # Handle other parameters + if result_count: + params["num"] = result_count + if date_range: + params["dateRestrict"] = date_range + if exactTerms: + params["exactTerms"] = exactTerms + if excludeTerms: + params["excludeTerms"] = excludeTerms + if filter: + params["filter"] = filter + if geolocation: + params["gl"] = geolocation + if ui_language: + params["hl"] = ui_language + if search_result_language: + params["lr"] = search_result_language + if safesearch is None: + safesearch = loaded_config_data['search_engines']['google_safe_search'] + if safesearch: + params["safe"] = safesearch + if site_blacklist: + params["siteSearch"] = site_blacklist + params["siteSearchFilter"] = "e" # Exclude these sites + if sort_results_by: + params["sort"] = sort_results_by + + logging.info(f"Prepared parameters for Google Search: {params}") + + # Make the API call + response = requests.get(search_url, params=params) + response.raise_for_status() + google_search_results = response.json() - # Logic for handling date range - if not date_range: - date_range = None - - # Logic for handling exact terms - FIXMe - # Identifies a phrase that all documents in the search results must contain. - if not exactTerms: - exactTerms = None - - # Logic for handling exclude terms - FIXME - # Identifies a word or phrase that should not appear in any documents in the search results. - if not excludeTerms: - excludeTerms = None - - # Logic for handling filter - # Controls turning on or off the duplicate content filter. - # Default is 1 (On). - if not filter: - filter = "1" - - # Logic for handling geolocation - # Country codes: https://developers.google.com/custom-search/docs/json_api_reference#countryCodes - # Country location of user - if not geolocation: - geolocation = None - - # Logic for handling UI language - # https://developers.google.com/custom-search/docs/json_api_reference#wsInterfaceLanguages - # https://developers.google.com/custom-search/docs/json_api_reference#interfaceLanguages - if not ui_language: - ui_language = None - - # Logic for handling search result language - if not search_result_language: - search_result_language = None - - if not safesearch: - safe_search = loaded_config_data['search-engines']['google_safe_search'] - else: - safe_search = safesearch + logging.info(f"Successfully retrieved search results. Items found: {len(google_search_results.get('items', []))}") - if not site_blacklist: - site_blacklist = None + return google_search_results - if not sort_results_by: - sort_results_by = None + except ValueError as ve: + logging.error(f"Configuration error: {str(ve)}") + raise - params = {"c2coff": enable_traditional_chinese, "key": google_search_api_key, "cx": google_search_engine_id, "q": search_query} - response = requests.get(search_url, params=params) - response.raise_for_status() - google_search_results = response.json() - return google_search_results + except RequestException as re: + logging.error(f"Error during API request: {str(re)}") + raise + + except Exception as e: + logging.error(f"Unexpected error occurred: {str(e)}") + raise def test_search_google(): + search_query = "How can I bake a cherry cake?" + google_search_api_key = loaded_config_data['search_engines']['google_search_api_key'] + google_search_engine_id = loaded_config_data['search_engines']['google_search_engine_id'] + result_count = 10 + c2coff = "1" + results_origin_country = "US" + date_range = None + exactTerms = None + excludeTerms = None + filter = None + geolocation = "us" + ui_language = "en" + search_result_language = "lang_en" + safesearch = "off" + site_blacklist = None + sort_results_by = None + result = search_web_google(search_query, google_search_api_key, google_search_engine_id, result_count, c2coff, + results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation,ui_language, + search_result_language, safesearch, site_blacklist, sort_results_by) + print(result) + + +######################### Jina.ai Search ######################### +# +# https://jina.ai/reader/ +def search_web_jina(arg1, arg2, arg3): pass +def test_search_jina(arg1, arg2, arg3): + result = search_web_jina(arg1, arg2, arg3) + return result + +######################### Kagi Search ######################### +# # https://help.kagi.com/kagi/api/search.html def search_web_kagi(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, result_filter=None, kagi_api_key=None): @@ -441,11 +433,13 @@ def test_search_kagi(): pass +######################### SearX Search ######################### +# # https://searx.space # https://searx.github.io/searx/dev/search_api.html def search_web_searx(search_query): # Check if API URL is configured - searx_url = loaded_config_data['search-engines']['searx_search_api_url'] + searx_url = loaded_config_data['search_engines']['searx_search_api_url'] if not searx_url: return "Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet." @@ -495,6 +489,8 @@ def test_search_searx(): pass +######################### Serper.dev Search ######################### +# # https://github.com/YassKhazzan/openperplex_backend_os/blob/main/sources_searcher.py def search_web_serper(): pass @@ -504,6 +500,8 @@ def test_search_serper(): pass +######################### Tavily Search ######################### +# # https://docs.tavily.com/docs/rest-api/api-reference def search_web_tavily(): tavily_url = "https://api.tavily.com/search" @@ -514,6 +512,8 @@ def test_search_tavily(): pass +######################### Yandex Search ######################### +# # https://yandex.cloud/en/docs/search-api/operations/web-search # https://yandex.cloud/en/docs/search-api/quickstart/ # https://yandex.cloud/en/docs/search-api/concepts/response diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index 7fa5ce41b..775de3166 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -5,6 +5,6 @@ https://arxiv.org/abs/2412.02035 https://github.com/andreamust/NEON-GPT https://excalidraw.com/ - +https://arxiv.org/abs/2411.07407 diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 118e693e5..17099eda0 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -16,11 +16,17 @@ https://www.ginkgonotes.com/ https://www.reddit.com/r/Anki/comments/17u01ge/spaced_repetition_algorithm_a_threeday_journey/ https://github.com/open-spaced-repetition/fsrs4anki/wiki/Spaced-Repetition-Algorithm:-A-Three%E2%80%90Day-Journey-from-Novice-to-Expert#day-3-the-latest-progress https://www.scrapingdog.com/blog/scrape-google-news/ - - - - - +https://github.com/mistralai/cookbook/blob/main/third_party/LlamaIndex/llamaindex_arxiv_agentic_rag.ipynb +https://github.com/ai-christianson/RA.Aid +https://github.com/cbuccella/perplexity_research_prompt/blob/main/general_research_prompt.md +https://github.com/0xeb/TheBigPromptLibrary/blob/main/SystemPrompts/Perplexity.ai/20241024-Perplexity-Desktop-App.md +https://github.com/rashadphz/farfalle +https://github.com/cbuccella/perplexity_research_prompt/blob/main/general_research_prompt.md +https://www.emergentmind.com/ +https://github.com/neuml/paperai +https://github.com/neuml/paperetl +https://github.com/ai-christianson/RA.Aid +https://github.com/Future-House/paper-qa ### Ideas diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index cbdc9d674..932e41e72 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -89,6 +89,10 @@ MaskGCT https://github.com/open-mmlab/Amphion/blob/main/models/tts/maskgct/maskgct_inference.py https://huggingface.co/amphion/MaskGCT +Parler + https://github.com/huggingface/parler-tts + + TTS Pipeline https://www.astramind.ai/post/auralis @@ -160,11 +164,192 @@ https://github.com/fishaudio/fish-speech/blob/main/Start_Agent.md https://huggingface.co/fishaudio/fish-agent-v0.1-3b/tree/main https://github.com/pixelpump/Ai-Interview-Assistant-Python - +https://github.com/coqui-ai/TTS https://github.com/Standard-Intelligence/hertz-dev +https://github.com/2noise/ChatTTS https://github.com/edwko/OuteTTS https://huggingface.co/OuteAI/OuteTTS-0.2-500M-GGUF https://huggingface.co/NexaAIDev/Qwen2-Audio-7B-GGUF -https://www.twilio.com/en-us/blog/twilio-openai-realtime-api-launch-integration \ No newline at end of file +https://www.twilio.com/en-us/blog/twilio-openai-realtime-api-launch-integration +https://github.com/huggingface/speech-to-speech +https://github.com/harvestingmoon/S2S +https://github.com/collabora/WhisperLive +https://github.com/JarodMica/audiobook_maker +https://github.com/myshell-ai/OpenVoice +https://github.com/JarodMica/GPT-SoVITS-Package +https://github.com/shagunmistry/NotebookLM_Alternative/tree/main/ai_helper +https://docs.cartesia.ai/get-started/make-an-api-request +https://github.com/JarodMica/open-neruosama +https://github.com/flatmax/speech-to-text + + +Google +https://github.com/google-gemini/cookbook/tree/main/gemini-2 +https://discuss.ai.google.dev/t/how-does-one-get-access-to-the-api-for-tts-features-of-gemini-2-0/53925/15 +``` +import asyncio +import base64 +import json +import numpy as np +import os +import websockets +import wave +import contextlib +import pygame +from IPython.display import display, Markdown + +# ANSI color codes +GREEN = "\033[92m" +YELLOW = "\033[93m" +RED = "\033[91m" +BLUE = "\033[94m" +RESET = "\033[0m" + +voices = {"Puck", "Charon", "Kore", "Fenrir", "Aoede"} + +# --- Configuration --- +MODEL = 'models/gemini-2.0-flash-exp' +GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY") +if not GOOGLE_API_KEY: + raise EnvironmentError("GOOGLE_API_KEY environment variable is not set.") +HOST = 'generativelanguage.googleapis.com' +URI = f'wss://{HOST}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={GOOGLE_API_KEY}' + +# Audio parameters +WAVE_CHANNELS = 1 # Mono audio +WAVE_RATE = 24000 +WAVE_SAMPLE_WIDTH = 2 + + +@contextlib.contextmanager +def wave_file(filename, channels=WAVE_CHANNELS, rate=WAVE_RATE, sample_width=WAVE_SAMPLE_WIDTH): + """Context manager for creating and managing wave files.""" + try: + with wave.open(filename, "wb") as wf: + wf.setnchannels(channels) + wf.setsampwidth(sample_width) + wf.setframerate(rate) + yield wf + except wave.Error as e: + print(f"{RED}Error opening wave file '{filename}': {e}{RESET}") + raise + + +async def audio_playback_task(file_name, stop_event): + """Plays audio using pygame until stopped.""" + print(f"{BLUE}Starting playback: {file_name}{RESET}") + try: + pygame.mixer.music.load(file_name) + pygame.mixer.music.play() + while pygame.mixer.music.get_busy() and not stop_event.is_set(): + await asyncio.sleep(0.1) + except pygame.error as e: + print(f"{RED}Pygame error during playback: {e}{RESET}") + except Exception as e: + print(f"{RED}Unexpected error during playback: {e}{RESET}") + finally: + print(f"{BLUE}Playback complete: {file_name}{RESET}") + + +async def generate_audio(ws, text_input: str, voice_name="Kore") -> None: + """ + Sends text input to the Gemini API, receives an audio response, saves it to a file, and plays it back. + Relies on the server to maintain the session history. + """ + pygame.mixer.init() # Initialize pygame mixer + + msg = { + "client_content": { + "turns": [{"role": "user", "parts": [{"text": text_input}]}], + "turn_complete": True, + } + } + await ws.send(json.dumps(msg)) + + responses = [] + async for raw_response in ws: + response = json.loads(raw_response.decode()) + server_content = response.get("serverContent") + if server_content is None: + break + + model_turn = server_content.get("modelTurn") + if model_turn: + parts = model_turn.get("parts") + if parts: + for part in parts: + if "inlineData" in part and "data" in part["inlineData"]: + pcm_data = base64.b64decode(part["inlineData"]["data"]) + responses.append(np.frombuffer(pcm_data, dtype=np.int16)) + + turn_complete = server_content.get("turnComplete") + if turn_complete: + break + + if responses: + display(Markdown(f"{YELLOW}**Response >**{RESET}")) + audio_array = np.concatenate(responses) + file_name = 'output.wav' + with wave_file(file_name) as wf: + wf.writeframes(audio_array.tobytes()) + stop_event = asyncio.Event() + try: + await audio_playback_task(file_name, stop_event) + except Exception as e: + print(f"{RED}Error during audio playback: {e}{RESET}") + else: + print(f"{YELLOW}No audio returned{RESET}") + pygame.mixer.quit() # clean up pygame mixer + + +async def main(): + print(f"{GREEN}Available voices: {', '.join(voices)}{RESET}") + default_voice = "Kore" + print(f"{GREEN}Default voice is set to: {default_voice}, you can change it in the code{RESET}") + + config = { + "response_modalities": ["AUDIO"], + "speech_config": { + "voice_config": { + "prebuilt_voice_config": { + "voice_name": default_voice # Set voice + } + } + } + } + + async with websockets.connect(URI) as ws: + + async def setup(ws) -> None: + await ws.send( + json.dumps( + { + "setup": { + "model": MODEL, + "generation_config": config, + } + } + ) + ) + + raw_response = await ws.recv(decode=False) + setup_response = json.loads(raw_response.decode("ascii")) + print(f"{GREEN}Connected: {setup_response}{RESET}") + + await setup(ws) + while True: + text_prompt = input(f"{YELLOW}Enter your text (or type 'exit' to quit): {RESET}") + if text_prompt.lower() == "exit": + break + + try: + await generate_audio(ws, text_prompt, default_voice) + except Exception as e: + print(f"{RED}An error occurred: {e}{RESET}") + + +if __name__ == "__main__": + asyncio.run(main()) +``` \ No newline at end of file diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 6626267c3..c35a70bdb 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -50,6 +50,10 @@ https://bolt.new/ https://github.com/rmusser01/agentic_rag/tree/main https://github.com/astramind-ai/Pulsar https://github.com/woshixiaobai2019/mirau-chat-ui +https://uxdesign.cc/the-importance-of-hover-states-c9312d7fd516 +https://trends.uxdesign.cc/ +https://bolters.io/ + https://github.com/Vali-98/ChatterUI https://www.nngroup.com/videos/efficient-error-messages/ https://uxdesign.cc/a-good-design-is-a-good-essay-197f4bd31c92 @@ -68,6 +72,7 @@ https://github.com/stackblitz-labs/bolt.diy https://github.com/Haervwe/open-webui-tools https://uxdesign.cc/the-evolution-of-attention-c6154276f1b4 https://otranscribe.com/ +https://www.dive.club/ideas/new-types-of-ui-that-generative-ai-has-necessitated https://interactjs.io/ https://perchance.org/ai-chat https://uxdesign.cc/youre-trying-to-turn-an-oil-tanker-288ed30876e4 diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index da9e9df02..9fce2d2e0 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -17,6 +17,9 @@ https://huggingface.co/tencent/HunyuanVideo https://huggingface.co/OpenGVLab/InternVL2_5-38B https://ivy-lvlm.github.io/Video-MA2MBA/ https://internvl.github.io/blog/2024-12-20-InternVL-2.5-MPO/ +https://huggingface.co/allenai/Molmo-7B-D-0924 +https://github.com/deepseek-ai/DeepSeek-VL2 +https://huggingface.co/Qwen/QVQ-72B-Preview https://huggingface.co/Qwen/Qwen2-VL-72B https://huggingface.co/Qwen/Qwen2-VL-7B https://huggingface.co/Qwen/Qwen2-VL-2B diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 6c3b0bfc7..20ecba7b3 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -3,6 +3,22 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. + + +### Setting the Stage +The web search functionality is a core component of the tldw system, allowing users to quickly and easily search the web for information on a given topic. This functionality is designed to be simple and intuitive, providing users with the information they need in a clear and concise manner. + +#### Bing Search +- [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) +- Getting Started with Bing Search API + - Sign up for a Bing Search API key via Azure (1000 Free Searches a month) - https://www.microsoft.com/en-us/bing/apis/pricing + - Use the Bing Search API to perform searches - Add the generated subscription key to your config.txt file. + - If for some reason you're doing modifications to the code, be aware: https://github.com/Azure-Samples/cognitive-services-REST-api-samples/issues/139 + - Perform searches using Bing! + + + + https://github.com/scrapinghub/article-extraction-benchmark ask.py https://github.com/pengfeng/ask.py diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 977cc4c9d..8b674b51f 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -42,7 +42,7 @@ https://github.com/chigkim/openai-api-gpqa https://github.com/chigkim/Ollama-MMLU-Pro https://huggingface.co/ymcki/Llama-3_1-Nemotron-51B-Instruct-GGUF https://pub.towardsai.net/streamline-your-llm-evaluation-a-step-by-step-guide-to-rag-metrics-with-streamlit-38ed9efbdc9a - +https://huggingface.co/QuantFactory/granite-3.1-8b-instruct-GGUF Have LLMs play Social deception games diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index ca3aa28be..c27593d41 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -37,7 +37,9 @@ https://weaviate.io/blog/late-chunking https://github.com/superlinear-ai/raglite https://arxiv.org/abs/2412.15101 https://arxiv.org/abs/2412.14510 - +https://towardsdatascience.com/advanced-retrieval-techniques-for-better-rags-c53e1b03c183 +https://ai.gopubby.com/agentic-rags-extending-rags-to-sql-databases-1509b25ca3e7 +https://ai.plainenglish.io/a-deep-dive-into-golden-retriever-eea3396af3b4 @@ -67,7 +69,11 @@ GraphRAG https://memgraph.com/webinars/optimizing-insulin-management-the-role-of-graphrag-in-patient-care https://ai.plainenglish.io/metagraphs-and-hypergraphs-for-complex-ai-agent-memory-and-rag-717f6f3589f5 https://arxiv.org/html/2411.15671v1 -https://arxiv.org/pdf/2410.04739v1 + https://arxiv.org/pdf/2410.04739v1 + https://towardsdatascience.com/how-to-query-a-knowledge-graph-with-llms-using-grag-38bfac47a322 + https://arxiv.org/abs/2408.04187 + + ### Links From 11814d175ce21f49214aab54bfb74c7535dd880c Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 26 Dec 2024 22:43:45 -0800 Subject: [PATCH 26/88] search APIs --- .../models/pyannote_diarization_config.yaml | 13 ++ App_Function_Libraries/Utils/Utils.py | 5 +- .../Web_Scraping/WebSearch_APIs.py | 128 +++++++++--------- Docs/Design/Education.md | 3 +- Docs/Design/RSS_Ranking.md | 2 +- Docs/Design/Researcher.md | 6 + Docs/Design/Search.md | 6 +- Docs/Design/TTS_STT.md | 4 +- Docs/Design/WebSearch.md | 12 ++ Docs/Handy_Dandy_Papers.md | 3 +- Docs/Issues/Citations_and_Confabulations.md | 1 + Docs/Issues/Evaluation_Plans.md | 2 + Docs/RAG_Notes.md | 4 +- 13 files changed, 116 insertions(+), 73 deletions(-) create mode 100644 App_Function_Libraries/Audio/models/pyannote_diarization_config.yaml diff --git a/App_Function_Libraries/Audio/models/pyannote_diarization_config.yaml b/App_Function_Libraries/Audio/models/pyannote_diarization_config.yaml new file mode 100644 index 000000000..675f70f26 --- /dev/null +++ b/App_Function_Libraries/Audio/models/pyannote_diarization_config.yaml @@ -0,0 +1,13 @@ +pipeline: + params: + clustering: AgglomerativeClustering + embedding: /FULL/PATH/TO/SCRIPT/tldw/App_Function_Libraries/models/pyannote_model_wespeaker-voxceleb-resnet34-LM.bin #models/pyannote_model_wespeaker-voxceleb-resnet34-LM.bin + segmentation: /FULL/PATH/TO/SCRIPT/tldw/App_Function_Libraries/models/pyannote_model_segmentation-3.0.bin #models/pyannote_model_segmentation-3.0.bin + +params: + segmentation: + min_duration_off: 0.0 + clustering: + method: centroid + min_cluster_size: 12 + threshold: 0.7045654963945799 \ No newline at end of file diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 9a9f6f3af..efa49142e 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -362,7 +362,8 @@ def load_and_log_configs(): bing_country_code = config.get('Search-Engines', 'search_engine_country_code_bing', fallback='us') bing_search_api_url = config.get('Search-Engines', 'search_engine_api_url_bing', fallback='') # Brave Search Settings - brave_search_api_key = config.get('Search-Engines', 'search_engine_api_key_brave', fallback='') + brave_search_api_key = config.get('Search-Engines', 'search_engine_api_key_brave_regular', fallback='') + brave_search_ai_api_key = config.get('Search-Engines', 'search_engine_api_key_brave_ai', fallback='') brave_country_code = config.get('Search-Engines', 'search_engine_country_code_brave', fallback='us') # DuckDuckGo Search Settings duckduckgo_search_api_key = config.get('Search-Engines', 'search_engine_api_key_duckduckgo', fallback='') @@ -510,8 +511,10 @@ def load_and_log_configs(): 'bing_country_code': bing_country_code, 'bing_search_api_url': bing_search_api_url, 'brave_search_api_key': brave_search_api_key, + 'brave_search_ai_api_key': brave_search_ai_api_key, 'brave_country_code': brave_country_code, 'duckduckgo_search_api_key': duckduckgo_search_api_key, + 'google_search_api_url': google_search_api_url, 'google_search_api_key': google_search_api_key, 'google_search_engine_id': google_search_engine_id, 'google_simp_trad_chinese': google_simp_trad_chinese, diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index f196298c5..3c7214b7c 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -23,6 +23,8 @@ ####################################################################################################################### # + + def perform_websearch(search_engine, search_query, country, search_lang, output_lang, result_count, date_range, safesearch, site_blacklist): if search_engine.lower() == "baidu": @@ -33,7 +35,7 @@ def perform_websearch(search_engine, search_query, country, search_lang, output_ return search_web_brave(search_query, country, search_lang, output_lang, result_count, safesearch, site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": - return search_web_ddg() + return search_web_duckduckgo(arg1, arg2) elif search_engine.lower() == "google": return search_web_google(search_query, result_count, results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, @@ -53,6 +55,25 @@ def perform_websearch(search_engine, search_query, country, search_lang, output_ return f"Error: Invalid Search Engine Name {search_engine}" +######################### Search Results Parsing ######################### +def parse_html_search_results_generic(soup): + results = [] + for result in soup.find_all('div', class_='result'): + title = result.find('h3').text if result.find('h3') else '' + url = result.find('a', class_='url')['href'] if result.find('a', class_='url') else '' + content = result.find('p', class_='content').text if result.find('p', class_='content') else '' + published_date = result.find('span', class_='published_date').text if result.find('span', + class_='published_date') else '' + + results.append({ + 'title': title, + 'url': url, + 'content': content, + 'publishedDate': published_date + }) + return results + + ######################### Baidu Search ######################### # # https://cloud.baidu.com/doc/APIGUIDE/s/Xk1myz05f @@ -145,7 +166,7 @@ def test_search_web_bing(): # https://brave.com/search/api/ # https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/tools/llama-index-tools-brave-search/README.md def search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", - brave_api_key=None, result_filter=None, date_range=None, ): + brave_api_key=None, result_filter=None, search_type="ai", date_range=None): search_url = "https://api.search.brave.com/res/v1/web/search" if not brave_api_key: # load key from config file @@ -166,6 +187,10 @@ def search_web_brave(search_term, country, search_lang, ui_lang, result_count, s # date_range = "month" if not result_filter: result_filter = "webpages" + if search_type == "ai": + # FIXME - Option for switching between AI/Regular search + pass + headers = {"Accept": "application/json", "Accept-Encoding": "gzip", "X-Subscription-Token": brave_api_key} @@ -180,48 +205,27 @@ def search_web_brave(search_term, country, search_lang, ui_lang, result_count, s return brave_search_results -def test_search_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, - result_filter=None, brave_api_key=None): +def test_search_brave(): + search_term = "How can I bake a cherry cake" + country = "US" + search_lang = "en" + ui_lang = "en" + result_count = 10 + safesearch = "moderate" + date_range = None + result_filter = None result = search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch, date_range, - result_filter, brave_api_key) + result_filter) + print(result) return result ######################### DuckDuckGo Search ######################### # -# https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py -# FIXME - 1shot gen with sonnet 3.5, untested. -def search_web_ddg(): - # fuck it https://github.com/deedy5/duckduckgo_search/tree/main?tab=readme-ov-file - #return results - pass +# https://github.com/deedy5/duckduckgo_search +def search_web_duckduckgo(arg1, arg2) -def test_search_web_ddg(): - """Example usage of the DuckDuckGo search function""" - try: - # Basic search - results = search_web_ddg("How to bake a cherry cake") - print(f"Found {len(results)} results for 'How to bake a cherry cake'") - - # Print first 3 results - for i, result in enumerate(results[:3], 1): - print(f"\nResult {i}:") - print(f"Title: {result['title']}") - print(f"URL: {result['href']}") - print(f"Description: {result['body'][:150]}...") - - # Search with different parameters - limited_results = search_web_ddg( - keywords="artificial intelligence news", - region="us-en", - safesearch="on", - max_results=5 - ) - print(f"\nFound {len(limited_results)} limited results") - - except Exception as e: - print(f"Search failed: {e}") ######################### Google Search ######################### @@ -376,17 +380,6 @@ def test_search_google(): print(result) -######################### Jina.ai Search ######################### -# -# https://jina.ai/reader/ -def search_web_jina(arg1, arg2, arg3): - pass - -def test_search_jina(arg1, arg2, arg3): - result = search_web_jina(arg1, arg2, arg3) - return result - - ######################### Kagi Search ######################### # # https://help.kagi.com/kagi/api/search.html @@ -420,13 +413,8 @@ def search_web_kagi(search_term, country, search_lang, ui_lang, result_count, sa response = requests.get(search_url, headers=headers, params=params) response.raise_for_status() # Response: https://api.search.brave.com/app/documentation/web-search/responses#WebSearchApiResponse - brave_search_results = response.json() - return brave_search_results - # curl - v \ - # - H - # "Authorization: Bot $TOKEN" \ - # https: // kagi.com / api / v0 / search\?q = steve + jobs - pass + kagi_search_results = response.json() + return kagi_search_results def test_search_kagi(): @@ -437,18 +425,23 @@ def test_search_kagi(): # # https://searx.space # https://searx.github.io/searx/dev/search_api.html -def search_web_searx(search_query): +def search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general'): + # Check if API URL is configured searx_url = loaded_config_data['search_engines']['searx_search_api_url'] if not searx_url: - return "Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet." + return "SearX Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet." # Validate and construct URL try: parsed_url = urlparse(searx_url) params = { 'q': search_query, - 'format': 'json' + 'language': language, + 'time_range': time_range, + 'safesearch': safesearch, + 'pageno': pageno, + 'categories': categories } search_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}?{urlencode(params)}" except Exception as e: @@ -457,17 +450,25 @@ def search_web_searx(search_query): # Perform the search request try: headers = { - 'Content-Type': 'application/json', 'User-Agent': 'anything-llm' } response = requests.get(search_url, headers=headers) response.raise_for_status() - search_data = response.json() + + # Check if the response is JSON + content_type = response.headers.get('Content-Type', '') + if 'application/json' in content_type: + search_data = response.json() + else: + # If not JSON, assume it's HTML and parse it + from bs4 import BeautifulSoup + soup = BeautifulSoup(response.text, 'html.parser') + search_data = parse_html_search_results_generic(soup) # Process results data = [] - for result in search_data.get('results', []): + for result in search_data: data.append({ 'title': result.get('title'), 'link': result.get('url'), @@ -485,7 +486,8 @@ def search_web_searx(search_query): def test_search_searx(): - search_web_searx("How can I bake a cherry cake?") + result = search_web_searx("How can I bake a cherry cake?") + print(result) pass @@ -502,16 +504,14 @@ def test_search_serper(): ######################### Tavily Search ######################### # -# https://docs.tavily.com/docs/rest-api/api-reference +# https://github.com/YassKhazzan/openperplex_backend_os/blob/main/sources_searcher.py def search_web_tavily(): - tavily_url = "https://api.tavily.com/search" pass def test_search_tavily(): pass - ######################### Yandex Search ######################### # # https://yandex.cloud/en/docs/search-api/operations/web-search diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index 775de3166..a07a32ad4 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -6,5 +6,6 @@ https://arxiv.org/abs/2412.02035 https://github.com/andreamust/NEON-GPT https://excalidraw.com/ https://arxiv.org/abs/2411.07407 - +https://arxiv.org/abs/2412.16429 +https://huggingface.co/papers/2412.15443 diff --git a/Docs/Design/RSS_Ranking.md b/Docs/Design/RSS_Ranking.md index c633626ef..7e47157e0 100644 --- a/Docs/Design/RSS_Ranking.md +++ b/Docs/Design/RSS_Ranking.md @@ -15,7 +15,7 @@ https://blog.det.life/from-scrolls-to-similarity-search-building-a-movie-recomme https://www.dogesec.com/blog/full_text_rss_atom_blog_feeds/ https://arxiv.org/abs/2411.19352 - +https://arxiv.org/abs/2412.18082 https://blog.badsectorlabs.com/files/blogs.txt diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 17099eda0..0bf37f59d 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -27,6 +27,12 @@ https://github.com/neuml/paperai https://github.com/neuml/paperetl https://github.com/ai-christianson/RA.Aid https://github.com/Future-House/paper-qa +https://openreview.net/ +https://www.researchrabbit.ai/ +https://github.com/faraz18001/Sales-Llama +https://github.com/memgraph/memgraph + + ### Ideas diff --git a/Docs/Design/Search.md b/Docs/Design/Search.md index c4f068d20..65c7fd0ec 100644 --- a/Docs/Design/Search.md +++ b/Docs/Design/Search.md @@ -2,4 +2,8 @@ ### Link Dump -https://pub.towardsai.net/hnsw-small-world-yes-but-how-in-the-world-is-it-navigable-77701ed37e20 \ No newline at end of file +https://pub.towardsai.net/hnsw-small-world-yes-but-how-in-the-world-is-it-navigable-77701ed37e20 +https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1 + + + diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 932e41e72..b112f091d 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -43,7 +43,7 @@ https://github.com/FanaHOVA/smol-podcaster https://docs.inferless.com/cookbook/serverless-customer-service-bot https://github.com/aedocw/epub2tts https://github.com/microsoft/SpeechT5 - +https://github.com/smellslikeml/dolla_llama @@ -183,7 +183,7 @@ https://github.com/shagunmistry/NotebookLM_Alternative/tree/main/ai_helper https://docs.cartesia.ai/get-started/make-an-api-request https://github.com/JarodMica/open-neruosama https://github.com/flatmax/speech-to-text - +https://arxiv.org/abs/2412.18566 Google https://github.com/google-gemini/cookbook/tree/main/gemini-2 diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 20ecba7b3..ee6857b23 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -17,6 +17,18 @@ The web search functionality is a core component of the tldw system, allowing us - Perform searches using Bing! +#### Brave Search +Two APIs, 1 for 'AI' the other for 'regular' search + + + +#### DuckDuckGo Search +using DuckDuckGo website + +#### Google Search +Have to create a custom search engine first, get the ID and then the API key + + https://github.com/scrapinghub/article-extraction-benchmark diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 257efd12d..8ced292b2 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -48,7 +48,8 @@ https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute - https://arxiv.org/abs/2408.03314 - https://github.com/huggingface/search-and-learn - https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute - +https://arxiv.org/abs/2412.18319 +- https://github.com/hkust-nlp/B-STaR ### Personalization https://arxiv.org/abs/2411.16034 diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index 01dfe404d..27c001c47 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -14,6 +14,7 @@ https://huggingface.co/PleIAs/Pleias-Nano https://deepmind.google/discover/blog/facts-grounding-a-new-benchmark-for-evaluating-the-factuality-of-large-language-models/ https://arxiv.org/abs/2412.14860 https://arxiv.org/abs/2412.14686 +https://arxiv.org/abs/2412.18069 diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 8b674b51f..787228071 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -43,6 +43,8 @@ https://github.com/chigkim/Ollama-MMLU-Pro https://huggingface.co/ymcki/Llama-3_1-Nemotron-51B-Instruct-GGUF https://pub.towardsai.net/streamline-your-llm-evaluation-a-step-by-step-guide-to-rag-metrics-with-streamlit-38ed9efbdc9a https://huggingface.co/QuantFactory/granite-3.1-8b-instruct-GGUF +https://huggingface.co/CohereForAI/c4ai-command-r7b-12-2024 +https://arxiv.org/abs/2412.17758 Have LLMs play Social deception games diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index c27593d41..c22bf7ba2 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -40,7 +40,7 @@ https://arxiv.org/abs/2412.14510 https://towardsdatascience.com/advanced-retrieval-techniques-for-better-rags-c53e1b03c183 https://ai.gopubby.com/agentic-rags-extending-rags-to-sql-databases-1509b25ca3e7 https://ai.plainenglish.io/a-deep-dive-into-golden-retriever-eea3396af3b4 - +https://github.com/AhmedAl93/multimodal-agentic-RAG GraphRAG @@ -72,7 +72,7 @@ GraphRAG https://arxiv.org/pdf/2410.04739v1 https://towardsdatascience.com/how-to-query-a-knowledge-graph-with-llms-using-grag-38bfac47a322 https://arxiv.org/abs/2408.04187 - + https://pub.towardsai.net/building-a-knowledge-graph-from-unstructured-text-data-a-step-by-step-guide-c14c926c2229 From c9203041fb9e292e8766692489f9077af441ed4e Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 26 Dec 2024 23:36:25 -0800 Subject: [PATCH 27/88] ddg --- .../Web_Scraping/WebSearch_APIs.py | 136 ++++++++++++++++-- 1 file changed, 124 insertions(+), 12 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 3c7214b7c..88b2c3c2f 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -4,12 +4,19 @@ # Imports import json import logging -from typing import Optional, Dict, List, Any -from urllib.parse import urlparse, urlencode +import re +from html import unescape +from typing import Optional, Dict, Any +from urllib.parse import urlparse, urlencode, unquote # # 3rd-Party Imports import requests +from lxml.etree import _Element +from lxml.html import document_fromstring from requests import RequestException +from requests.adapters import HTTPAdapter +from urllib3 import Retry + # # Local Imports from App_Function_Libraries.Utils.Utils import loaded_config_data @@ -25,30 +32,30 @@ -def perform_websearch(search_engine, search_query, country, search_lang, output_lang, result_count, date_range, - safesearch, site_blacklist): +def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, + safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): if search_engine.lower() == "baidu": return search_web_baidu() elif search_engine.lower() == "bing": - return search_web_bing(search_query, search_lang, country, date_range, result_count) + return search_web_bing(search_query, search_lang, content_country, date_range, result_count) elif search_engine.lower() == "brave": - return search_web_brave(search_query, country, search_lang, output_lang, result_count, safesearch, + return search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": - return search_web_duckduckgo(arg1, arg2) + return search_web_duckduckgo(search_query, arg2) elif search_engine.lower() == "google": - return search_web_google(search_query, result_count, results_origin_country, date_range, exactTerms, + return search_web_google(search_query, result_count, content_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, search_result_language, safesearch, site_blacklist, sort_results_by) elif search_engine.lower() == "kagi": - return search_web_kagi(search_query, country, search_lang, output_lang, result_count, safesearch, date_range, + return search_web_kagi(search_query, content_country, search_lang, output_lang, result_count, safesearch, date_range, site_blacklist) elif search_engine.lower() == "serper": return search_web_serper() elif search_engine.lower() == "tavily": return search_web_tavily() elif search_engine.lower() == "searx": - return search_web_searx() + return search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') elif search_engine.lower() == "yandex": return search_web_yandex() else: @@ -121,7 +128,7 @@ def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bi # do config check for default search language setlang = bing_lang - # Country settings + # Returns content for this Country market code if not bing_country: # do config check for default search country bing_country = loaded_config_data['search_engines']['bing_country_code'] @@ -223,9 +230,114 @@ def test_search_brave(): ######################### DuckDuckGo Search ######################### # # https://github.com/deedy5/duckduckgo_search -def search_web_duckduckgo(arg1, arg2) +# Copied request format/structure from https://github.com/deedy5/duckduckgo_search/blob/main/duckduckgo_search/duckduckgo_search.py +def create_session() -> requests.Session: + session = requests.Session() + retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[429, 500, 502, 503, 504]) + session.mount('https://', HTTPAdapter(max_retries=retries)) + return session + +def search_web_duckduckgo( + keywords: str, + region: str = "wt-wt", + timelimit: str | None = None, + max_results: int | None = None, +) -> list[dict[str, str]]: + assert keywords, "keywords is mandatory" + + payload = { + "q": keywords, + "s": "0", + "o": "json", + "api": "d.js", + "vqd": "", + "kl": region, + "bing_market": region, + } + + def _normalize_url(url: str) -> str: + """Unquote URL and replace spaces with '+'.""" + return unquote(url).replace(" ", "+") if url else "" + + def _normalize(raw_html: str) -> str: + """Strip HTML tags from the raw_html string.""" + REGEX_STRIP_TAGS = re.compile("<.*?>") + return unescape(REGEX_STRIP_TAGS.sub("", raw_html)) if raw_html else "" + + if timelimit: + payload["df"] = timelimit + + cache = set() + results: list[dict[str, str]] = [] + + for _ in range(5): + response = requests.post("https://html.duckduckgo.com/html", data=payload) + resp_content = response.content + if b"No results." in resp_content: + return results + + tree = document_fromstring(resp_content) + elements = tree.xpath("//div[h2]") + if not isinstance(elements, list): + return results + + for e in elements: + if isinstance(e, _Element): + hrefxpath = e.xpath("./a/@href") + href = str(hrefxpath[0]) if hrefxpath and isinstance(hrefxpath, list) else None + if ( + href + and href not in cache + and not href.startswith( + ("http://www.google.com/search?q=", "https://duckduckgo.com/y.js?ad_domain") + ) + ): + cache.add(href) + titlexpath = e.xpath("./h2/a/text()") + title = str(titlexpath[0]) if titlexpath and isinstance(titlexpath, list) else "" + bodyxpath = e.xpath("./a//text()") + body = "".join(str(x) for x in bodyxpath) if bodyxpath and isinstance(bodyxpath, list) else "" + results.append( + { + "title": _normalize(title), + "href": _normalize_url(href), + "body": _normalize(body), + } + ) + if max_results and len(results) >= max_results: + return results + + npx = tree.xpath('.//div[@class="nav-link"]') + if not npx or not max_results: + return results + next_page = npx[-1] if isinstance(npx, list) else None + if isinstance(next_page, _Element): + names = next_page.xpath('.//input[@type="hidden"]/@name') + values = next_page.xpath('.//input[@type="hidden"]/@value') + if isinstance(names, list) and isinstance(values, list): + payload = {str(n): str(v) for n, v in zip(names, values)} + return results +def test_search_duckduckgo(): + try: + results = search_web_duckduckgo( + keywords="How can I bake a cherry cake?", + region="us-en", + timelimit="w", + max_results=10 + ) + print(f"Number of results: {len(results)}") + for result in results: + print(f"Title: {result['title']}") + print(f"URL: {result['href']}") + print(f"Snippet: {result['body']}") + print("---") + + except ValueError as e: + print(f"Invalid input: {str(e)}") + except requests.RequestException as e: + print(f"Request error: {str(e)}") ######################### Google Search ######################### From 737bc31b61757f9b190c7af7d3d447c8c8e8458e Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 26 Dec 2024 23:37:36 -0800 Subject: [PATCH 28/88] Update WebSearch_APIs.py --- App_Function_Libraries/Web_Scraping/WebSearch_APIs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 88b2c3c2f..c5eb77cc9 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -42,7 +42,7 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, return search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": - return search_web_duckduckgo(search_query, arg2) + return search_web_duckduckgo(search_query, content_country, date_range, result_count) elif search_engine.lower() == "google": return search_web_google(search_query, result_count, content_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, From 9d78c721c821564cfddc6a367d0138519fffa77e Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 27 Dec 2024 00:01:44 -0800 Subject: [PATCH 29/88] p --- .../Web_Scraping/WebSearch_APIs.py | 2 - Docs/Design/UX.md | 1 + Docs/Design/WebSearch.md | 94 ++++++++++++++++++- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index c5eb77cc9..4d46b2ae4 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -30,8 +30,6 @@ ####################################################################################################################### # - - def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): if search_engine.lower() == "baidu": diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index c35a70bdb..c861658ad 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -53,6 +53,7 @@ https://github.com/woshixiaobai2019/mirau-chat-ui https://uxdesign.cc/the-importance-of-hover-states-c9312d7fd516 https://trends.uxdesign.cc/ https://bolters.io/ +https://github.com/Cloud-Code-AI/AkiraDocs https://github.com/Vali-98/ChatterUI https://www.nngroup.com/videos/efficient-error-messages/ diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index ee6857b23..d07564c9b 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -4,9 +4,75 @@ This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. - ### Setting the Stage -The web search functionality is a core component of the tldw system, allowing users to quickly and easily search the web for information on a given topic. This functionality is designed to be simple and intuitive, providing users with the information they need in a clear and concise manner. +- All the web searches are simple HTTP requests to an API or to the direct endpoint and then scraping the results. +- Parsing results is TODO. +- The goal is to provide a simple, easy-to-use interface for searching the web and retrieving results. +- Other modules are responsible for anything else, this module just performs the search, and delivers the results. +- **Main Function:** + - `def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None)` + - `search_engine` - The search engine to use for the search + - `search_query` - The query to search for + - `content_country` - The country of the content to search for + - `search_lang` - The language to use for the search + - `output_lang` - The language to use for the output + - `result_count` - The number of results to return + - `date_range` - The date range to search within + - `safesearch` - Whether to enable safe search + - `site_blacklist` - A list of sites to exclude from the search results + - `exactTerms` - Terms that must be in the search results + - `excludeTerms` - Terms that must not be in the search results + - `filter` - A filter to apply to the search results + - `geolocation` - The geolocation to use for the search + - `search_result_language` - The language to use for the search results + - `sort_results_by` - How to sort the search results + - **Returns:** A list of search results as a dictionary. - FIXME: Define the structure of the dictionary + - Each result should contain the title, URL, content, and metadata of the search result. + +Search is performed -> Results obtained, each individual item is first analyzed based on snippet, if relevant, entire page is fetched and analyzed, this is then stored in the results dictionary, and the process is repeated until all results are analyzed/limit is hit. +Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. + - + - +Results dictionary: +``` +{ + "search_engine": str, + "search_query": str, + "content_country": str, + "search_lang": str, + "output_lang": str, + "result_count": int, + "date_range": Optional[str], + "safesearch": Optional[bool], + "site_blacklist": Optional[List[str]], + "exactTerms": Optional[List[str]], + "excludeTerms": Optional[List[str]], + "filter": Optional[str], + "geolocation": Optional[str], + "search_result_language": Optional[str], + "sort_results_by": Optional[str], + "results": [ + { + "title": str, + "url": str, + "content": str, + "metadata": { + "date_published": Optional[str], + "author": Optional[str], + "source": Optional[str], + "language": Optional[str], + "relevance_score": Optional[float], + "snippet": Optional[str] + } + }, + # ... more results ... + ], + "total_results_found": int, + "search_time": float, + "error": Optional[str] +} +``` + #### Bing Search - [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) @@ -21,15 +87,35 @@ The web search functionality is a core component of the tldw system, allowing us Two APIs, 1 for 'AI' the other for 'regular' search - #### DuckDuckGo Search -using DuckDuckGo website +Uses query to direct DDG search, then scrape the results. +Structure/approach taken from https://github.com/deedy5/duckduckgo_search + #### Google Search Have to create a custom search engine first, get the ID and then the API key +- Documentation for making requests: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list + + +### Kagi Search +- [Kagi Search API](https://help.kagi.com/kagi/api/search.html) +- Really straightforward, just a simple search API + + +### SearX Search +- [Searx Search Documentation](https://searx.github.io/searx/) +- `SearXNG is a free internet metasearch engine which aggregates results from more than 70 search services. Users are neither tracked nor profiled.` +- Can host your own instance or use someone else's. + + +### Serper Search + +### Tavily Search +### Yandex Search +- https://yandex.cloud/en/docs/search-api/quickstart/ https://github.com/scrapinghub/article-extraction-benchmark From 2f48ffcea77c4d09569eaabab2936e4dd087e6fe Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 27 Dec 2024 17:50:25 -0800 Subject: [PATCH 30/88] F --- App_Function_Libraries/Chat/Chat_Pipeline.py | 19 + .../Prompt_Engineering/Prompt_Engineering.py | 3 + .../Researcher/Research_Manager.py | 24 + App_Function_Libraries/Researcher/__init__.py | 0 App_Function_Libraries/Utils/Utils.py | 5 + .../Web_Scraping/WebSearch_APIs.py | 56 +- Docs/Design/Chunking.md | 25 + Docs/Design/Coding_Page.md | 3 + Docs/Design/Prompts.md | 1 + Docs/Design/Researcher.md | 397 +++++++++++-- Docs/Design/TTS_STT.md | 9 +- Docs/Design/UX.md | 1 + Docs/Design/WebSearch.md | 521 ++++-------------- Docs/Handy_Dandy_Papers.md | 2 +- Docs/Issues/Evaluation_Plans.md | 3 + Docs/RAG_Notes.md | 1 + README.md | 3 +- 17 files changed, 630 insertions(+), 443 deletions(-) create mode 100644 App_Function_Libraries/Chat/Chat_Pipeline.py create mode 100644 App_Function_Libraries/Researcher/Research_Manager.py create mode 100644 App_Function_Libraries/Researcher/__init__.py create mode 100644 Docs/Design/Chunking.md diff --git a/App_Function_Libraries/Chat/Chat_Pipeline.py b/App_Function_Libraries/Chat/Chat_Pipeline.py new file mode 100644 index 000000000..d27ffa81d --- /dev/null +++ b/App_Function_Libraries/Chat/Chat_Pipeline.py @@ -0,0 +1,19 @@ +# Chat_Pipeline.py +# +# Description: This file contains functions related to the prompt modification pipeline, available as a means of +# complex prompt modification/replacement without modification of original intent/messaging. +# +# Imports +import os +# +# 3rd-party Libraries +# +# Local Imports +# +####################################################################################################################### +# +# Functions: + +# +# End of Chat_Pipeline.py +####################################################################################################################### diff --git a/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py b/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py index c037eeadb..b24a3f2b7 100644 --- a/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py +++ b/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py @@ -14,6 +14,9 @@ # # Function Definitions + +################################## Meta Prompt Engineering Functions ############################################## + # Function to generate prompt using metaprompt def generate_prompt(api_endpoint, api_key, task, variables_str, temperature): # Convert variables into a list from comma-separated input diff --git a/App_Function_Libraries/Researcher/Research_Manager.py b/App_Function_Libraries/Researcher/Research_Manager.py new file mode 100644 index 000000000..a31f61ed3 --- /dev/null +++ b/App_Function_Libraries/Researcher/Research_Manager.py @@ -0,0 +1,24 @@ +# Research_Manager.py +# +# Description: This file contains functions related to the research manager functionality +# Handles all functionality related to 'research', including the management of 'research' data, the retrieval of +# 'research' data, and final creation of 'research' reports. +# +# Imports +import os +# +# 3rd-party Libraries +# +# Local Imports +# +######################################################################################################################## +# +# Functions: + + + + + +# +# End of Research_Manager.py +####################################################################################################################### diff --git a/App_Function_Libraries/Researcher/__init__.py b/App_Function_Libraries/Researcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index efa49142e..fefd1f412 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -388,6 +388,8 @@ def load_and_log_configs(): kagi_search_api_key = config.get('Search-Engines', 'search_engine_api_key_kagi', fallback='') # Searx Search Settings search_engine_searx_api = config.get('Search-Engines', 'search_engine_searx_api', fallback='') + # Tavily Search Settings + tavily_search_api_key = config.get('Search-Engines', 'search_engine_api_key_tavily', fallback='') # Yandex Search Settings yandex_search_api_key = config.get('Search-Engines', 'search_engine_api_key_yandex', fallback='') yandex_search_engine_id = config.get('Search-Engines', 'search_engine_id_yandex', fallback='') @@ -532,6 +534,7 @@ def load_and_log_configs(): 'google_enable_site_search' : google_enable_site_search, 'kagi_search_api_key': kagi_search_api_key, 'searx_search_api_url': search_engine_searx_api, + 'tavily_search_api_key': tavily_search_api_key, 'yandex_search_api_key': yandex_search_api_key, 'yandex_search_engine_id': yandex_search_engine_id } @@ -544,6 +547,8 @@ def load_and_log_configs(): global_api_endpoints = ["anthropic", "cohere", "google", "groq", "openai", "huggingface", "openrouter", "deepseek", "mistral", "custom_openai_api", "llama", "ooba", "kobold", "tabby", "vllm", "ollama", "aphrodite"] +global_search_engines = ["baidu", "bing", "brave", "duckduckgo", "google", "kagi", "searx", "tavily", "yandex"] + openai_tts_voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] # Setup Default API Endpoint diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 4d46b2ae4..5c36a6a52 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -16,7 +16,6 @@ from requests import RequestException from requests.adapters import HTTPAdapter from urllib3 import Retry - # # Local Imports from App_Function_Libraries.Utils.Utils import loaded_config_data @@ -26,9 +25,23 @@ # Functions: # 1. perform_websearch # 2. search_web_baidu +# 3. search_web_bing +# 4. search_web_brave +# 5. search_web_duckduckgo +# 6. search_web_google +# 7. search_web_kagi +# 8. search_web_serper +# 9. search_web_tavily +# 10. search_web_searx +# 11. search_web_yandex +# 12. parse_html_search_results_generic # ####################################################################################################################### # +# Functions: + +# FIXME - parsing for results from each search engine +# FIXME - Create results dictionary format/specification def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): @@ -51,7 +64,7 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, elif search_engine.lower() == "serper": return search_web_serper() elif search_engine.lower() == "tavily": - return search_web_tavily() + return search_web_tavily(search_query, result_count, site_blacklist) elif search_engine.lower() == "searx": return search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') elif search_engine.lower() == "yandex": @@ -536,7 +549,6 @@ def test_search_kagi(): # https://searx.space # https://searx.github.io/searx/dev/search_api.html def search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general'): - # Check if API URL is configured searx_url = loaded_config_data['search_engines']['searx_search_api_url'] if not searx_url: @@ -560,7 +572,7 @@ def search_web_searx(search_query, language='auto', time_range='', safesearch=0, # Perform the search request try: headers = { - 'User-Agent': 'anything-llm' + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0' } response = requests.get(search_url, headers=headers) @@ -615,12 +627,42 @@ def test_search_serper(): ######################### Tavily Search ######################### # # https://github.com/YassKhazzan/openperplex_backend_os/blob/main/sources_searcher.py -def search_web_tavily(): - pass +def search_web_tavily(search_query, result_count=10, site_whitelist=None, site_blacklist=None): + # Check if API URL is configured + tavily_api_url = "https://api.tavily.com/search" + + tavily_api_key = loaded_config_data['search_engines']['tavily_search_api_key'] + + # Prepare the request payload + payload = { + "api_key": tavily_api_key, + "query": search_query, + "max_results": result_count + } + + # Add optional parameters if provided + if site_whitelist: + payload["include_domains"] = site_whitelist + if site_blacklist: + payload["exclude_domains"] = site_blacklist + + # Perform the search request + try: + headers = { + 'Content-Type': 'application/json', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0' + } + + response = requests.post(tavily_api_url, headers=headers, data=json.dumps(payload)) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + return f"There was an error searching for content. {str(e)}" def test_search_tavily(): - pass + result = search_web_tavily("How can I bake a cherry cake?") + print(result) ######################### Yandex Search ######################### # diff --git a/Docs/Design/Chunking.md b/Docs/Design/Chunking.md new file mode 100644 index 000000000..f9a4f05de --- /dev/null +++ b/Docs/Design/Chunking.md @@ -0,0 +1,25 @@ +# Chunking + +## Overview +- Chunking is the process of breaking down a document into smaller pieces, or "chunks". This is useful for a variety of reasons, such as: +- + + + + + + +### Types of Chunking + + +### Implementation in tldw +- + + + +### Link Dump: +https://gleen.ai/blog/agentic-chunking-enhancing-rag-answers-for-completeness-and-accuracy/ + + + + diff --git a/Docs/Design/Coding_Page.md b/Docs/Design/Coding_Page.md index 8d00cb69c..061f90337 100644 --- a/Docs/Design/Coding_Page.md +++ b/Docs/Design/Coding_Page.md @@ -15,6 +15,9 @@ https://github.com/abinthomasonline/repo2txt https://github.com/charmandercha/ArchiDoc https://pythontutor.com/c.html#mode=edit https://pythontutor.com/articles/c-cpp-visualizer.html +https://gitingest.com/ +https://gitdiagram.com/ +https://www.ilograph.com/blog/posts/diagrams-ai-can-and-cannot-generate/#system-diagramming-with-ai diff --git a/Docs/Design/Prompts.md b/Docs/Design/Prompts.md index 9120c2656..2a396ba2c 100644 --- a/Docs/Design/Prompts.md +++ b/Docs/Design/Prompts.md @@ -8,3 +8,4 @@ https://github.com/LouisShark/chatgpt_system_prompt https://github.com/microsoft/PromptWizard +https://medium.com/@camauger/crafting-effective-chatgpt-prompts-for-tabletop-roleplaying-games-a-step-by-step-guide-part-1-b81a791d278d \ No newline at end of file diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 0bf37f59d..c1d01cc00 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -1,38 +1,17 @@ # Automated Researcher ## Introduction +- This page is to document efforts towards creating a 'research' agent workflow for use within the project. The goal is to create a system that can automatically generate research reports, summaries, and other research-related tasks. -This page is to document efforts towards creating a 'research' agent workflow for use within the project. The goal is to create a system that can automatically generate research reports, summaries, and other research-related tasks. - -### Link Dump: -https://github.com/assafelovic/gpt-researcher -https://arxiv.org/abs/2411.15114 -https://journaliststudio.google.com/pinpoint/about/ -https://blog.google/products/gemini/google-gemini-deep-research/ -https://github.com/neuml/annotateai -https://pub.towardsai.net/learn-anything-with-ai-and-the-feynman-technique-00a33f6a02bc -https://help.openalex.org/hc/en-us/articles/24396686889751-About-us -https://www.ginkgonotes.com/ -https://www.reddit.com/r/Anki/comments/17u01ge/spaced_repetition_algorithm_a_threeday_journey/ -https://github.com/open-spaced-repetition/fsrs4anki/wiki/Spaced-Repetition-Algorithm:-A-Three%E2%80%90Day-Journey-from-Novice-to-Expert#day-3-the-latest-progress -https://www.scrapingdog.com/blog/scrape-google-news/ -https://github.com/mistralai/cookbook/blob/main/third_party/LlamaIndex/llamaindex_arxiv_agentic_rag.ipynb -https://github.com/ai-christianson/RA.Aid -https://github.com/cbuccella/perplexity_research_prompt/blob/main/general_research_prompt.md -https://github.com/0xeb/TheBigPromptLibrary/blob/main/SystemPrompts/Perplexity.ai/20241024-Perplexity-Desktop-App.md -https://github.com/rashadphz/farfalle -https://github.com/cbuccella/perplexity_research_prompt/blob/main/general_research_prompt.md -https://www.emergentmind.com/ -https://github.com/neuml/paperai -https://github.com/neuml/paperetl -https://github.com/ai-christianson/RA.Aid -https://github.com/Future-House/paper-qa -https://openreview.net/ -https://www.researchrabbit.ai/ -https://github.com/faraz18001/Sales-Llama -https://github.com/memgraph/memgraph - - +### Researcher Goals +1. f +2. f +3. f +4. f +5. f +6. f +7. f +8. ### Ideas @@ -41,8 +20,34 @@ s Follow gptresearchers method at first, planner LLM -> query LLM -> analyzer LLM -> summarizer LLM +### Researcher Workflow + + +### Researcher Components +1. **Query/Search Engine** + - f + - f +2. **Planner** + - f + - f +3. **Analyzer** + - f + - f +4. **(Optional: Summarizer)** + - f + - f +5. **Report Generator** + - f + - f +6. **Knowledge Base Management** + - f + - f + + +### Researcher Config Definitions +- `default_search_engine`: The default search engine to use for queries -Researcher config section +- Researcher config section ``` [researcher] # Researcher settings @@ -77,13 +82,53 @@ search_blacklist_URLs = "URL1,URL2,URL3" ``` + + +### Link Dump: +Articles + https://docs.gptr.dev/blog/gptr-hybrid + https://docs.gptr.dev/docs/gpt-researcher/context/local-docs + https://docs.gptr.dev/docs/gpt-researcher/context/tailored-research# + https://docs.gptr.dev/docs/gpt-researcher/gptr/pip-package + + +https://github.com/assafelovic/gpt-researcher +https://arxiv.org/abs/2411.15114 +https://journaliststudio.google.com/pinpoint/about/ +https://github.com/assafelovic/gpt-researcher/tree/master/gpt_researcher +https://blog.google/products/gemini/google-gemini-deep-research/ +https://github.com/neuml/annotateai +https://docs.gptr.dev/docs/gpt-researcher/multi_agents/langgraph +https://pub.towardsai.net/learn-anything-with-ai-and-the-feynman-technique-00a33f6a02bc +https://help.openalex.org/hc/en-us/articles/24396686889751-About-us +https://www.ginkgonotes.com/ +https://github.com/assafelovic/gpt-researcher/tree/master/multi_agents +https://www.reddit.com/r/Anki/comments/17u01ge/spaced_repetition_algorithm_a_threeday_journey/ +https://github.com/open-spaced-repetition/fsrs4anki/wiki/Spaced-Repetition-Algorithm:-A-Three%E2%80%90Day-Journey-from-Novice-to-Expert#day-3-the-latest-progress +https://www.scrapingdog.com/blog/scrape-google-news/ +https://github.com/mistralai/cookbook/blob/main/third_party/LlamaIndex/llamaindex_arxiv_agentic_rag.ipynb +https://github.com/ai-christianson/RA.Aid +https://github.com/cbuccella/perplexity_research_prompt/blob/main/general_research_prompt.md +https://github.com/0xeb/TheBigPromptLibrary/blob/main/SystemPrompts/Perplexity.ai/20241024-Perplexity-Desktop-App.md +https://github.com/rashadphz/farfalle +https://github.com/cbuccella/perplexity_research_prompt/blob/main/general_research_prompt.md +https://www.emergentmind.com/ +https://github.com/neuml/paperai +https://github.com/neuml/paperetl +https://github.com/ai-christianson/RA.Aid +https://github.com/Future-House/paper-qa +https://openreview.net/ +https://www.researchrabbit.ai/ +https://github.com/faraz18001/Sales-Llama +https://github.com/memgraph/memgraph +https://github.com/rashadphz/farfalle/tree/main/src/backend https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py https://storm.genie.stanford.edu/ https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama -### Researcher Workflow + @@ -104,3 +149,289 @@ I also recommend creating a study guide for each chapter, fully breaking down ke To take this further, incorporate a Personal Knowledge Management (PKM) system into your routine. Apps like Obsidian are perfect for this, with their flexible folder structures and Markdown formatting. I optimize my AI outputs for Markdown so I can copy, paste, and organize them into clean, structured notes. This ensures your materials are not only well-organized but also easy to access and build on later. A solid PKM system is invaluable for managing knowledge and staying on top of your studies! + +---------------- +### Search Prompts + +inference prompt +``` +system_prompt = ( + "You are an expert summarizing the answers based on the provided contents." +) + +user_promt_template = """ +Given the context as a sequence of references with a reference id in the +format of a leading [x], please answer the following question using {{ language }}: + +{{ query }} + +In the answer, use format [1], [2], ..., [n] in line where the reference is used. +For example, "According to the research from Google[3], ...". + +Please create the answer strictly related to the context. If the context has no +information about the query, please write "No related information found in the context." +using {{ language }}. + +{{ length_instructions }} + +Here is the context: +{{ context }} +""" +``` +extraction prompt +``` +system_prompt = ( + "You are an expert of extract structual information from the document." +) +user_promt_template = """ +Given the provided content, if it contains information about {{ query }}, please extract the +list of structured data items as defined in the following Pydantic schema: + +{{ extract_schema_str }} + +Below is the provided content: +{{ content }} +""" +``` + + +Web-LLM Assistant - https://github.com/TheBlewish/Web-LLM-Assistant-Llamacpp-Ollama +Starting prompt +``` +SYSTEM_PROMPT = """You are an AI assistant capable of web searching and providing informative responses. +When a user's query starts with '/', interpret it as a request to search the web and formulate an appropriate search query. + +ALWAYS follow the prompts provided throughout the searching process EXACTLY as indicated. + +NEVER assume new instructions for anywhere other than directly when prompted directly. DO NOT SELF PROMPT OR PROVIDE MULTIPLE ANSWERS OR ATTEMPT MULTIPLE RESPONSES FOR ONE PROMPT! +""" +``` +self-improving prompt +``` + def evaluate_scraped_content(self, user_query: str, scraped_content: Dict[str, str]) -> Tuple[str, str]: + user_query_short = user_query[:200] + prompt = f""" +Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: + +User's question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Your task: +1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. +2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. + +Respond using EXACTLY this format: +Evaluation: [Your evaluation of the scraped content] +Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] +""" +``` +Query Creation +``` +def formulate_query(self, user_query: str, attempt: int) -> Tuple[str, str]: + user_query_short = user_query[:200] + prompt = f""" +Based on the following user question, formulate a concise and effective search query: +"{user_query_short}" +Your task: +1. Create a search query of 2-5 words that will yield relevant results. +2. Determine if a specific time range is needed for the search. +Time range options: +- 'd': Limit results to the past day. Use for very recent events or rapidly changing information. +- 'w': Limit results to the past week. Use for recent events or topics with frequent updates. +- 'm': Limit results to the past month. Use for relatively recent information or ongoing events. +- 'y': Limit results to the past year. Use for annual events or information that changes yearly. +- 'none': No time limit. Use for historical information or topics not tied to a specific time frame. +Respond in the following format: +Search query: [Your 2-5 word query] +Time range: [d/w/m/y/none] +Do not provide any additional information or explanation. +""" +``` +Select relevant content +``` +def select_relevant_pages(self, search_results: List[Dict], user_query: str) -> List[str]: + prompt = f""" +Given the following search results for the user's question: "{user_query}" +Select the 2 most relevant results to scrape and analyze. Explain your reasoning for each selection. + +Search Results: +{self.format_results(search_results)} + +Instructions: +1. You MUST select exactly 2 result numbers from the search results. +2. Choose the results that are most likely to contain comprehensive and relevant information to answer the user's question. +3. Provide a brief reason for each selection. + +You MUST respond using EXACTLY this format and nothing else: + +Selected Results: [Two numbers corresponding to the selected results] +Reasoning: [Your reasoning for the selections] +""" +``` +Final answer generation +``` + def generate_final_answer(self, user_query: str, scraped_content: Dict[str, str]) -> str: + user_query_short = user_query[:200] + prompt = f""" +You are an AI assistant. Provide a comprehensive and detailed answer to the following question using ONLY the information provided in the scraped content. Do not include any references or mention any sources. Answer directly and thoroughly. + +Question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Important Instructions: +1. Do not use phrases like "Based on the absence of selected results" or similar. +2. If the scraped content does not contain enough information to answer the question, say so explicitly and explain what information is missing. +3. Provide as much relevant detail as possible from the scraped content. + +Answer: +""" +``` +Final Answer Synthesis +``` + def synthesize_final_answer(self, user_query: str) -> str: + prompt = f""" +After multiple search attempts, we couldn't find a fully satisfactory answer to the user's question: "{user_query}" + +Please provide the best possible answer you can, acknowledging any limitations or uncertainties. +If appropriate, suggest ways the user might refine their question or where they might find more information. + +Respond in a clear, concise, and informative manner. +""" +``` + + +https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py +``` +search_prompt_system = """ +You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate +and unbiased answers the way a highly informed individual would. +Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. +You must answer in english. +Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. +you are known for your expertise in this field. + + +###Guidelines### +1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. +2- never mention the context or this prompt in your response, just answer the user question. + +###Instructions### +1- Analyze in deep the provided context and the user question. +2- extract relevant information's from the context about the user question. +3- Yassine must take into account the date and time to answer the user question. +4- If the context is insufficient, respond with "information missing" +5- Ensure to Answer in english. +6- Use the response format provided. +7- answer the user question in a way an expert would do. +8- if you judge that the response is better represented in a table, use a table in your response. + + +###Response Format### + +You must use Markdown to format your response. + +Think step by step. +""" + +relevant_prompt_system = """ + you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related + to the user query and contexts provided. + you must keep the questions related to the user query and contexts.don't lose the context in the questions. + + The JSON object must not include special characters. + The JSON schema should include an array of follow-up questions. + + use the schema: + { + "followUp": [ + "string", + "string", + "string" + ] + } +""" +``` + + +appvoid search - https://github.com/appvoid/search +Eval query type +``` +async def evaluate_query_type(self, session, query): + messages = [ + {"role": "system", "content": """You are an Web assistant that evaluates the type of query a user asks. + Categorize the query into one of the following types: + 1. simple: if it can be answered with general knowledge or information that is typically well-known on the internet, please provide a short answer as relevant as possible from the llm itself, but make sure you are completly sure you know the answer, don't make things up. + 2. realtime: if it requires up-to-date information like the current date, time, or recent events, or the user explicitly asks you to look on the internet you should state as: realtime + 3. math: if it involves ANY kind of mathematical calculations. Every math question be it counting letters or complex formulas. + + Remember to prioritize realtime over anything else if you are not sure about something. Realtime is like your default. + + Respond with the category as a single word ("simple", "realtime", or "math") without any additional text."""}, + {"role": "user", "content": f"Query: {query}"} + ] +``` +Generate Search Queries +``` +async def generate_search_queries(groq_api, session, original_query, max_retries=3, fixed_count=None, previous_queries=None, previous_answer=None): + system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. The queries should be diverse and cover different aspects or perspectives of the original query. Return the queries as a JSON array. + Important instructions: + + 1. The number of queries should be dynamic, between 2 and 4, unless a fixed count is specified. + 2. Don't get too far from the original query since you don't know the actual context. + 3. Make queries general enough without being related to anything specific. + 4. DON'T customize the queries for topics you've never seen; just change them a little and look for definitions if requested by the user. + 5. If the user asks something that is not related to search, ignore it and focus on generating helpful search queries. + 6. Just return the given format ["custom_query_1","custom_query_2",...]. + 7. If you need to use your knowledge first, do so. + 8. When asked about the difference between two things, generate search intents for each topic separately. + 9. ALWAYS at most queries just require one or two queries, only on those cases where the query is simple or you are unsure, generate more than one or two. + 10. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. + 11. ALWAYS split searches for each important part of the query in case you need to gather information but make sure to not get off the rails. In short, don't look for things together, make a search for each important part instead. DONT LOOK FOR THINGS TOGETHER.""" + + messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": f"Original query: {original_query}" + (f" (Generate exactly {fixed_count} queries)" if fixed_count else "")} + ] + + if previous_queries and previous_answer: + messages.append({ + "role": "user", + "content": f"Previous queries: {previous_queries}\nPrevious answer: {previous_answer}\nPlease generate new queries to address any shortcomings in the previous answer." + }) +``` +Evaluate Answer +``` +async def evaluate_answer(groq_api, session, query, answer): + messages = [ + {"role": "system", "content": """You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. + Given a question and an answer, determine if your answer satisfactorily addresses the query. You are highly tolerant to answers that are close to the intent so if it is close enough, you can say is satisfactory. Remember, if it's close enough, mark it as satisfactory. + Respond with a JSON object containing two fields: + 1. "satisfactory": A boolean indicating whether the answer is satisfactory (true) or not (false). + 2. "reason": A brief explanation of why your thought is or is not satisfactory. Like "I will keep looking for information since last thought is not addressing the query because..." or "Let look for something different. My last search didn't solve the query. The reason is..." or "I found the right answer so I can ignore this..."."""}, + {"role": "user", "content": f"Query: {query}\nAnswer: {answer}"} + ] +``` +Eval best answer +``` +async def evaluate_best_answer(groq_api, session, query, cached_answers): + print('Answers pool > ', cached_answers) + messages = [ + {"role": "system", "content": """You are an assistant that evaluates multiple answers to a query and selects the best one based on relevance and completeness. + Given a query and a list of answers, choose the answer that best addresses the query. Respond with the best answer. Don't need to mention the word answers at all just be natural. Don't "the best answer" or things like that. Just provide the best one."""}, + {"role": "user", "content": f"Query: {query}\nAnswers: {json.dumps(cached_answers)}"} + ] +``` +Summarization +``` +messages = [ + {"role": "system", "content": """You are a web assistant that helps users find information from web search results. +Given a question and a set of search results, provide a concise response based on the information +available in the search results. If the information is not available in the search results, +state that you don't have enough information to answer the question. You MUST not comment on anything, just follow the instruction. Don't add additional details about anything."""}, + {"role": "user", "content": f"Question: {query}\nSearch Results: {json.dumps(all_results)}"} +] +``` \ No newline at end of file diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index b112f091d..64a5f4733 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -41,6 +41,10 @@ https://github.com/Azure-Samples/aisearch-openai-rag-audio https://www.reddit.com/r/LocalLLaMA/comments/1f0awd6/best_local_open_source_texttospeech_and/ https://github.com/FanaHOVA/smol-podcaster https://docs.inferless.com/cookbook/serverless-customer-service-bot +https://wave-pulse.io/ +https://huggingface.co/spaces/saq1b/podcastgen/blob/main/app.py +https://huggingface.co/spaces/mozilla-ai/document-to-podcast/blob/main/app.py +https://huggingface.co/spaces/Nymbo/Voice-Clone-Multilingual/tree/main https://github.com/aedocw/epub2tts https://github.com/microsoft/SpeechT5 https://github.com/smellslikeml/dolla_llama @@ -50,7 +54,10 @@ https://github.com/smellslikeml/dolla_llama - +Podcastfy + https://github.com/souzatharsis/podcastfy/blob/main/podcastfy/tts/base.py + https://github.com/souzatharsis/podcastfy/blob/main/podcastfy/text_to_speech.py + https://github.com/souzatharsis/podcastfy/blob/main/podcastfy/content_generator.py GLM-4-Voice https://github.com/THUDM/GLM-4-Voice/blob/main/README_en.md diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index c861658ad..733af5b42 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -76,6 +76,7 @@ https://otranscribe.com/ https://www.dive.club/ideas/new-types-of-ui-that-generative-ai-has-necessitated https://interactjs.io/ https://perchance.org/ai-chat +https://uiverse.io/elements https://uxdesign.cc/youre-trying-to-turn-an-oil-tanker-288ed30876e4 https://uxdesign.cc/simple-recall-testing-for-b2b-ux-134e18330da9 https://lovable.dev/ diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index d07564c9b..f817eda02 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -3,7 +3,7 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. - +---------------- ### Setting the Stage - All the web searches are simple HTTP requests to an API or to the direct endpoint and then scraping the results. - Parsing results is TODO. @@ -31,91 +31,6 @@ This page serves as documentation regarding the web search functionality within Search is performed -> Results obtained, each individual item is first analyzed based on snippet, if relevant, entire page is fetched and analyzed, this is then stored in the results dictionary, and the process is repeated until all results are analyzed/limit is hit. Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. - - - - -Results dictionary: -``` -{ - "search_engine": str, - "search_query": str, - "content_country": str, - "search_lang": str, - "output_lang": str, - "result_count": int, - "date_range": Optional[str], - "safesearch": Optional[bool], - "site_blacklist": Optional[List[str]], - "exactTerms": Optional[List[str]], - "excludeTerms": Optional[List[str]], - "filter": Optional[str], - "geolocation": Optional[str], - "search_result_language": Optional[str], - "sort_results_by": Optional[str], - "results": [ - { - "title": str, - "url": str, - "content": str, - "metadata": { - "date_published": Optional[str], - "author": Optional[str], - "source": Optional[str], - "language": Optional[str], - "relevance_score": Optional[float], - "snippet": Optional[str] - } - }, - # ... more results ... - ], - "total_results_found": int, - "search_time": float, - "error": Optional[str] -} -``` - - -#### Bing Search -- [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) -- Getting Started with Bing Search API - - Sign up for a Bing Search API key via Azure (1000 Free Searches a month) - https://www.microsoft.com/en-us/bing/apis/pricing - - Use the Bing Search API to perform searches - Add the generated subscription key to your config.txt file. - - If for some reason you're doing modifications to the code, be aware: https://github.com/Azure-Samples/cognitive-services-REST-api-samples/issues/139 - - Perform searches using Bing! - - -#### Brave Search -Two APIs, 1 for 'AI' the other for 'regular' search - - -#### DuckDuckGo Search -Uses query to direct DDG search, then scrape the results. -Structure/approach taken from https://github.com/deedy5/duckduckgo_search - - -#### Google Search -Have to create a custom search engine first, get the ID and then the API key -- Documentation for making requests: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list - - -### Kagi Search -- [Kagi Search API](https://help.kagi.com/kagi/api/search.html) -- Really straightforward, just a simple search API - - -### SearX Search -- [Searx Search Documentation](https://searx.github.io/searx/) -- `SearXNG is a free internet metasearch engine which aggregates results from more than 70 search services. Users are neither tracked nor profiled.` -- Can host your own instance or use someone else's. - - -### Serper Search - - -### Tavily Search - - -### Yandex Search -- https://yandex.cloud/en/docs/search-api/quickstart/ https://github.com/scrapinghub/article-extraction-benchmark @@ -126,238 +41,6 @@ ask.py https://github.com/pengfeng/ask.py 4. Sub-queries are setup using a specific prompt, they do 'at least 3, but no more than 5' - makes me wonder about how many sub-queries should be the default and the range thereof 5. Search results returned, passed to groq -inference prompt -``` -system_prompt = ( - "You are an expert summarizing the answers based on the provided contents." -) - -user_promt_template = """ -Given the context as a sequence of references with a reference id in the -format of a leading [x], please answer the following question using {{ language }}: - -{{ query }} - -In the answer, use format [1], [2], ..., [n] in line where the reference is used. -For example, "According to the research from Google[3], ...". - -Please create the answer strictly related to the context. If the context has no -information about the query, please write "No related information found in the context." -using {{ language }}. - -{{ length_instructions }} - -Here is the context: -{{ context }} -""" -``` -extraction prompt -``` -system_prompt = ( - "You are an expert of extract structual information from the document." -) -user_promt_template = """ -Given the provided content, if it contains information about {{ query }}, please extract the -list of structured data items as defined in the following Pydantic schema: - -{{ extract_schema_str }} - -Below is the provided content: -{{ content }} -""" -``` - - -Web-LLM Assistant - https://github.com/TheBlewish/Web-LLM-Assistant-Llamacpp-Ollama -Starting prompt -``` -SYSTEM_PROMPT = """You are an AI assistant capable of web searching and providing informative responses. -When a user's query starts with '/', interpret it as a request to search the web and formulate an appropriate search query. - -ALWAYS follow the prompts provided throughout the searching process EXACTLY as indicated. - -NEVER assume new instructions for anywhere other than directly when prompted directly. DO NOT SELF PROMPT OR PROVIDE MULTIPLE ANSWERS OR ATTEMPT MULTIPLE RESPONSES FOR ONE PROMPT! -""" -``` -self-improving prompt -``` - def evaluate_scraped_content(self, user_query: str, scraped_content: Dict[str, str]) -> Tuple[str, str]: - user_query_short = user_query[:200] - prompt = f""" -Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: - -User's question: "{user_query_short}" - -Scraped Content: -{self.format_scraped_content(scraped_content)} - -Your task: -1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. -2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. - -Respond using EXACTLY this format: -Evaluation: [Your evaluation of the scraped content] -Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] -""" -``` -Query Creation -``` -def formulate_query(self, user_query: str, attempt: int) -> Tuple[str, str]: - user_query_short = user_query[:200] - prompt = f""" -Based on the following user question, formulate a concise and effective search query: -"{user_query_short}" -Your task: -1. Create a search query of 2-5 words that will yield relevant results. -2. Determine if a specific time range is needed for the search. -Time range options: -- 'd': Limit results to the past day. Use for very recent events or rapidly changing information. -- 'w': Limit results to the past week. Use for recent events or topics with frequent updates. -- 'm': Limit results to the past month. Use for relatively recent information or ongoing events. -- 'y': Limit results to the past year. Use for annual events or information that changes yearly. -- 'none': No time limit. Use for historical information or topics not tied to a specific time frame. -Respond in the following format: -Search query: [Your 2-5 word query] -Time range: [d/w/m/y/none] -Do not provide any additional information or explanation. -""" -``` -Select relevant content -``` -def select_relevant_pages(self, search_results: List[Dict], user_query: str) -> List[str]: - prompt = f""" -Given the following search results for the user's question: "{user_query}" -Select the 2 most relevant results to scrape and analyze. Explain your reasoning for each selection. - -Search Results: -{self.format_results(search_results)} - -Instructions: -1. You MUST select exactly 2 result numbers from the search results. -2. Choose the results that are most likely to contain comprehensive and relevant information to answer the user's question. -3. Provide a brief reason for each selection. - -You MUST respond using EXACTLY this format and nothing else: - -Selected Results: [Two numbers corresponding to the selected results] -Reasoning: [Your reasoning for the selections] -""" -``` -Final answer generation -``` - def generate_final_answer(self, user_query: str, scraped_content: Dict[str, str]) -> str: - user_query_short = user_query[:200] - prompt = f""" -You are an AI assistant. Provide a comprehensive and detailed answer to the following question using ONLY the information provided in the scraped content. Do not include any references or mention any sources. Answer directly and thoroughly. - -Question: "{user_query_short}" - -Scraped Content: -{self.format_scraped_content(scraped_content)} - -Important Instructions: -1. Do not use phrases like "Based on the absence of selected results" or similar. -2. If the scraped content does not contain enough information to answer the question, say so explicitly and explain what information is missing. -3. Provide as much relevant detail as possible from the scraped content. - -Answer: -""" -``` -Final Answer Synthesis -``` - def synthesize_final_answer(self, user_query: str) -> str: - prompt = f""" -After multiple search attempts, we couldn't find a fully satisfactory answer to the user's question: "{user_query}" - -Please provide the best possible answer you can, acknowledging any limitations or uncertainties. -If appropriate, suggest ways the user might refine their question or where they might find more information. - -Respond in a clear, concise, and informative manner. -""" -``` - - - -appvoid search - https://github.com/appvoid/search -Eval query type -``` -async def evaluate_query_type(self, session, query): - messages = [ - {"role": "system", "content": """You are an Web assistant that evaluates the type of query a user asks. - Categorize the query into one of the following types: - 1. simple: if it can be answered with general knowledge or information that is typically well-known on the internet, please provide a short answer as relevant as possible from the llm itself, but make sure you are completly sure you know the answer, don't make things up. - 2. realtime: if it requires up-to-date information like the current date, time, or recent events, or the user explicitly asks you to look on the internet you should state as: realtime - 3. math: if it involves ANY kind of mathematical calculations. Every math question be it counting letters or complex formulas. - - Remember to prioritize realtime over anything else if you are not sure about something. Realtime is like your default. - - Respond with the category as a single word ("simple", "realtime", or "math") without any additional text."""}, - {"role": "user", "content": f"Query: {query}"} - ] -``` -Generate Search Queries -``` -async def generate_search_queries(groq_api, session, original_query, max_retries=3, fixed_count=None, previous_queries=None, previous_answer=None): - system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. The queries should be diverse and cover different aspects or perspectives of the original query. Return the queries as a JSON array. - Important instructions: - - 1. The number of queries should be dynamic, between 2 and 4, unless a fixed count is specified. - 2. Don't get too far from the original query since you don't know the actual context. - 3. Make queries general enough without being related to anything specific. - 4. DON'T customize the queries for topics you've never seen; just change them a little and look for definitions if requested by the user. - 5. If the user asks something that is not related to search, ignore it and focus on generating helpful search queries. - 6. Just return the given format ["custom_query_1","custom_query_2",...]. - 7. If you need to use your knowledge first, do so. - 8. When asked about the difference between two things, generate search intents for each topic separately. - 9. ALWAYS at most queries just require one or two queries, only on those cases where the query is simple or you are unsure, generate more than one or two. - 10. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. - 11. ALWAYS split searches for each important part of the query in case you need to gather information but make sure to not get off the rails. In short, don't look for things together, make a search for each important part instead. DONT LOOK FOR THINGS TOGETHER.""" - - messages = [ - {"role": "system", "content": system_content}, - {"role": "user", "content": f"Original query: {original_query}" + (f" (Generate exactly {fixed_count} queries)" if fixed_count else "")} - ] - - if previous_queries and previous_answer: - messages.append({ - "role": "user", - "content": f"Previous queries: {previous_queries}\nPrevious answer: {previous_answer}\nPlease generate new queries to address any shortcomings in the previous answer." - }) -``` -Evaluate Answer -``` -async def evaluate_answer(groq_api, session, query, answer): - messages = [ - {"role": "system", "content": """You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. - Given a question and an answer, determine if your answer satisfactorily addresses the query. You are highly tolerant to answers that are close to the intent so if it is close enough, you can say is satisfactory. Remember, if it's close enough, mark it as satisfactory. - Respond with a JSON object containing two fields: - 1. "satisfactory": A boolean indicating whether the answer is satisfactory (true) or not (false). - 2. "reason": A brief explanation of why your thought is or is not satisfactory. Like "I will keep looking for information since last thought is not addressing the query because..." or "Let look for something different. My last search didn't solve the query. The reason is..." or "I found the right answer so I can ignore this..."."""}, - {"role": "user", "content": f"Query: {query}\nAnswer: {answer}"} - ] -``` -Eval best answer -``` -async def evaluate_best_answer(groq_api, session, query, cached_answers): - print('Answers pool > ', cached_answers) - messages = [ - {"role": "system", "content": """You are an assistant that evaluates multiple answers to a query and selects the best one based on relevance and completeness. - Given a query and a list of answers, choose the answer that best addresses the query. Respond with the best answer. Don't need to mention the word answers at all just be natural. Don't "the best answer" or things like that. Just provide the best one."""}, - {"role": "user", "content": f"Query: {query}\nAnswers: {json.dumps(cached_answers)}"} - ] -``` -Summarization -``` -messages = [ - {"role": "system", "content": """You are a web assistant that helps users find information from web search results. -Given a question and a set of search results, provide a concise response based on the information -available in the search results. If the information is not available in the search results, -state that you don't have enough information to answer the question. You MUST not comment on anything, just follow the instruction. Don't add additional details about anything."""}, - {"role": "user", "content": f"Question: {query}\nSearch Results: {json.dumps(all_results)}"} -] -``` - - OpenPerplex 1. Search Query @@ -408,34 +91,6 @@ Perplexity -### Search Engines -- **Google Search** - - [Google Search API](https://developers.google.com/custom-search/v1/overview) - - Setup: - - Setup a `Programmable Search Engine` - - Get the `API Key` - - 100 Search queries per day for free -- **Bing Search** - - [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) - - -- **Yandex Search** - - [Yandex Search API](https://yandex.com/dev/search/) - - -- **Baidu Search** - - [Baidu Search API](https://www.baidu.com/) - - -- **Searx Search** - - [Searx Search API](https://searx.github.io/searx/) - - -- **Kagi** - - [Kagi Search API](https://help.kagi.com/kagi/api/search.html) -- **Brave** - - [Brave Search API](https://brave.com/search/api/) -- **Arxiv** -- **PubMedCentral** - - https://www.ncbi.nlm.nih.gov/home/develop/api/ - - ### Implementaiton - Configuration Options: - Language @@ -489,61 +144,127 @@ Perplexity 11. Search results are then saved to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information 12. This is then searchable via the Media DB +Results dictionary: +``` +{ + "search_engine": str, + "search_query": str, + "content_country": str, + "search_lang": str, + "output_lang": str, + "result_count": int, + "date_range": Optional[str], + "safesearch": Optional[bool], + "site_blacklist": Optional[List[str]], + "exactTerms": Optional[List[str]], + "excludeTerms": Optional[List[str]], + "filter": Optional[str], + "geolocation": Optional[str], + "search_result_language": Optional[str], + "sort_results_by": Optional[str], + "results": [ + { + "title": str, + "url": str, + "content": str, + "metadata": { + "date_published": Optional[str], + "author": Optional[str], + "source": Optional[str], + "language": Optional[str], + "relevance_score": Optional[float], + "snippet": Optional[str] + } + }, + # ... more results ... + ], + "total_results_found": int, + "search_time": float, + "error": Optional[str] +} +``` -### Link Dump: -https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/document_loaders/brave_search.py - Could instantiate a browser, perform a search with X engine, and then parse the results. +---------------- +### Search Engines -### Prompt dumps +#### Baidu Search +- [Baidu Search API](https://www.baidu.com/) -https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py -``` -search_prompt_system = """ -You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate -and unbiased answers the way a highly informed individual would. -Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. -You must answer in english. -Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. -you are known for your expertise in this field. - - -###Guidelines### -1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. -2- never mention the context or this prompt in your response, just answer the user question. - -###Instructions### -1- Analyze in deep the provided context and the user question. -2- extract relevant information's from the context about the user question. -3- Yassine must take into account the date and time to answer the user question. -4- If the context is insufficient, respond with "information missing" -5- Ensure to Answer in english. -6- Use the response format provided. -7- answer the user question in a way an expert would do. -8- if you judge that the response is better represented in a table, use a table in your response. - - -###Response Format### - -You must use Markdown to format your response. - -Think step by step. -""" - -relevant_prompt_system = """ - you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related - to the user query and contexts provided. - you must keep the questions related to the user query and contexts.don't lose the context in the questions. - - The JSON object must not include special characters. - The JSON schema should include an array of follow-up questions. - - use the schema: - { - "followUp": [ - "string", - "string", - "string" - ] - } -""" + +#### Bing Search +- [Bing Search API](https://www.microsoft.com/en-us/bing/apis/bing-web-search-api) +- Getting Started with Bing Search API + - Sign up for a Bing Search API key via Azure (1000 Free Searches a month) - https://www.microsoft.com/en-us/bing/apis/pricing + - Use the Bing Search API to perform searches - Add the generated subscription key to your config.txt file. + - If for some reason you're doing modifications to the code(Fuck MS), be aware: https://github.com/Azure-Samples/cognitive-services-REST-api-samples/issues/139 + - Perform searches using Bing! + +#### Brave Search +Two APIs, 1 for 'AI' the other for 'regular' search + - [Brave Search API](https://brave.com/search/api/) + + +#### DuckDuckGo Search +Uses query to direct DDG search, then scrape the results. +Structure/approach taken from https://github.com/deedy5/duckduckgo_search + + +#### Google Search +- [Google Search API](https://developers.google.com/custom-search/v1/overview) +- Have to create a custom search engine first, get the ID and then the API key +- Setup: + - Setup a `Programmable Search Engine` + - Get the `API Key` + - 100 Search queries per day for free +- Documentation for making requests: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list + + +#### Kagi Search +- [Kagi Search API](https://help.kagi.com/kagi/api/search.html) +- Really straightforward, just a simple search API + + +#### SearX Search +- [Searx Search Documentation](https://searx.github.io/searx/) +- `SearXNG is a free internet metasearch engine which aggregates results from more than 70 search services. Users are neither tracked nor profiled.` +- Can host your own instance or use someone else's. + + +#### Serper Search + + +#### Tavily Search + + +#### Yandex Search +- https://yandex.cloud/en/docs/search-api/quickstart/ +- [Yandex Search API](https://yandex.com/dev/search/) + +#### Arxiv Search + + +#### PubMedCentral Search +- https://www.ncbi.nlm.nih.gov/home/develop/api/ + + + +---------------- +### Web Search API +- TBD +```def perform_websearch(search_engine, + search_query, + content_country, + search_lang, + output_lang, + result_count, + date_range=None, + safesearch=None, + site_blacklist=None, + exactTerms=None, + excludeTerms=None, + filter=None, + geolocation=None, + search_result_language=None, + sort_results_by=None ``` + diff --git a/Docs/Handy_Dandy_Papers.md b/Docs/Handy_Dandy_Papers.md index 8ced292b2..dea511fe7 100644 --- a/Docs/Handy_Dandy_Papers.md +++ b/Docs/Handy_Dandy_Papers.md @@ -40,7 +40,7 @@ https://arxiv.org/abs/2412.06769 - https://arxiv.org/abs/2412.01113 https://huggingface.co/spaces/HuggingFaceH4/blogpost-scaling-test-time-compute - https://machinelearning.apple.com/research/gsm-symbolic - +https://arxiv.org/abs/2402.10200 - ### Test-Time Compute diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 787228071..289f9715a 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -45,6 +45,9 @@ https://pub.towardsai.net/streamline-your-llm-evaluation-a-step-by-step-guide-to https://huggingface.co/QuantFactory/granite-3.1-8b-instruct-GGUF https://huggingface.co/CohereForAI/c4ai-command-r7b-12-2024 https://arxiv.org/abs/2412.17758 +https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard +https://www.atla-ai.com/post/evaluating-the-evaluator +https://hamel.dev/blog/posts/llm-judge/ Have LLMs play Social deception games diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index c22bf7ba2..36a9df810 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -41,6 +41,7 @@ https://towardsdatascience.com/advanced-retrieval-techniques-for-better-rags-c53 https://ai.gopubby.com/agentic-rags-extending-rags-to-sql-databases-1509b25ca3e7 https://ai.plainenglish.io/a-deep-dive-into-golden-retriever-eea3396af3b4 https://github.com/AhmedAl93/multimodal-agentic-RAG +https://hub.athina.ai/athina-originals/end-to-end-implementation-of-unstructured-rag/ GraphRAG diff --git a/README.md b/README.md index 39d06ab79..911b788b3 100644 --- a/README.md +++ b/README.md @@ -189,9 +189,10 @@ All features are designed to run **locally** on your device, ensuring privacy an - Pytorch + other ML libraries will also cause the size to increase. - As such, I would say you want at least 12GB of free space on your system to devote to the app. - Text content itself is tiny, but the supporting libraries + ML models can be quite large. -- **Linux** +- **Linux (Tested on Debian/Ubuntu/Fedora)** 1. Download necessary packages (Python3, ffmpeg, portaudio19-dev - `sudo apt install ffmpeg portaudio19-dev gcc build-essential python3-dev` or `dnf install ffmpeg portaudio19-dev gcc build-essential python3-dev`, Update your GPU Drivers/CUDA drivers if you'll be running an LLM locally) * `portaudio19-dev` for pyaudio, `python3-dev gcc build-essential` for building it. + * If you're using another system, you can try `pip install pyaudio` and see if that works. Otherwise, you can run it as a container. 2. Open a terminal, navigate to the directory you want to install the script in, and run the following commands: 3. `git clone https://github.com/rmusser01/tldw` 4. `cd tldw` From cb63eb910b9078a19beb468432a86392a244217c Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 27 Dec 2024 23:08:33 -0800 Subject: [PATCH 31/88] search parsing + building the chain & prompts --- .../Web_Scraping/Search_Prompt.md | 616 ++++++++++++++++++ .../Web_Scraping/Search_Prompt.py | 342 ---------- .../Web_Scraping/WebSearch_APIs.py | 580 +++++++++++++++-- Docs/Design/Education.md | 21 + Docs/Design/Researcher.md | 331 ++-------- Docs/Design/TTS_STT.md | 6 + Docs/Design/WebSearch.md | 32 +- Server_API/API_README.md | 134 ++++ Server_API/app/main.py | 17 +- 9 files changed, 1401 insertions(+), 678 deletions(-) create mode 100644 App_Function_Libraries/Web_Scraping/Search_Prompt.md delete mode 100644 App_Function_Libraries/Web_Scraping/Search_Prompt.py create mode 100644 Server_API/API_README.md diff --git a/App_Function_Libraries/Web_Scraping/Search_Prompt.md b/App_Function_Libraries/Web_Scraping/Search_Prompt.md new file mode 100644 index 000000000..b4995f2a0 --- /dev/null +++ b/App_Function_Libraries/Web_Scraping/Search_Prompt.md @@ -0,0 +1,616 @@ +# Taken from https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py + + +######################################################################################################################## +Initial Search Prompts +``` +CHAT_PROMPT = """\ +Generate a comprehensive, well-structured, and informative answer for a given question, +using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). +Use an unbiased, journalistic tone, adapting the level of formality to match the user’s question. + +• Cite your statements using [number] notation, placing citations at the end of the relevant sentence. +• Only cite the most relevant results. If multiple sources support the same point, cite all relevant sources [e.g., 1, 2, 3]. +• If sources conflict, present both perspectives clearly and cite the respective sources. +• If different sources refer to different entities with the same name, provide separate answers. +• Do not add any external or fabricated information. +• Do not include URLs or a reference section; cite inline with [number] format only. +• Do not repeat the question or include unnecessary redundancy. +• Use markdown formatting (e.g., **bold**, bullet points, ## headings) to organize the information. +• If the provided results are insufficient to answer the question, explicitly state what information is missing or unclear. + +Structure your answer like this: +1. **Short introduction**: Briefly summarize the topic (1–2 sentences). +2. **Bulleted points**: Present key details, each with appropriate citations. +3. **Conclusion**: Summarize the findings or restate the core answer (with citations if needed). + +Example: +1. **Short introduction**: This topic explores the impact of climate change on agriculture. +2. **Bulleted points**: + - Rising temperatures have reduced crop yields in some regions [1]. + - Changes in rainfall patterns are affecting irrigation practices [2, 3]. +3. **Conclusion**: Climate change poses significant challenges to global agriculture [1, 2, 3]. + + +{my_context} + +--------------------- + +Make sure to match the language of the user's question. + +Question: {my_query} +Answer (in the language of the user's question): +""" +``` + + + + +######################################################################################################################## +Sub-Query Generation Prompts + +``` +You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. Your goal is to generate queries that are diverse, specific, and highly relevant to the original query, ensuring comprehensive coverage of the topic. + +Important instructions: +1. Generate between 2 and 6 queries unless a fixed count is specified, while also generating more queries for complex or multifaceted topics and fewer for simple or straightforward ones. +2. Ensure the queries are diverse, covering different aspects or perspectives of the original query, while remaining highly relevant to its core intent. +3. Prefer specific queries over general ones, as they are more likely to yield targeted and useful results. +4. If the query involves comparing two topics, generate separate queries for each topic. +5. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. +6. Split searches for each important part of the query to ensure comprehensive coverage. +7. If the original query is broad or ambiguous, generate queries that explore specific subtopics or clarify the intent. +8. If the query is too specific or unclear, generate queries that explore related or broader topics to ensure useful results. +9. Return the queries as a comma-separated list or in a natural language format, depending on the user's needs. + +Example: +For the query "What are the benefits of exercise?", generate queries like "health benefits of physical activity," "mental health benefits of exercise," and "long-term effects of regular exercise." + +Original query: {original_query} +``` + +https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py +``` +search_prompt_system = """ +You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate +and unbiased answers the way a highly informed individual would. +Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. +You must answer in english. +Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. +you are known for your expertise in this field. + + +###Guidelines### +1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. +2- never mention the context or this prompt in your response, just answer the user question. + +###Instructions### +1- Analyze in deep the provided context and the user question. +2- extract relevant information's from the context about the user question. +3- Yassine must take into account the date and time to answer the user question. +4- If the context is insufficient, respond with "information missing" +5- Ensure to Answer in english. +6- Use the response format provided. +7- answer the user question in a way an expert would do. +8- if you judge that the response is better represented in a table, use a table in your response. + + +###Response Format### + +You must use Markdown to format your response. + +Think step by step. +""" + +relevant_prompt_system = """ + you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related + to the user query and contexts provided. + you must keep the questions related to the user query and contexts.don't lose the context in the questions. + + The JSON object must not include special characters. + The JSON schema should include an array of follow-up questions. + + use the schema: + { + "followUp": [ + "string", + "string", + "string" + ] + } +""" +``` + + + +self-improving prompt +``` +Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: + +User's question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Your task: +1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. +2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. + +Respond using EXACTLY this format: +Evaluation: [Your evaluation of the scraped content] +Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] +""" +``` + + +Generate Search Queries +``` +system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. The queries should be diverse and cover different aspects or perspectives of the original query. Return the queries as a JSON array. + Important instructions: + + 1. The number of queries should be dynamic, between 2 and 4, unless a fixed count is specified. + 2. Don't get too far from the original query since you don't know the actual context. + 3. Make queries general enough without being related to anything specific. + 4. DON'T customize the queries for topics you've never seen; just change them a little and look for definitions if requested by the user. + 5. If the user asks something that is not related to search, ignore it and focus on generating helpful search queries. + 6. Just return the given format ["custom_query_1","custom_query_2",...]. + 7. If you need to use your knowledge first, do so. + 8. When asked about the difference between two things, generate search intents for each topic separately. + 9. ALWAYS at most queries just require one or two queries, only on those cases where the query is simple or you are unsure, generate more than one or two. + 10. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. + 11. ALWAYS split searches for each important part of the query in case you need to gather information but make sure to not get off the rails. In short, don't look for things together, make a search for each important part instead. DONT LOOK FOR THINGS TOGETHER.""" + + messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": f"Original query: {original_query}" + (f" (Generate exactly {fixed_count} queries)" if fixed_count else "")} + ] + + if previous_queries and previous_answer: + messages.append({ + "role": "user", + "content": f"Previous queries: {previous_queries}\nPrevious answer: {previous_answer}\nPlease generate new queries to address any shortcomings in the previous answer." + }) +``` + + + + + +######################################################################################################################## +Content Extraction Prompts + +``` +system_prompt = ( + "You are an expert of extract structual information from the document." +) +user_promt_template = """ +Given the provided content, if it contains information about {{ query }}, please extract the +list of structured data items as defined in the following Pydantic schema: + +{{ extract_schema_str }} + +Below is the provided content: +{{ content }} +""" +``` + +### Eval + +Evaluate Answer +``` +messages = [ + {"role": "system", "content": """You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. + Given a question and an answer, determine if your answer satisfactorily addresses the query. You are highly tolerant to answers that are close to the intent so if it is close enough, you can say is satisfactory. Remember, if it's close enough, mark it as satisfactory. + Respond with a JSON object containing two fields: + 1. "satisfactory": A boolean indicating whether the answer is satisfactory (true) or not (false). + 2. "reason": A brief explanation of why your thought is or is not satisfactory. Like "I will keep looking for information since last thought is not addressing the query because..." or "Let look for something different. My last search didn't solve the query. The reason is..." or "I found the right answer so I can ignore this..."."""}, + {"role": "user", "content": f"Query: {query}\nAnswer: {answer}"} + ] +``` +Eval best answer +``` +messages = [ + {"role": "system", "content": """You are an assistant that evaluates multiple answers to a query and selects the best one based on relevance and completeness. + Given a query and a list of answers, choose the answer that best addresses the query. Respond with the best answer. Don't need to mention the word answers at all just be natural. Don't "the best answer" or things like that. Just provide the best one."""}, + {"role": "user", "content": f"Query: {query}\nAnswers: {json.dumps(cached_answers)}"} + ] +``` + + +Select relevant content +``` +Given the following search results for the user's question: "{user_query}" +Select the 2 most relevant results to scrape and analyze. Explain your reasoning for each selection. + +Search Results: +{self.format_results(search_results)} + +Instructions: +1. You MUST select exactly 2 result numbers from the search results. +2. Choose the results that are most likely to contain comprehensive and relevant information to answer the user's question. +3. Provide a brief reason for each selection. + +You MUST respond using EXACTLY this format and nothing else: + +Selected Results: [Two numbers corresponding to the selected results] +Reasoning: [Your reasoning for the selections] +""" +``` + + +######################################################################################################################## +Final Answer Generation Prompts + + +Final-Answer-1 +``` +You are an AI assistant. Provide a comprehensive and detailed answer to the following question using ONLY the information provided in the scraped content. Do not include any references or mention any sources unless explicitly instructed. Answer directly and thoroughly, using a clear and professional tone. + +Question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Important Instructions: +1. Structure your answer as follows: + - **Introduction**: Briefly summarize the topic or main point (1–2 sentences). + - **Details**: Provide key information, facts, or insights from the scraped content. Use bullet points or paragraphs for clarity. + - **Conclusion**: Summarize the findings or restate the core answer (1–2 sentences). +2. Adapt the tone and style of the answer to match the user’s question. Use a formal tone for technical or professional queries and a conversational tone for casual questions. +3. If the scraped content contains conflicting information, present both perspectives clearly and neutrally, noting the discrepancy. +4. Focus on the most relevant and important information in the scraped content, and avoid including minor or tangential details. +5. If the scraped content does not contain enough information to answer the question, say so explicitly and explain what information is missing. +6. Provide as much relevant detail as possible from the scraped content, but avoid redundancy or unnecessary repetition. +7. If the question is ambiguous or overly broad, clarify the intent or focus on specific subtopics to provide a more targeted answer. +8. Avoid generating content that is discriminatory, offensive, or harmful. If the topic is sensitive, provide a neutral and respectful response. +9. If the user specifies a preferred format (e.g., bullet points, paragraphs) or level of detail (e.g., brief, comprehensive), tailor the answer accordingly. +10. If the user requests revisions, adjust the answer based on their feedback while adhering to the above guidelines. + +Examples: +1. Short Answer (3–4 Sentences) + - **Question:** "What is photosynthesis?" + - **Answer:** + - **Introduction:** Photosynthesis is the process by which plants convert sunlight into energy. + - **Details:** + * It occurs in the chloroplasts of plant cells, using chlorophyll to absorb light. + * During photosynthesis, plants take in carbon dioxide and release oxygen as a byproduct. + - **Conclusion:** This process is essential for plant growth and oxygen production, supporting life on Earth. +2. Medium Answer (5–8 Sentences) + - **Question:** "What are the benefits of exercise?" + - **Answer:** + - **Introduction:** Exercise offers numerous physical and mental health benefits. + - **Details:** + * It improves cardiovascular health by strengthening the heart and improving circulation. + * Regular exercise helps maintain a healthy weight and reduces the risk of chronic diseases like diabetes. + * It also enhances mental health by reducing stress, anxiety, and depression through the release of endorphins. + * Exercise can improve sleep quality and boost overall energy levels. + - **Conclusion:** Incorporating regular exercise into your routine is essential for long-term physical and mental well-being. +3. Long Answer (9–12 Sentences) + - **Question**: "What are the causes and effects of climate change?" + - **Answer:** + - **Introduction**: Climate change refers to long-term changes in temperature and weather patterns, primarily caused by human activities. + - **Details:** + * The main cause is the increase in greenhouse gases, such as carbon dioxide and methane, from burning fossil fuels, deforestation, and industrial processes. + * These gases trap heat in the atmosphere, leading to a rise in global temperatures. + * Effects of climate change include more frequent and severe weather events, such as hurricanes, droughts, and heatwaves. + * Melting polar ice caps and glaciers contribute to rising sea levels, threatening coastal communities. + * Changes in precipitation patterns affect agriculture, leading to food and water shortages in some regions. + * Ecosystems are disrupted, causing species extinction and loss of biodiversity. + * Climate change also has economic impacts, such as increased costs for disaster recovery and healthcare. + - **Conclusion:** Addressing climate change requires global cooperation, sustainable practices, and a transition to renewable energy sources. +4. Very Long Answer (13–20 Sentences) + - **Question:** "What are the pros and cons of remote work?" + - **Answer:** + - **Introduction**: Remote work has become increasingly popular, offering both advantages and disadvantages for employees and employers. + - **Details:** + - **Pros for Employees:** + * Increased flexibility allows employees to manage their schedules and achieve better work-life balance. + * Eliminating commuting saves time and reduces transportation costs. + * Remote work can reduce stress by providing a more comfortable and personalized work environment. + * Employees have the opportunity to live in locations with a lower cost of living or closer to family. + - **Pros for Employers:** + * Remote work can reduce overhead costs, such as office space and utilities. + * Employers can access a global talent pool, increasing diversity and expertise. + * Studies show that remote workers are often more productive due to fewer office distractions. + * Offering remote work can improve employee satisfaction and retention. + - **Cons for Employees:** + * Remote work can lead to feelings of isolation and reduced team cohesion. + * Blurred boundaries between work and personal life may result in longer working hours. + * Limited access to office resources and face-to-face collaboration can hinder creativity and problem-solving. + - **Cons for Employers:** + * Managing remote teams requires robust communication tools and strategies. + * Ensuring data security and compliance can be more challenging in a remote setup. + * Remote work may reduce opportunities for spontaneous collaboration and innovation. + - **Conclusion:** While remote work offers significant benefits, it also presents challenges that require careful management to maximize its potential. + +Answer: +``` + + +Final-Answer-2 +``` +system_prompt = ( + "You are an eagle-eyed researcher, skilled at summarizing lengthy documents with precision and clarity. " + "Your task is to create a comprehensive summary of the provided document, capturing the main ideas, key details, and essential arguments presented. " + "Use the provided context to answer the user's question, and always reference your sources clearly." +) + +user_prompt_template = """ +**Task:** +Create a detailed summary of the provided document to answer the following question using {{ language }}: + +{{ query }} + +**Instructions:** + +1. **Source Referencing:** + - Use the format [1], [2], ..., [n] to reference sources in line with the text. For example, "According to the research from Google[3], ...". + - Ensure all claims, data, or examples are backed by a reference from the provided context. + +2. **Structure:** + - Organize the summary logically using clear headings and subheadings for different sections or themes. + - Include the following sections: + - **Introduction:** Briefly introduce the main theme or purpose of the document. + - **Key Points:** Summarize the main arguments, ideas, or findings. + - **Supporting Details:** Include key data, examples, or evidence that strengthen the main points. + - **Conclusion:** Succinctly encapsulate the overarching message or significance of the document. + +3. **Tone:** + - Use an objective, neutral tone, delivering precise and insightful analysis without personal opinions or interpretations. + +4. **Length and Depth:** + - Adjust the length of the summary based on the complexity and depth of the document. + - Ensure the summary is comprehensive without omitting crucial information. + +5. **Contextual Awareness:** + - If the context does not contain information relevant to the query, state: "No related information found in the context." using {{ language }}. + +**Example Output:** + +**Introduction to Main Theme** +The document begins by discussing [main idea], outlining [initial point] with supporting data like [example][1]. + +**Key Points** +The text presents several main arguments, such as [supporting detail][2]. Notably, [data or statistic][3] is used to reinforce the main concept. + +**Supporting Details** +Additional evidence includes [example][4], which highlights [key point][5]. + +**Conclusion** +In summary, [document's conclusion statement][6], highlighting the broader implications like [significance][7]. + +**Context:** +{{ context }} +""" +``` + + + + +Perplexity Samples +``` +Example 1: Technology + +- **Query:** What are TPUs? +- **Answer:** + * TPUs (Tensor Processing Units) are specialized hardware accelerators developed by Google for machine learning (ML) and artificial intelligence (AI) workloads. They are designed to optimize the performance of tensor operations, which are fundamental to neural network computations. + - **Key Features: + * **Architecture:** TPUs are built on Google’s custom ASIC (Application-Specific Integrated Circuit) technology, which is optimized for high-speed matrix multiplications and other tensor operations. + * **Performance:** TPUs offer significantly faster processing speeds for ML tasks compared to traditional CPUs and GPUs. For example, a single TPU v4 pod can deliver over 1 exaflop of computing power. + * **Scalability:** TPUs are designed to work in clusters, allowing for scalable performance across large datasets and complex models. + * **Energy Efficiency:** TPUs are highly energy-efficient, reducing the power consumption required for large-scale ML training and inference. + - **Applications:** + * Training large-scale neural networks, such as those used in natural language processing (NLP) and computer vision. + * Accelerating inference tasks for real-time applications like image recognition and language translation. + * Supporting research and development in AI and ML by providing high-performance computing resources. + - By offering unparalleled speed and efficiency for tensor operations, TPUs have become a critical tool for advancing AI and ML technologies. + + +Example 2: Comparative Analysis + +- **Query:** How does the TPU v4 compare to the TPU v3 in terms of performance? +- **Answer:** + - The TPU v4 represents a significant performance improvement over the TPU v3, with advancements in speed, scalability, and efficiency. + - **Performance Improvements:** + - **Compute Power: + The TPU v4 offers up to 2.7 times the peak compute performance of the TPU v3, achieving over 275 teraflops per chip. + For large-scale ML models, the TPU v4 demonstrates up to 3x faster training times compared to the TPU v3. + - **Memory and Bandwidth: + The TPU v4 features 32GB of high-bandwidth memory (HBM2), providing 1.5 times the memory capacity of the TPU v3. + Memory bandwidth has also increased to 1.2 TB/s, enabling faster data access and processing. + - **Scalability: + TPU v4 pods can scale up to 4,096 chips, delivering over 1 exaflop of computing power. This is a significant improvement over the TPU v3’s maximum pod size of 1,024 chips. + - **Energy Efficiency: + The TPU v4 is designed to be more energy-efficient, reducing power consumption per operation by up to 20% compared to the TPU v3. + - **Architectural Advancements: + - **Interconnect Technology: + * The TPU v4 uses Google’s next-generation interconnect technology, which provides higher bandwidth and lower latency for communication between chips. + - **Software Support: + * The TPU v4 is fully compatible with TensorFlow and JAX, with optimizations for newer ML frameworks and algorithms. + - **Applications: + * Training state-of-the-art AI models, such as GPT-4 and other large language models. + * Accelerating inference for real-time applications like autonomous driving and recommendation systems. + * Supporting scientific research in areas like climate modeling and drug discovery. + - **The TPU v4’s advancements in performance, scalability, and efficiency make it a powerful upgrade over the TPU v3 for demanding AI and ML workloads. + + +Example 3: Consumer Electronics +- **Query:** What are the key features of the iPhone 15 Pro? +- **Answer:** + * The iPhone 15 Pro is Apple’s latest flagship smartphone, offering a range of advanced features and improvements over its predecessors. + - **Key Features:** + - **Design:** + * The iPhone 15 Pro features a titanium frame, making it lighter and more durable than previous models. + * It retains the Ceramic Shield front cover for enhanced drop protection. + - **Display:** + * The 6.1-inch Super Retina XDR display supports ProMotion technology with a 120Hz refresh rate for smoother scrolling and responsiveness. + * It also features Always-On Display functionality, allowing users to view key information without waking the phone. + - **Performance:** + * Powered by the A17 Bionic chip, the iPhone 15 Pro delivers up to 20% faster CPU and GPU performance compared to the A16 Bionic. + * The new 6-core GPU supports hardware-accelerated ray tracing for enhanced gaming and graphics. + - **Camera System:** + * The triple-camera system includes a 48MP main sensor, a 12MP ultra-wide lens, and a 12MP telephoto lens with 5x optical zoom. + * New computational photography features, such as Photonic Engine and Smart HDR 5, improve low-light performance and dynamic range. + **Battery Life:** + * The iPhone 15 Pro offers up to 22 hours of video playback, thanks to its larger battery and energy-efficient A17 Bionic chip. + **Connectivity:** + * It supports Wi-Fi 6E and Bluetooth 5.3 for faster wireless connectivity. + * The USB-C port replaces Lightning, offering faster data transfer speeds and compatibility with a wider range of accessories. + - **Additional Features:** + * Action Button: A customizable button replaces the traditional mute switch, allowing users to assign shortcuts for quick access to frequently used functions. + * iOS 17: The iPhone 15 Pro comes with the latest version of iOS, featuring new customization options, enhanced privacy controls, and improved messaging capabilities. + - **The iPhone 15 Pro combines cutting-edge technology with a sleek design, making it a top choice for consumers seeking a premium smartphone experience.** + + +Example 4: Comparative Analysis +- **Query:** How does the iPhone 15 Pro compare to the iPhone 14 Pro in terms of performance? +- **Answer:** + * The iPhone 15 Pro offers several performance improvements over the iPhone 14 Pro, making it a significant upgrade for users. + - **Performance Improvements:** + + - **Processor:** + * The iPhone 15 Pro is powered by the A17 Bionic chip, which delivers up to 20% faster CPU and GPU performance compared to the A16 Bionic in the iPhone 14 Pro. + * The A17 Bionic also features a 6-core GPU with hardware-accelerated ray tracing, enhancing gaming and graphics performance. + - **Memory and Storage:** + * The iPhone 15 Pro starts with 8GB of RAM, compared to 6GB in the iPhone 14 Pro, allowing for better multitasking and app performance. + * Storage options remain the same, ranging from 128GB to 1TB. + - **Battery Life:** + * The iPhone 15 Pro offers up to 22 hours of video playback, a slight improvement over the iPhone 14 Pro’s 20 hours. + * The A17 Bionic’s energy efficiency contributes to longer battery life under heavy usage. + - **Design and Features:** + - Build Material: + * The iPhone 15 Pro features a titanium frame, making it lighter and more durable than the stainless steel frame of the iPhone 14 Pro. + - Display: + * Both models feature a 6.1-inch Super Retina XDR display with ProMotion, but the iPhone 15 Pro introduces Always-On Display functionality. + - Camera System: + * The iPhone 15 Pro’s 48MP main sensor and 5x optical zoom offer improved photography capabilities compared to the iPhone 14 Pro’s 3x optical zoom. + - Connectivity: + * The iPhone 15 Pro replaces the Lightning port with USB-C, offering faster data transfer speeds and broader compatibility. + - **Software:** + * Both models run iOS 17, but the iPhone 15 Pro’s hardware optimizations ensure smoother performance and better support for new features. + - **The iPhone 15 Pro’s advancements in processing power, design, and camera technology make it a compelling upgrade over the iPhone 14 Pro.** +``` + + + + + +######################################################################################################################## +Failed Search Query Prompt +``` +After multiple search attempts, we couldn't find a fully satisfactory answer to the user's question: "{user_query}" + +Please provide the best possible answer you can, acknowledging any limitations or uncertainties. +If appropriate, suggest ways the user might refine their question or where they might find more information. + +Respond in a clear, concise, and informative manner. +""" +``` + + + + +######################################################################################################################## +Related Questions Prompts + +``` +RELATED_QUESTION_PROMPT = """\ +Given a user's question and the context from search results, generate exactly 3 concise and relevant follow-up questions the user might ask. + +**Instructions:** +1. **Relevance:** Ensure the questions are directly related to the original question and context. +2. **Conciseness:** Keep the questions short and simple. +3. **Language Match:** Use the same language as the user's original question. +4. **Depth:** Include questions that explore different aspects of the topic (e.g., clarification, deeper exploration, or related subtopics). + +**Original Question:** {query} + +**Context:** + +{context} + + +**Output Format:** +related_questions: A list of EXACTLY three concise, simple, and relevant follow-up questions. + +**Example:** +related_questions: [ + "What are the benefits of renewable energy?", + "How does solar energy reduce greenhouse gas emissions?", + "What are the costs of implementing wind energy?" +] +""" +``` + +``` +HISTORY_QUERY_REPHRASE = """\ +Given a conversation history and a follow-up input, rephrase the follow-up into a SHORT, standalone query that captures relevant context from previous messages. + +**Instructions:** +1. **Conciseness:** Make the query as short and compressed as possible. +2. **Relevance:** Include only information relevant to the retrieval task. +3. **Topic Change:** If there is a clear change in topic, disregard the previous messages and focus only on the follow-up input. +4. **Language Match:** Use the same language as the user's original question. + +**Chat History:** +{chat_history} + +**Follow-Up Input:** +{question} + +**Output Format:** +Respond with ONLY the short, standalone query. + +**Example:** +Follow-Up Input: "What about the costs?" +Standalone Query: "Costs of implementing renewable energy" +""" +``` + + +Query Plan Prompt +``` +QUERY_PLAN_PROMPT = """\ +You are an expert at creating search task lists to answer queries. Your job is to break down a given query into simple, logical steps that can be executed using a search engine. + +Rules: +1. Use up to 4 steps maximum, but use fewer if possible. +2. Keep steps simple, concise, and easy to understand. +3. Ensure proper use of dependencies between steps. +4. Always include a final step to summarize/combine/compare information from previous steps. + +Instructions for creating the Query Plan: +1. Break down the query into logical search steps. +2. For each step, specify an "id" (starting from 0) and a "step" description. +3. List dependencies for each step as an array of previous step ids. +4. The first step should always have an empty dependencies array. +5. Subsequent steps should list all step ids they depend on. + +Example Query: +Given the query "Compare Perplexity and You.com in terms of revenue, number of employees, and valuation" + +Example Query Plan: +[ + {{ + "id": 0, + "step": "Research Perplexity's revenue, employee count, and valuation", + "dependencies": [] + }}, + {{ + "id": 1, + "step": "Research You.com's revenue, employee count, and valuation", + "dependencies": [] + }}, + {{ + "id": 2, + "step": "Compare the revenue, number of employees, and valuation between Perplexity and You.com", + "dependencies": [0, 1] + }} +] + +Query: {query} +Query Plan (with a final summarize/combine/compare step): +""" +``` + + diff --git a/App_Function_Libraries/Web_Scraping/Search_Prompt.py b/App_Function_Libraries/Web_Scraping/Search_Prompt.py deleted file mode 100644 index 92ca6f169..000000000 --- a/App_Function_Libraries/Web_Scraping/Search_Prompt.py +++ /dev/null @@ -1,342 +0,0 @@ -# Taken from https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py -CHAT_PROMPT = """\ -Generate a comprehensive and informative answer for a given question solely based on the provided web Search Results (URL, Page Title, Summary). You must only use information from the provided search results. Use an unbiased and journalistic tone. - -You must cite the answer using [number] notation. You must cite sentences with their relevant citation number. Cite every part of the answer. -Place citations at the end of the sentence. You can do multiple citations in a row with the format [number1][number2]. - -Only cite the most relevant results that answer the question accurately. If different results refer to different entities with the same name, write separate answers for each entity. - -ONLY cite inline. -DO NOT include a reference section, DO NOT include URLs. -DO NOT repeat the question. - - -You can use markdown formatting. You should include bullets to list the information in your answer. - - -{my_context} - ---------------------- - -Make sure to match the language of the user's question. - -Question: {my_query} -Answer (in the language of the user's question): \ -""" - -RELATED_QUESTION_PROMPT = """\ -Given a question and search result context, generate 3 follow-up questions the user might ask. Use the original question and context. - -Instructions: -- Generate exactly 3 questions. -- These questions should be concise, and simple. -- Ensure the follow-up questions are relevant to the original question and context. -Make sure to match the language of the user's question. - -Original Question: {query} - -{context} - - -Output: -related_questions: A list of EXACTLY three concise, simple follow-up questions -""" - -HISTORY_QUERY_REPHRASE = """ -Given the following conversation and a follow up input, rephrase the follow up into a SHORT, \ -standalone query (which captures any relevant context from previous messages). -IMPORTANT: EDIT THE QUERY TO BE CONCISE. Respond with a short, compressed phrase. \ -If there is a clear change in topic, disregard the previous messages. -Strip out any information that is not relevant for the retrieval task. - -Chat History: -{chat_history} - -Make sure to match the language of the user's question. - -Follow Up Input: {question} -Standalone question (Respond with only the short combined query): -""".strip() - - -QUERY_PLAN_PROMPT = """\ -You are an expert at creating search task lists to answer queries. Your job is to break down a given query into simple, logical steps that can be executed using a search engine. - -Rules: -1. Use up to 4 steps maximum, but use fewer if possible. -2. Keep steps simple, concise, and easy to understand. -3. Ensure proper use of dependencies between steps. -4. Always include a final step to summarize/combine/compare information from previous steps. - -Instructions for creating the Query Plan: -1. Break down the query into logical search steps. -2. For each step, specify an "id" (starting from 0) and a "step" description. -3. List dependencies for each step as an array of previous step ids. -4. The first step should always have an empty dependencies array. -5. Subsequent steps should list all step ids they depend on. - -Example Query: -Given the query "Compare Perplexity and You.com in terms of revenue, number of employees, and valuation" - -Example Query Plan: -[ - {{ - "id": 0, - "step": "Research Perplexity's revenue, employee count, and valuation", - "dependencies": [] - }}, - {{ - "id": 1, - "step": "Research You.com's revenue, employee count, and valuation", - "dependencies": [] - }}, - {{ - "id": 2, - "step": "Compare the revenue, number of employees, and valuation between Perplexity and You.com", - "dependencies": [0, 1] - }} -] - -Query: {query} -Query Plan (with a final summarize/combine/compare step): -""" - -SEARCH_QUERY_PROMPT = """\ -Generate a concise list of search queries to gather information for executing the given step. - -You will be provided with: -1. A specific step to execute -2. The user's original query -3. Context from previous steps (if available) - -Use this information to create targeted search queries that will help complete the current step effectively. Aim for the minimum number of queries necessary while ensuring they cover all aspects of the step. - -IMPORTANT: Always incorporate relevant information from previous steps into your queries. This ensures continuity and builds upon already gathered information. - -Input: ---- -User's original query: {user_query} ---- -Context from previous steps: -{prev_steps_context} - -Your task: -1. Analyze the current step and its requirements -2. Consider the user's original query and any relevant previous context -3. Consider the user's original query -4. Generate a list of specific, focused search queries that: - - Incorporate relevant information from previous steps - - Address the requirements of the current step - - Build upon the information already gathered ---- -Current step to execute: {current_step} ---- - -Your search queries based: -""" - - -# License Reproduction: -# Apache License -# Version 2.0, January 2004 -# http://www.apache.org/licenses/ -# -# TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -# -# 1. Definitions. -# -# "License" shall mean the terms and conditions for use, reproduction, -# and distribution as defined by Sections 1 through 9 of this document. -# -# "Licensor" shall mean the copyright owner or entity authorized by -# the copyright owner that is granting the License. -# -# "Legal Entity" shall mean the union of the acting entity and all -# other entities that control, are controlled by, or are under common -# control with that entity. For the purposes of this definition, -# "control" means (i) the power, direct or indirect, to cause the -# direction or management of such entity, whether by contract or -# otherwise, or (ii) ownership of fifty percent (50%) or more of the -# outstanding shares, or (iii) beneficial ownership of such entity. -# -# "You" (or "Your") shall mean an individual or Legal Entity -# exercising permissions granted by this License. -# -# "Source" form shall mean the preferred form for making modifications, -# including but not limited to software source code, documentation -# source, and configuration files. -# -# "Object" form shall mean any form resulting from mechanical -# transformation or translation of a Source form, including but -# not limited to compiled object code, generated documentation, -# and conversions to other media types. -# -# "Work" shall mean the work of authorship, whether in Source or -# Object form, made available under the License, as indicated by a -# copyright notice that is included in or attached to the work -# (an example is provided in the Appendix below). -# -# "Derivative Works" shall mean any work, whether in Source or Object -# form, that is based on (or derived from) the Work and for which the -# editorial revisions, annotations, elaborations, or other modifications -# represent, as a whole, an original work of authorship. For the purposes -# of this License, Derivative Works shall not include works that remain -# separable from, or merely link (or bind by name) to the interfaces of, -# the Work and Derivative Works thereof. -# -# "Contribution" shall mean any work of authorship, including -# the original version of the Work and any modifications or additions -# to that Work or Derivative Works thereof, that is intentionally -# submitted to Licensor for inclusion in the Work by the copyright owner -# or by an individual or Legal Entity authorized to submit on behalf of -# the copyright owner. For the purposes of this definition, "submitted" -# means any form of electronic, verbal, or written communication sent -# to the Licensor or its representatives, including but not limited to -# communication on electronic mailing lists, source code control systems, -# and issue tracking systems that are managed by, or on behalf of, the -# Licensor for the purpose of discussing and improving the Work, but -# excluding communication that is conspicuously marked or otherwise -# designated in writing by the copyright owner as "Not a Contribution." -# -# "Contributor" shall mean Licensor and any individual or Legal Entity -# on behalf of whom a Contribution has been received by Licensor and -# subsequently incorporated within the Work. -# -# 2. Grant of Copyright License. Subject to the terms and conditions of -# this License, each Contributor hereby grants to You a perpetual, -# worldwide, non-exclusive, no-charge, royalty-free, irrevocable -# copyright license to reproduce, prepare Derivative Works of, -# publicly display, publicly perform, sublicense, and distribute the -# Work and such Derivative Works in Source or Object form. -# -# 3. Grant of Patent License. Subject to the terms and conditions of -# this License, each Contributor hereby grants to You a perpetual, -# worldwide, non-exclusive, no-charge, royalty-free, irrevocable -# (except as stated in this section) patent license to make, have made, -# use, offer to sell, sell, import, and otherwise transfer the Work, -# where such license applies only to those patent claims licensable -# by such Contributor that are necessarily infringed by their -# Contribution(s) alone or by combination of their Contribution(s) -# with the Work to which such Contribution(s) was submitted. If You -# institute patent litigation against any entity (including a -# cross-claim or counterclaim in a lawsuit) alleging that the Work -# or a Contribution incorporated within the Work constitutes direct -# or contributory patent infringement, then any patent licenses -# granted to You under this License for that Work shall terminate -# as of the date such litigation is filed. -# -# 4. Redistribution. You may reproduce and distribute copies of the -# Work or Derivative Works thereof in any medium, with or without -# modifications, and in Source or Object form, provided that You -# meet the following conditions: -# -# (a) You must give any other recipients of the Work or -# Derivative Works a copy of this License; and -# -# (b) You must cause any modified files to carry prominent notices -# stating that You changed the files; and -# -# (c) You must retain, in the Source form of any Derivative Works -# that You distribute, all copyright, patent, trademark, and -# attribution notices from the Source form of the Work, -# excluding those notices that do not pertain to any part of -# the Derivative Works; and -# -# (d) If the Work includes a "NOTICE" text file as part of its -# distribution, then any Derivative Works that You distribute must -# include a readable copy of the attribution notices contained -# within such NOTICE file, excluding those notices that do not -# pertain to any part of the Derivative Works, in at least one -# of the following places: within a NOTICE text file distributed -# as part of the Derivative Works; within the Source form or -# documentation, if provided along with the Derivative Works; or, -# within a display generated by the Derivative Works, if and -# wherever such third-party notices normally appear. The contents -# of the NOTICE file are for informational purposes only and -# do not modify the License. You may add Your own attribution -# notices within Derivative Works that You distribute, alongside -# or as an addendum to the NOTICE text from the Work, provided -# that such additional attribution notices cannot be construed -# as modifying the License. -# -# You may add Your own copyright statement to Your modifications and -# may provide additional or different license terms and conditions -# for use, reproduction, or distribution of Your modifications, or -# for any such Derivative Works as a whole, provided Your use, -# reproduction, and distribution of the Work otherwise complies with -# the conditions stated in this License. -# -# 5. Submission of Contributions. Unless You explicitly state otherwise, -# any Contribution intentionally submitted for inclusion in the Work -# by You to the Licensor shall be under the terms and conditions of -# this License, without any additional terms or conditions. -# Notwithstanding the above, nothing herein shall supersede or modify -# the terms of any separate license agreement you may have executed -# with Licensor regarding such Contributions. -# -# 6. Trademarks. This License does not grant permission to use the trade -# names, trademarks, service marks, or product names of the Licensor, -# except as required for reasonable and customary use in describing the -# origin of the Work and reproducing the content of the NOTICE file. -# -# 7. Disclaimer of Warranty. Unless required by applicable law or -# agreed to in writing, Licensor provides the Work (and each -# Contributor provides its Contributions) on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied, including, without limitation, any warranties or conditions -# of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -# PARTICULAR PURPOSE. You are solely responsible for determining the -# appropriateness of using or redistributing the Work and assume any -# risks associated with Your exercise of permissions under this License. -# -# 8. Limitation of Liability. In no event and under no legal theory, -# whether in tort (including negligence), contract, or otherwise, -# unless required by applicable law (such as deliberate and grossly -# negligent acts) or agreed to in writing, shall any Contributor be -# liable to You for damages, including any direct, indirect, special, -# incidental, or consequential damages of any character arising as a -# result of this License or out of the use or inability to use the -# Work (including but not limited to damages for loss of goodwill, -# work stoppage, computer failure or malfunction, or any and all -# other commercial damages or losses), even if such Contributor -# has been advised of the possibility of such damages. -# -# 9. Accepting Warranty or Additional Liability. While redistributing -# the Work or Derivative Works thereof, You may choose to offer, -# and charge a fee for, acceptance of support, warranty, indemnity, -# or other liability obligations and/or rights consistent with this -# License. However, in accepting such obligations, You may act only -# on Your own behalf and on Your sole responsibility, not on behalf -# of any other Contributor, and only if You agree to indemnify, -# defend, and hold each Contributor harmless for any liability -# incurred by, or claims asserted against, such Contributor by reason -# of your accepting any such warranty or additional liability. -# -# END OF TERMS AND CONDITIONS -# -# APPENDIX: How to apply the Apache License to your work. -# -# To apply the Apache License to your work, attach the following -# boilerplate notice, with the fields enclosed by brackets "[]" -# replaced with your own identifying information. (Don't include -# the brackets!) The text should be enclosed in the appropriate -# comment syntax for the file format. We also recommend that a -# file or class name and description of purpose be included on the -# same "printed page" as the copyright notice for easier -# identification within third-party archives. -# -# Copyright [yyyy] [name of copyright owner] -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 5c36a6a52..a463e923e 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -46,34 +46,157 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): if search_engine.lower() == "baidu": - return search_web_baidu() + web_search_results = search_web_baidu(search_query, None, None) + process_results = process_web_search_results(web_search_results, "baidu") + elif search_engine.lower() == "bing": - return search_web_bing(search_query, search_lang, content_country, date_range, result_count) + web_search_results = search_web_bing(search_query, search_lang, content_country, date_range, result_count) + process_results = process_web_search_results(web_search_results, "bing") + elif search_engine.lower() == "brave": - return search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, + web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, site_blacklist, date_range) + process_results = process_web_search_results(web_search_results, "brave") + elif search_engine.lower() == "duckduckgo": - return search_web_duckduckgo(search_query, content_country, date_range, result_count) + web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) + process_results = process_web_search_results(web_search_results, "ddg") + elif search_engine.lower() == "google": - return search_web_google(search_query, result_count, content_country, date_range, exactTerms, + web_search_results = search_web_google(search_query, result_count, content_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, search_result_language, safesearch, site_blacklist, sort_results_by) + process_results = process_web_search_results(web_search_results, "google") + elif search_engine.lower() == "kagi": - return search_web_kagi(search_query, content_country, search_lang, output_lang, result_count, safesearch, date_range, + web_search_results = search_web_kagi(search_query, content_country, search_lang, output_lang, result_count, safesearch, date_range, site_blacklist) + process_results = process_web_search_results(web_search_results, "kagi") + elif search_engine.lower() == "serper": - return search_web_serper() + web_search_results = search_web_serper() + process_results = process_web_search_results(web_search_results, "serper") + elif search_engine.lower() == "tavily": - return search_web_tavily(search_query, result_count, site_blacklist) + web_search_results = search_web_tavily(search_query, result_count, site_blacklist) + process_results = process_web_search_results(web_search_results, "tavily") + elif search_engine.lower() == "searx": - return search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') + web_search_results = search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') + process_results = process_web_search_results(web_search_results, "bing") + elif search_engine.lower() == "yandex": - return search_web_yandex() + web_search_results = search_web_yandex() + process_results = process_web_search_results(web_search_results, "bing") + else: return f"Error: Invalid Search Engine Name {search_engine}" -######################### Search Results Parsing ######################### +# +######################### Search Result Parsing ################################################################## +# + +def process_web_search_results(search_results: Dict, search_engine: str) -> Dict: + """ + Processes search results from a search engine and formats them into a standardized dictionary structure. + + Args: + search_results (Dict): The raw search results from the search engine. + search_engine (str): The name of the search engine (e.g., "Google", "Bing"). + + Returns: + Dict: A dictionary containing the processed search results in the specified structure. + """ + # Initialize the output dictionary with default values + web_search_results_dict = { + "search_engine": search_engine, + "search_query": search_results.get("search_query", ""), + "content_country": search_results.get("content_country", ""), + "search_lang": search_results.get("search_lang", ""), + "output_lang": search_results.get("output_lang", ""), + "result_count": search_results.get("result_count", 0), + "date_range": search_results.get("date_range", None), + "safesearch": search_results.get("safesearch", None), + "site_blacklist": search_results.get("site_blacklist", None), + "exactTerms": search_results.get("exactTerms", None), + "excludeTerms": search_results.get("excludeTerms", None), + "filter": search_results.get("filter", None), + "geolocation": search_results.get("geolocation", None), + "search_result_language": search_results.get("search_result_language", None), + "sort_results_by": search_results.get("sort_results_by", None), + "results": [], + "total_results_found": search_results.get("total_results_found", 0), + "search_time": search_results.get("search_time", 0.0), + "error": search_results.get("error", None), + "processing_error": None + } + try: + if search_engine.lower() == "baidu": + pass + elif search_engine.lower() == "bing": + parse_bing_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "brave": + parse_brave_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "duckduckgo": + parse_duckduckgo_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "google": + parse_google_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "kagi": + parse_kagi_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "serper": + parse_serper_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "tavily": + parse_tavily_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "searx": + parse_searx_results(search_results, web_search_results_dict) + pass + + elif search_engine.lower() == "yandex": + parse_yandex_results(search_results, web_search_results_dict) + pass + + else: + web_search_results_dict["processing_error"] = f"Error: Invalid Search Engine Name {search_engine}" + raise ValueError(f"Error: Invalid Search Engine Name {search_engine}") + except Exception as e: + web_search_results_dict["processing_error"] = f"Error processing search results: {str(e)}" + raise + + # Process individual search results + for result in search_results.get("results", []): + processed_result = { + "title": result.get("title", ""), + "url": result.get("url", ""), + "content": result.get("content", ""), + "metadata": { + "date_published": result.get("metadata", {}).get("date_published", None), + "author": result.get("metadata", {}).get("author", None), + "source": result.get("metadata", {}).get("source", None), + "language": result.get("metadata", {}).get("language", None), + "relevance_score": result.get("metadata", {}).get("relevance_score", None), + "snippet": result.get("metadata", {}).get("snippet", None) + } + } + web_search_results_dict["results"].append(processed_result) + + return web_search_results_dict + + def parse_html_search_results_generic(soup): results = [] for result in soup.find_all('div', class_='result'): @@ -96,7 +219,7 @@ def parse_html_search_results_generic(soup): # # https://cloud.baidu.com/doc/APIGUIDE/s/Xk1myz05f # https://oxylabs.io/blog/how-to-scrape-baidu-search-results -def search_web_baidu(): +def search_web_baidu(arg1, arg2, arg3): pass @@ -104,6 +227,9 @@ def test_baidu_search(arg1, arg2, arg3): result = search_web_baidu(arg1, arg2, arg3) return result +def search_parse_baidu_results(): + pass + ######################### Bing Search ######################### # @@ -157,11 +283,11 @@ def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bi response = requests.get(search_url, headers=headers, params=params) response.raise_for_status() - print("Headers: ") - print(response.headers) + logging.debug("Headers: ") + logging.debug(response.headers) - print("JSON Response: ") - print(response.json()) + logging.debug("JSON Response: ") + logging.debug(response.json()) bing_search_results = response.json() return bing_search_results except Exception as ex: @@ -169,16 +295,85 @@ def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bi def test_search_web_bing(): - search_query = "How can I bake a cherry cake" + search_query = "How can I get started learning machine learning?" bing_lang = "en" bing_country = "US" result_count = 10 bing_api_key = None date_range = None result = search_web_bing(search_query, bing_lang, bing_country, result_count, bing_api_key, date_range) + print("Bing Search Results:") print(result) +# FIXME - untested +def parse_bing_results(raw_results: Dict, output_dict: Dict) -> None: + """ + Parse Bing search results and update the output dictionary + + Args: + raw_results (Dict): Raw Bing API response + output_dict (Dict): Dictionary to store processed results + """ + try: + # Extract web pages results + if "webPages" in raw_results: + web_pages = raw_results["webPages"] + output_dict["total_results_found"] = web_pages.get("totalEstimatedMatches", 0) + + for result in web_pages.get("value", []): + processed_result = { + "title": result.get("name", ""), + "url": result.get("url", ""), + "content": result.get("snippet", ""), + "metadata": { + "date_published": None, # Bing doesn't typically provide this + "author": None, # Bing doesn't typically provide this + "source": result.get("displayUrl", None), + "language": None, # Could be extracted from result.get("language") if available + "relevance_score": None, # Could be calculated from result.get("rank") if available + "snippet": result.get("snippet", None) + } + } + output_dict["results"].append(processed_result) + + # Optionally process other result types + if "news" in raw_results: + for news_item in raw_results["news"].get("value", []): + processed_result = { + "title": news_item.get("name", ""), + "url": news_item.get("url", ""), + "content": news_item.get("description", ""), + "metadata": { + "date_published": news_item.get("datePublished", None), + "author": news_item.get("provider", [{}])[0].get("name", None), + "source": news_item.get("provider", [{}])[0].get("name", None), + "language": None, + "relevance_score": None, + "snippet": news_item.get("description", None) + } + } + output_dict["results"].append(processed_result) + + # Add spell suggestions if available + if "spellSuggestion" in raw_results: + output_dict["spell_suggestions"] = raw_results["spellSuggestion"] + + # Add related searches if available + if "relatedSearches" in raw_results: + output_dict["related_searches"] = [ + item.get("text", "") + for item in raw_results["relatedSearches"].get("value", []) + ] + + except Exception as e: + output_dict["processing_error"] = f"Error processing Bing results: {str(e)}" + + +def test_parse_bing_results(): + pass + + ######################### Brave Search ######################### # # https://brave.com/search/api/ @@ -238,6 +433,65 @@ def test_search_brave(): return result +# FIXME - untested +def parse_brave_results(raw_results: Dict, output_dict: Dict) -> None: + """ + Parse Brave search results and update the output dictionary + + Args: + raw_results (Dict): Raw Brave API response + output_dict (Dict): Dictionary to store processed results + """ + try: + # Extract query information + if "query" in raw_results: + query_info = raw_results["query"] + output_dict.update({ + "search_query": query_info.get("original", ""), + "content_country": query_info.get("country", ""), + "city": query_info.get("city", ""), + "state": query_info.get("state", ""), + "more_results_available": query_info.get("more_results_available", False) + }) + + # Process web results + if "web" in raw_results and "results" in raw_results["web"]: + for result in raw_results["web"]["results"]: + processed_result = { + "title": result.get("title", ""), + "url": result.get("url", ""), + "content": result.get("description", ""), + "metadata": { + "date_published": result.get("page_age", None), + "author": None, + "source": result.get("profile", {}).get("name", None), + "language": result.get("language", None), + "relevance_score": None, + "snippet": result.get("description", None), + "family_friendly": result.get("family_friendly", None), + "type": result.get("type", None), + "subtype": result.get("subtype", None), + "thumbnail": result.get("thumbnail", {}).get("src", None) + } + } + output_dict["results"].append(processed_result) + + # Update total results count + if "mixed" in raw_results: + output_dict["total_results_found"] = len(raw_results["mixed"].get("main", [])) + + # Set family friendly status + if "mixed" in raw_results: + output_dict["family_friendly"] = raw_results.get("family_friendly", True) + + except Exception as e: + output_dict["processing_error"] = f"Error processing Brave results: {str(e)}" + raise + +def test_parse_brave_results(): + pass + + ######################### DuckDuckGo Search ######################### # # https://github.com/deedy5/duckduckgo_search @@ -351,6 +605,81 @@ def test_search_duckduckgo(): print(f"Request error: {str(e)}") +def parse_duckduckgo_results(raw_results: Dict, output_dict: Dict) -> None: + """ + Parse DuckDuckGo search results and update the output dictionary + + Args: + raw_results (Dict): Raw DuckDuckGo response + output_dict (Dict): Dictionary to store processed results + """ + try: + # DuckDuckGo results appear to be in a simple list format + # Each result is separated by "---" + results = raw_results.get("results", []) + + for result in results: + # Extract information using the consistent format in results + title = "" + url = "" + snippet = "" + + # Parse the result text + lines = result.split('\n') + for line in lines: + if line.startswith("Title: "): + title = line.replace("Title: ", "").strip() + elif line.startswith("URL: "): + url = line.replace("URL: ", "").strip() + elif line.startswith("Snippet: "): + snippet = line.replace("Snippet: ", "").strip() + + processed_result = { + "title": title, + "url": url, + "content": snippet, + "metadata": { + "date_published": None, # DuckDuckGo doesn't typically provide this + "author": None, # DuckDuckGo doesn't typically provide this + "source": extract_domain(url) if url else None, + "language": None, # DuckDuckGo doesn't typically provide this + "relevance_score": None, # DuckDuckGo doesn't typically provide this + "snippet": snippet + } + } + + output_dict["results"].append(processed_result) + + # Update total results count + output_dict["total_results_found"] = len(output_dict["results"]) + + except Exception as e: + output_dict["processing_error"] = f"Error processing DuckDuckGo results: {str(e)}" + + +def extract_domain(url: str) -> str: + """ + Extract domain name from URL + + Args: + url (str): Full URL + + Returns: + str: Domain name + """ + try: + from urllib.parse import urlparse + parsed_uri = urlparse(url) + domain = parsed_uri.netloc + return domain.replace('www.', '') + except: + return url + +def test_parse_duckduckgo_results(): + pass + + + ######################### Google Search ######################### # # https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list @@ -503,47 +832,184 @@ def test_search_google(): print(result) +# FIXME - untested +def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: + """ + Parse Google Custom Search API results and update the output dictionary + + Args: + raw_results (Dict): Raw Google API response + output_dict (Dict): Dictionary to store processed results + """ + try: + # Extract search information + if "searchInformation" in raw_results: + search_info = raw_results["searchInformation"] + output_dict["total_results_found"] = int(search_info.get("totalResults", "0")) + output_dict["search_time"] = search_info.get("searchTime", 0.0) + + # Extract spelling suggestions + if "spelling" in raw_results: + output_dict["spell_suggestions"] = raw_results["spelling"].get("correctedQuery") + + # Extract search parameters from queries + if "queries" in raw_results and "request" in raw_results["queries"]: + request = raw_results["queries"]["request"][0] + output_dict.update({ + "search_query": request.get("searchTerms", ""), + "search_lang": request.get("language", ""), + "result_count": request.get("count", 0), + "safesearch": request.get("safe", None), + "exactTerms": request.get("exactTerms", None), + "excludeTerms": request.get("excludeTerms", None), + "filter": request.get("filter", None), + "geolocation": request.get("gl", None), + "search_result_language": request.get("hl", None), + "sort_results_by": request.get("sort", None) + }) + + # Process search results + if "items" in raw_results: + for item in raw_results["items"]: + processed_result = { + "title": item.get("title", ""), + "url": item.get("link", ""), + "content": item.get("snippet", ""), + "metadata": { + "date_published": item.get("pagemap", {}).get("metatags", [{}])[0].get( + "article:published_time"), + "author": item.get("pagemap", {}).get("metatags", [{}])[0].get("article:author"), + "source": item.get("displayLink", None), + "language": item.get("language", None), + "relevance_score": None, # Google doesn't provide this directly + "snippet": item.get("snippet", None), + "file_format": item.get("fileFormat", None), + "mime_type": item.get("mime", None), + "cache_url": item.get("cacheId", None) + } + } + + # Extract additional metadata if available + if "pagemap" in item: + pagemap = item["pagemap"] + if "metatags" in pagemap and pagemap["metatags"]: + metatags = pagemap["metatags"][0] + processed_result["metadata"].update({ + "description": metatags.get("og:description", + metatags.get("description")), + "keywords": metatags.get("keywords"), + "site_name": metatags.get("og:site_name") + }) + + output_dict["results"].append(processed_result) + + # Add pagination information + output_dict["pagination"] = { + "has_next": "nextPage" in raw_results.get("queries", {}), + "has_previous": "previousPage" in raw_results.get("queries", {}), + "current_page": raw_results.get("queries", {}).get("request", [{}])[0].get("startIndex", 1) + } + + except Exception as e: + output_dict["processing_error"] = f"Error processing Google results: {str(e)}" + +def test_parse_google_results(): + pass + + + ######################### Kagi Search ######################### # # https://help.kagi.com/kagi/api/search.html -def search_web_kagi(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", date_range=None, - result_filter=None, kagi_api_key=None): - search_url = "https://api.search.brave.com/res/v1/web/search" +def search_web_kagi(query: str, limit: int = 10) -> Dict: + search_url = "https://kagi.com/api/v0/search" + # load key from config file + kagi_api_key = loaded_config_data['search_engines']['kagi_search_api_key'] if not kagi_api_key: - # load key from config file - if not kagi_api_key: - raise ValueError("Please provide a valid Kagi Search API subscription key") - if not country: - country = "US" - if not search_lang: - search_lang = "en" - if not ui_lang: - ui_lang = "en" - if not result_count: - result_count = 10 - # if not date_range: - # date_range = "month" - if not result_filter: - result_filter = "webpages" + raise ValueError("Please provide a valid Kagi Search API subscription key") - headers = {"Authorization: Bot " + kagi_api_key} + """ + Queries the Kagi Search API with the given query and limit. + """ + if kagi_api_key is None: + raise ValueError("API key is required.") - # https://api.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters - params = {"q": search_term, "textDecorations": True, "textFormat": "HTML", "count": result_count, - "freshness": date_range, "promote": "webpages", "safeSearch": "Moderate"} + headers = {"Authorization": f"Bot {kagi_api_key}"} + endpoint = f"{search_url}/search" + params = {"q": query, "limit": limit} - response = requests.get(search_url, headers=headers, params=params) + response = requests.get(endpoint, headers=headers, params=params) response.raise_for_status() - # Response: https://api.search.brave.com/app/documentation/web-search/responses#WebSearchApiResponse - kagi_search_results = response.json() - return kagi_search_results + print(response.json()) + return response.json() def test_search_kagi(): + search_term = "How can I bake a cherry cake" + result_count = 10 + result = search_web_kagi(search_term, result_count) + print(result) + + +def parse_kagi_results(raw_results: Dict, output_dict: Dict) -> None: + """ + Parse Kagi search results and update the output dictionary + + Args: + raw_results (Dict): Raw Kagi API response + output_dict (Dict): Dictionary to store processed results + """ + try: + # Extract metadata + if "meta" in raw_results: + meta = raw_results["meta"] + output_dict["search_time"] = meta.get("ms", 0) / 1000.0 # Convert to seconds + output_dict["api_balance"] = meta.get("api_balance") + output_dict["search_id"] = meta.get("id") + output_dict["node"] = meta.get("node") + + # Process search results + if "data" in raw_results: + for item in raw_results["data"]: + # Skip related searches (type 1) + if item.get("t") == 1: + output_dict["related_searches"] = item.get("list", []) + continue + + # Process regular search results (type 0) + if item.get("t") == 0: + processed_result = { + "title": item.get("title", ""), + "url": item.get("url", ""), + "content": item.get("snippet", ""), + "metadata": { + "date_published": item.get("published"), + "author": None, # Kagi doesn't typically provide this + "source": None, # Could be extracted from URL if needed + "language": None, # Kagi doesn't typically provide this + "relevance_score": None, + "snippet": item.get("snippet"), + "thumbnail": item.get("thumbnail", {}).get("url") if "thumbnail" in item else None + } + } + output_dict["results"].append(processed_result) + + # Update total results count + output_dict["total_results_found"] = len([ + item for item in raw_results["data"] + if item.get("t") == 0 + ]) + + except Exception as e: + output_dict["processing_error"] = f"Error processing Kagi results: {str(e)}" + + +def test_parse_kagi_results(): pass + ######################### SearX Search ######################### # # https://searx.space @@ -612,6 +1078,14 @@ def test_search_searx(): print(result) pass +def parse_searx_results(searx_search_results, web_search_results_dict): + pass + +def test_parse_searx_results(): + pass + + + ######################### Serper.dev Search ######################### # @@ -623,6 +1097,11 @@ def search_web_serper(): def test_search_serper(): pass +def parse_serper_results(serper_search_results, web_search_results_dict): + pass + + + ######################### Tavily Search ######################### # @@ -664,6 +1143,17 @@ def test_search_tavily(): result = search_web_tavily("How can I bake a cherry cake?") print(result) + +def parse_tavily_results(tavily_search_results, web_search_results_dict): + pass + + +def test_parse_tavily_results(): + pass + + + + ######################### Yandex Search ######################### # # https://yandex.cloud/en/docs/search-api/operations/web-search @@ -677,6 +1167,10 @@ def search_web_yandex(): def test_search_yandex(): pass +def parse_yandex_results(yandex_search_results, web_search_results_dict): + pass + + # # End of WebSearch_APIs.py ####################################################################################################################### diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index a07a32ad4..7def74312 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -9,3 +9,24 @@ https://arxiv.org/abs/2411.07407 https://arxiv.org/abs/2412.16429 https://huggingface.co/papers/2412.15443 + + + + + +one2manny + — +Today at 12:43 AM +A great way to make studying more efficient and convenient is to take a digital PDF textbook, split it into separate files for each chapter, and organize them individually. +I then create a dedicated notebook for each chapter, treating it as a focused single source. +From there, I convert each chapter into an audio format, like a podcast. +This approach makes it easy to study while commuting, relaxing in bed with your eyes closed, or at any time when reading isn’t practical. + +I also recommend creating a study guide for each chapter, fully breaking down key concepts and definitions. +For more complex topics, the “explain like I’m 5” method works wonders—it simplifies challenging ideas into digestible explanations. + +To take this further, incorporate a Personal Knowledge Management (PKM) system into your routine. +Apps like Obsidian are perfect for this, with their flexible folder structures and Markdown formatting. +I optimize my AI outputs for Markdown so I can copy, paste, and organize them into clean, structured notes. +This ensures your materials are not only well-organized but also easy to access and build on later. +A solid PKM system is invaluable for managing knowledge and staying on top of your studies! \ No newline at end of file diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index c1d01cc00..1ccbb7fa4 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -82,6 +82,14 @@ search_blacklist_URLs = "URL1,URL2,URL3" ``` +Perplexica + https://github.com/ItzCrazyKns/Perplexica/blob/master/src/search/metaSearchAgent.ts + https://github.com/ItzCrazyKns/Perplexica/blob/master/src/chains/suggestionGeneratorAgent.ts + https://github.com/ItzCrazyKns/Perplexica/blob/master/src/chains/imageSearchAgent.ts + https://github.com/ItzCrazyKns/Perplexica/blob/master/src/search/metaSearchAgent.ts + +Falle + https://github.com/rashadphz/farfalle/blob/main/src/backend/agent_search.py ### Link Dump: @@ -91,6 +99,9 @@ Articles https://docs.gptr.dev/docs/gpt-researcher/context/tailored-research# https://docs.gptr.dev/docs/gpt-researcher/gptr/pip-package +Standford STORM + https://arxiv.org/abs/2402.14207# + https://storm.genie.stanford.edu/ https://github.com/assafelovic/gpt-researcher https://arxiv.org/abs/2411.15114 @@ -123,7 +134,6 @@ https://github.com/faraz18001/Sales-Llama https://github.com/memgraph/memgraph https://github.com/rashadphz/farfalle/tree/main/src/backend https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py -https://storm.genie.stanford.edu/ https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama @@ -133,305 +143,44 @@ https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama ### Researcher Prompts +https://github.com/cbuccella/perplexity_research_prompt +https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py - - - +https://github.com/ItzCrazyKns/Perplexica/tree/master/src/prompts https://github.com/SakanaAI/AI-Scientist - -one2manny - — -Today at 12:43 AM -A great way to make studying more efficient and convenient is to take a digital PDF textbook, split it into separate files for each chapter, and organize them individually. I then create a dedicated notebook for each chapter, treating it as a focused single source. From there, I convert each chapter into an audio format, like a podcast. This approach makes it easy to study while commuting, relaxing in bed with your eyes closed, or at any time when reading isn’t practical. - -I also recommend creating a study guide for each chapter, fully breaking down key concepts and definitions. For more complex topics, the “explain like I’m 5” method works wonders—it simplifies challenging ideas into digestible explanations. - -To take this further, incorporate a Personal Knowledge Management (PKM) system into your routine. Apps like Obsidian are perfect for this, with their flexible folder structures and Markdown formatting. I optimize my AI outputs for Markdown so I can copy, paste, and organize them into clean, structured notes. This ensures your materials are not only well-organized but also easy to access and build on later. A solid PKM system is invaluable for managing knowledge and staying on top of your studies! - - ----------------- -### Search Prompts - -inference prompt -``` -system_prompt = ( - "You are an expert summarizing the answers based on the provided contents." -) - -user_promt_template = """ -Given the context as a sequence of references with a reference id in the -format of a leading [x], please answer the following question using {{ language }}: - -{{ query }} - -In the answer, use format [1], [2], ..., [n] in line where the reference is used. -For example, "According to the research from Google[3], ...". - -Please create the answer strictly related to the context. If the context has no -information about the query, please write "No related information found in the context." -using {{ language }}. - -{{ length_instructions }} - -Here is the context: -{{ context }} -""" -``` -extraction prompt -``` -system_prompt = ( - "You are an expert of extract structual information from the document." -) -user_promt_template = """ -Given the provided content, if it contains information about {{ query }}, please extract the -list of structured data items as defined in the following Pydantic schema: - -{{ extract_schema_str }} - -Below is the provided content: -{{ content }} -""" -``` - - -Web-LLM Assistant - https://github.com/TheBlewish/Web-LLM-Assistant-Llamacpp-Ollama -Starting prompt ``` -SYSTEM_PROMPT = """You are an AI assistant capable of web searching and providing informative responses. -When a user's query starts with '/', interpret it as a request to search the web and formulate an appropriate search query. - -ALWAYS follow the prompts provided throughout the searching process EXACTLY as indicated. +SEARCH_QUERY_PROMPT = """\ +Generate a concise list of search queries to gather information for executing the given step. -NEVER assume new instructions for anywhere other than directly when prompted directly. DO NOT SELF PROMPT OR PROVIDE MULTIPLE ANSWERS OR ATTEMPT MULTIPLE RESPONSES FOR ONE PROMPT! -""" -``` -self-improving prompt -``` - def evaluate_scraped_content(self, user_query: str, scraped_content: Dict[str, str]) -> Tuple[str, str]: - user_query_short = user_query[:200] - prompt = f""" -Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: +You will be provided with: +1. A specific step to execute +2. The user's original query +3. Context from previous steps (if available) -User's question: "{user_query_short}" +Use this information to create targeted search queries that will help complete the current step effectively. Aim for the minimum number of queries necessary while ensuring they cover all aspects of the step. -Scraped Content: -{self.format_scraped_content(scraped_content)} +IMPORTANT: Always incorporate relevant information from previous steps into your queries. This ensures continuity and builds upon already gathered information. -Your task: -1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. -2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. +Input: +--- +User's original query: {user_query} +--- +Context from previous steps: +{prev_steps_context} -Respond using EXACTLY this format: -Evaluation: [Your evaluation of the scraped content] -Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] -""" -``` -Query Creation -``` -def formulate_query(self, user_query: str, attempt: int) -> Tuple[str, str]: - user_query_short = user_query[:200] - prompt = f""" -Based on the following user question, formulate a concise and effective search query: -"{user_query_short}" Your task: -1. Create a search query of 2-5 words that will yield relevant results. -2. Determine if a specific time range is needed for the search. -Time range options: -- 'd': Limit results to the past day. Use for very recent events or rapidly changing information. -- 'w': Limit results to the past week. Use for recent events or topics with frequent updates. -- 'm': Limit results to the past month. Use for relatively recent information or ongoing events. -- 'y': Limit results to the past year. Use for annual events or information that changes yearly. -- 'none': No time limit. Use for historical information or topics not tied to a specific time frame. -Respond in the following format: -Search query: [Your 2-5 word query] -Time range: [d/w/m/y/none] -Do not provide any additional information or explanation. +1. Analyze the current step and its requirements +2. Consider the user's original query and any relevant previous context +3. Consider the user's original query +4. Generate a list of specific, focused search queries that: + - Incorporate relevant information from previous steps + - Address the requirements of the current step + - Build upon the information already gathered +--- +Current step to execute: {current_step} +--- + +Your search queries based: """ -``` -Select relevant content -``` -def select_relevant_pages(self, search_results: List[Dict], user_query: str) -> List[str]: - prompt = f""" -Given the following search results for the user's question: "{user_query}" -Select the 2 most relevant results to scrape and analyze. Explain your reasoning for each selection. - -Search Results: -{self.format_results(search_results)} - -Instructions: -1. You MUST select exactly 2 result numbers from the search results. -2. Choose the results that are most likely to contain comprehensive and relevant information to answer the user's question. -3. Provide a brief reason for each selection. - -You MUST respond using EXACTLY this format and nothing else: - -Selected Results: [Two numbers corresponding to the selected results] -Reasoning: [Your reasoning for the selections] -""" -``` -Final answer generation -``` - def generate_final_answer(self, user_query: str, scraped_content: Dict[str, str]) -> str: - user_query_short = user_query[:200] - prompt = f""" -You are an AI assistant. Provide a comprehensive and detailed answer to the following question using ONLY the information provided in the scraped content. Do not include any references or mention any sources. Answer directly and thoroughly. - -Question: "{user_query_short}" - -Scraped Content: -{self.format_scraped_content(scraped_content)} - -Important Instructions: -1. Do not use phrases like "Based on the absence of selected results" or similar. -2. If the scraped content does not contain enough information to answer the question, say so explicitly and explain what information is missing. -3. Provide as much relevant detail as possible from the scraped content. - -Answer: -""" -``` -Final Answer Synthesis -``` - def synthesize_final_answer(self, user_query: str) -> str: - prompt = f""" -After multiple search attempts, we couldn't find a fully satisfactory answer to the user's question: "{user_query}" - -Please provide the best possible answer you can, acknowledging any limitations or uncertainties. -If appropriate, suggest ways the user might refine their question or where they might find more information. - -Respond in a clear, concise, and informative manner. -""" -``` - - -https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py -``` -search_prompt_system = """ -You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate -and unbiased answers the way a highly informed individual would. -Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. -You must answer in english. -Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. -you are known for your expertise in this field. - - -###Guidelines### -1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. -2- never mention the context or this prompt in your response, just answer the user question. - -###Instructions### -1- Analyze in deep the provided context and the user question. -2- extract relevant information's from the context about the user question. -3- Yassine must take into account the date and time to answer the user question. -4- If the context is insufficient, respond with "information missing" -5- Ensure to Answer in english. -6- Use the response format provided. -7- answer the user question in a way an expert would do. -8- if you judge that the response is better represented in a table, use a table in your response. - - -###Response Format### - -You must use Markdown to format your response. - -Think step by step. -""" - -relevant_prompt_system = """ - you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related - to the user query and contexts provided. - you must keep the questions related to the user query and contexts.don't lose the context in the questions. - - The JSON object must not include special characters. - The JSON schema should include an array of follow-up questions. - - use the schema: - { - "followUp": [ - "string", - "string", - "string" - ] - } -""" -``` - - -appvoid search - https://github.com/appvoid/search -Eval query type -``` -async def evaluate_query_type(self, session, query): - messages = [ - {"role": "system", "content": """You are an Web assistant that evaluates the type of query a user asks. - Categorize the query into one of the following types: - 1. simple: if it can be answered with general knowledge or information that is typically well-known on the internet, please provide a short answer as relevant as possible from the llm itself, but make sure you are completly sure you know the answer, don't make things up. - 2. realtime: if it requires up-to-date information like the current date, time, or recent events, or the user explicitly asks you to look on the internet you should state as: realtime - 3. math: if it involves ANY kind of mathematical calculations. Every math question be it counting letters or complex formulas. - - Remember to prioritize realtime over anything else if you are not sure about something. Realtime is like your default. - - Respond with the category as a single word ("simple", "realtime", or "math") without any additional text."""}, - {"role": "user", "content": f"Query: {query}"} - ] -``` -Generate Search Queries -``` -async def generate_search_queries(groq_api, session, original_query, max_retries=3, fixed_count=None, previous_queries=None, previous_answer=None): - system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. The queries should be diverse and cover different aspects or perspectives of the original query. Return the queries as a JSON array. - Important instructions: - - 1. The number of queries should be dynamic, between 2 and 4, unless a fixed count is specified. - 2. Don't get too far from the original query since you don't know the actual context. - 3. Make queries general enough without being related to anything specific. - 4. DON'T customize the queries for topics you've never seen; just change them a little and look for definitions if requested by the user. - 5. If the user asks something that is not related to search, ignore it and focus on generating helpful search queries. - 6. Just return the given format ["custom_query_1","custom_query_2",...]. - 7. If you need to use your knowledge first, do so. - 8. When asked about the difference between two things, generate search intents for each topic separately. - 9. ALWAYS at most queries just require one or two queries, only on those cases where the query is simple or you are unsure, generate more than one or two. - 10. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. - 11. ALWAYS split searches for each important part of the query in case you need to gather information but make sure to not get off the rails. In short, don't look for things together, make a search for each important part instead. DONT LOOK FOR THINGS TOGETHER.""" - - messages = [ - {"role": "system", "content": system_content}, - {"role": "user", "content": f"Original query: {original_query}" + (f" (Generate exactly {fixed_count} queries)" if fixed_count else "")} - ] - - if previous_queries and previous_answer: - messages.append({ - "role": "user", - "content": f"Previous queries: {previous_queries}\nPrevious answer: {previous_answer}\nPlease generate new queries to address any shortcomings in the previous answer." - }) -``` -Evaluate Answer -``` -async def evaluate_answer(groq_api, session, query, answer): - messages = [ - {"role": "system", "content": """You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. - Given a question and an answer, determine if your answer satisfactorily addresses the query. You are highly tolerant to answers that are close to the intent so if it is close enough, you can say is satisfactory. Remember, if it's close enough, mark it as satisfactory. - Respond with a JSON object containing two fields: - 1. "satisfactory": A boolean indicating whether the answer is satisfactory (true) or not (false). - 2. "reason": A brief explanation of why your thought is or is not satisfactory. Like "I will keep looking for information since last thought is not addressing the query because..." or "Let look for something different. My last search didn't solve the query. The reason is..." or "I found the right answer so I can ignore this..."."""}, - {"role": "user", "content": f"Query: {query}\nAnswer: {answer}"} - ] -``` -Eval best answer -``` -async def evaluate_best_answer(groq_api, session, query, cached_answers): - print('Answers pool > ', cached_answers) - messages = [ - {"role": "system", "content": """You are an assistant that evaluates multiple answers to a query and selects the best one based on relevance and completeness. - Given a query and a list of answers, choose the answer that best addresses the query. Respond with the best answer. Don't need to mention the word answers at all just be natural. Don't "the best answer" or things like that. Just provide the best one."""}, - {"role": "user", "content": f"Query: {query}\nAnswers: {json.dumps(cached_answers)}"} - ] -``` -Summarization -``` -messages = [ - {"role": "system", "content": """You are a web assistant that helps users find information from web search results. -Given a question and a set of search results, provide a concise response based on the information -available in the search results. If the information is not available in the search results, -state that you don't have enough information to answer the question. You MUST not comment on anything, just follow the instruction. Don't add additional details about anything."""}, - {"role": "user", "content": f"Question: {query}\nSearch Results: {json.dumps(all_results)}"} -] ``` \ No newline at end of file diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 64a5f4733..ff3b554a9 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -50,8 +50,14 @@ https://github.com/microsoft/SpeechT5 https://github.com/smellslikeml/dolla_llama +Coqui TTS + https://github.com/idiap/coqui-ai-TTS +Cartesia + https://docs.cartesia.ai/get-started/make-an-api-request +F5 TTS + https://github.com/SWivid/F5-TTS Podcastfy diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index f817eda02..e432cf7ee 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -3,10 +3,40 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. + +Pipeline: +1. User posts question + - Gradio/UI/API +2. Question is analyzed + - Question is analyzed to identify most likely purpose/goal of question, and Sub-questions are generated to support this + - User has option of seeing/modifying prompt used for Analysis/sub-question creation +3. Search(es) is/are performed - User toggled + - Search is performed using the user's question and sub-questions +4. Results are collected, stored, and analyzed + - Results are collected, stored in a temp 'search_results' dict, and analyzed for relevance, based on initial snippet(? or full page?) + - User has the option of seeing all results, or only relevant results + - User has the option to select which results are 'relevant', + - User also has the option to select which 'relevant' results are used to answer the question +5. Relevant results are added to result dictionary + - Results determined to be relevant are then stored in a 'relevant_results' dictionary, and the process is repeated until all results are analyzed/limit is hit. +6. Once all results are collected, they are then used to answer the user's question/sub-questions + - The relevant results are then used to answer the user's question/sub-questions + - Each result is first abstract summarized, FIXME +7. The final answer/'briefing' is then presented to the user +8. User has the option to save the results to the DB +9. User has the option to ask follow-up questions / see potential other questions + + + + + ---------------- ### Setting the Stage +- The goal of this module is to provide a simple, easy-to-use interface for searching the web and retrieving results. - All the web searches are simple HTTP requests to an API or to the direct endpoint and then scraping the results. -- Parsing results is TODO. +- Results are then reviewed for relevancy, if relevant, the full page is fetched and analyzed. +- The results are then stored in a dictionary, and the process is repeated until all results are analyzed/limit is hit. +- Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. - The goal is to provide a simple, easy-to-use interface for searching the web and retrieving results. - Other modules are responsible for anything else, this module just performs the search, and delivers the results. - **Main Function:** diff --git a/Server_API/API_README.md b/Server_API/API_README.md new file mode 100644 index 000000000..7861c896d --- /dev/null +++ b/Server_API/API_README.md @@ -0,0 +1,134 @@ +# API Documentation + +## Overview + +API uses FastAPI to provide a RESTful interface to the backend services. The API is designed to be simple and easy to use, with a focus on providing a clean interface for the frontend to interact with. + +- **URLs** + - Main page: http://127.0.0.1:8000 + - API Documentation page: http://127.0.0.1:8000/docs + + + +## Endpoints + + + +``` +Here’s the important part. We’ll create: + + A global asyncio.Queue of “write tasks.” + A WriteTask class that holds the SQL, parameters, and an asyncio.Future to signal completion. + A background worker (writer_worker) that pops tasks from the queue, executes them, and sets the result in the Future. + Endpoints that push a WriteTask onto the queue, then await the Future before returning. + +# main.py +import asyncio +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from typing import Any, Tuple, Union + +from database import get_db_connection + +app = FastAPI() + +# ----------------------------- +# 1) A global queue + task class +# ----------------------------- +class WriteTask: + """Holds SQL, parameters, and a Future to let the enqueuing code wait for completion.""" + def __init__(self, sql: str, params: tuple[Any, ...]): + self.sql = sql + self.params = params + self.future: asyncio.Future = asyncio.get_event_loop().create_future() + +write_queue: asyncio.Queue[WriteTask] = asyncio.Queue() + + +# ----------------------------- +# 2) The background worker +# ----------------------------- +async def writer_worker(): + """Continuously processes write tasks from the queue, one at a time.""" + while True: + task: WriteTask = await write_queue.get() + try: + # Perform the write + with get_db_connection() as conn: + conn.execute(task.sql, task.params) + conn.commit() + + # If success, set the result of the Future + task.future.set_result(True) + except Exception as e: + # If failure, set the exception so the caller can handle it + task.future.set_exception(e) + finally: + write_queue.task_done() + + +# ----------------------------- +# 3) Start the worker on startup +# ----------------------------- +@app.on_event("startup") +async def startup_event(): + # Launch the writer worker as a background task + asyncio.create_task(writer_worker()) + + +# ----------------------------- +# 4) Pydantic model for input +# ----------------------------- +class ItemCreate(BaseModel): + name: str + + +# ----------------------------- +# 5) Write endpoint (POST) +# ----------------------------- +@app.post("/items") +async def create_item(item: ItemCreate): + """Queue a write to the database, then wait for its completion.""" + sql = "INSERT INTO items (name) VALUES (?)" + params = (item.name,) + + # Create a WriteTask + write_task = WriteTask(sql, params) + + # Put the task in the queue + await write_queue.put(write_task) + + # Wait for the task to complete + try: + result = await write_task.future # This will be True if successful + return {"status": "success", "name": item.name} + except Exception as exc: + # If the DB write failed for some reason, raise a 500 + raise HTTPException(status_code=500, detail=str(exc)) + + +# ----------------------------- +# 6) Read endpoint (GET) +# ----------------------------- +@app.get("/items") +def read_items(): + """Simple read operation that does not need the queue.""" + with get_db_connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id, name FROM items") + rows = cursor.fetchall() + return [{"id": row[0], "name": row[1]} for row in rows] + +Explanation + + WriteTask stores (sql, params, future). The future is how we pass success/failure back to the original request. + When a request hits POST /items, we: + Construct a WriteTask. + put() it on the write_queue. + Immediately await write_task.future. We don’t return until the DB operation is done. + The writer_worker loop picks tasks in FIFO order and executes them one-by-one, guaranteeing no concurrency for writes (thus avoiding locks). + On success, task.future.set_result(True) is called. On failure, task.future.set_exception(e). + The awaiting endpoint sees either a success (and returns HTTP 200) or an exception (and returns HTTP 500). + + This pattern means each request is effectively serialized for writes, but the user still gets a definitive success/failure response in the same request/response cycle. +``` \ No newline at end of file diff --git a/Server_API/app/main.py b/Server_API/app/main.py index 7b8a6591b..07a90a5a4 100644 --- a/Server_API/app/main.py +++ b/Server_API/app/main.py @@ -1,7 +1,22 @@ +# main.py +# Description: This file contains the main FastAPI application, which serves as the primary API for the tldw application. +# +# Imports +# +# 3rd-party Libraries from fastapi import FastAPI +# +# Local Imports +# +######################################################################################################################## +# +# Functions: -app = FastAPI(title="TLDW API", version="1.0.0") +# Usage: uvicorn main:app --reload +app = FastAPI(title="tldw API", version="1.0.0") @app.get("/") async def root(): return {"message": "Welcome to the tldw API"} + + From f108cd7b691431a7df6aa9449fbe12072a51bdb5 Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 28 Dec 2024 19:35:53 -0800 Subject: [PATCH 32/88] little bit closer to perplexity --- .../Web_Scraping/WebSearch_APIs.py | 305 +++++++++++++++++- Docs/Design/Coding_Page.md | 1 + Docs/Design/Education.md | 2 +- Docs/Design/Researcher.md | 4 +- Docs/Design/Summarization.md | 21 ++ Docs/Design/TTS_STT.md | 1 + Docs/Design/UX.md | 3 + Docs/Design/WebSearch.md | 13 +- .../Prompts/WebSearch}/Search_Prompt.md | 278 ++++++++-------- 9 files changed, 456 insertions(+), 172 deletions(-) create mode 100644 Docs/Design/Summarization.md rename {App_Function_Libraries/Web_Scraping => Docs/Prompts/WebSearch}/Search_Prompt.md (90%) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index a463e923e..296a30a86 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -6,7 +6,7 @@ import logging import re from html import unescape -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List from urllib.parse import urlparse, urlencode, unquote # # 3rd-Party Imports @@ -16,6 +16,8 @@ from requests import RequestException from requests.adapters import HTTPAdapter from urllib3 import Retry + +from App_Function_Libraries.Chat.Chat_Functions import chat_api_call # # Local Imports from App_Function_Libraries.Utils.Utils import loaded_config_data @@ -40,54 +42,327 @@ # # Functions: -# FIXME - parsing for results from each search engine -# FIXME - Create results dictionary format/specification +######################### Question Analysis ######################### +# +# +def analyze_question(question: str, api_endpoint) -> Dict: + logging.debug(f"Analyzing question: {question} with API endpoint: {api_endpoint}") + """ + Analyzes the input question and generates sub-questions + + Returns: + Dict containing: + - main_goal: str + - sub_questions: List[str] + - search_queries: List[str] + - analysis_prompt: str + """ + original_query = question + sub_question_generation_prompt = f""" + You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. Your goal is to generate queries that are diverse, specific, and highly relevant to the original query, ensuring comprehensive coverage of the topic. + + Important instructions: + 1. Generate between 2 and 6 queries unless a fixed count is specified. Generate more queries for complex or multifaceted topics and fewer for simple or straightforward ones. + 2. Ensure the queries are diverse, covering different aspects or perspectives of the original query, while remaining highly relevant to its core intent. + 3. Prefer specific queries over general ones, as they are more likely to yield targeted and useful results. + 4. If the query involves comparing two topics, generate separate queries for each topic. + 5. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. + 6. If the original query is broad or ambiguous, generate queries that explore specific subtopics or clarify the intent. + 7. If the query is too specific or unclear, generate queries that explore related or broader topics to ensure useful results. + 8. Return the queries as a JSON array in the format ["query_1", "query_2", ...]. + + Examples: + 1. For the query "What are the benefits of exercise?", generate queries like: + ["health benefits of physical activity", "mental health benefits of exercise", "long-term effects of regular exercise", "how exercise improves cardiovascular health", "role of exercise in weight management"] + + 2. For the query "Compare Python and JavaScript", generate queries like: + ["key features of Python programming language", "advantages of JavaScript for web development", "use cases for Python vs JavaScript", "performance comparison of Python and JavaScript", "ease of learning Python vs JavaScript"] + + 3. For the query "How does climate change affect biodiversity?", generate queries like: + ["impact of climate change on species extinction", "effects of global warming on ecosystems", "role of climate change in habitat loss", "how rising temperatures affect marine biodiversity", "climate change and its impact on migratory patterns"] + + 4. For the query "Best practices for remote work", generate queries like: + ["tips for staying productive while working from home", "how to maintain work-life balance in remote work", "tools for effective remote team collaboration", "managing communication in remote teams", "ergonomic setup for home offices"] + + 5. For the query "What is quantum computing?", generate queries like: + ["basic principles of quantum computing", "applications of quantum computing in real-world problems", "difference between classical and quantum computing", "key challenges in developing quantum computers", "future prospects of quantum computing"] + + Original query: {original_query} + """ + + input_data = "Follow the above instructions." + + sub_questions: List[str] = [] + try: + for attempt in range(3): + logging.info(f"Generating sub-questions (attempt {attempt + 1})") + response = chat_api_call(api_endpoint, None, input_data, sub_question_generation_prompt, temp=0.7) + if response: + try: + # Try to parse as JSON first + parsed_response = json.loads(response) + sub_questions = parsed_response.get("sub_questions", []) + if sub_questions: + logging.info("Successfully generated sub-questions from JSON") + break + except json.JSONDecodeError: + # If JSON parsing fails, attempt a regex-based fallback + logging.warning("Failed to parse as JSON. Attempting regex extraction.") + matches = re.findall(r'"([^"]*)"', response) + sub_questions = matches if matches else [] + if sub_questions: + logging.info("Successfully extracted sub-questions using regex") + break + + # If still no sub-questions, log an error or handle appropriately + if not sub_questions: + logging.error("Failed to extract sub-questions from API response after all attempts.") + + except Exception as e: + logging.error(f"Error generating sub-questions: {str(e)}") + + # Construct and return the result dictionary + logging.info("Sub-questions generated successfully") + return { + "main_goal": original_query, + "sub_questions": sub_questions, + "search_queries": sub_questions, + "analysis_prompt": sub_question_generation_prompt + } + + +######################### Relevance Scoring ######################### +# +def score_result_relevance( + result: Dict, + original_question: str, + sub_questions: List[str] +) -> float: + """ + Scores how relevant a search result is to the question + """ + # Implement scoring logic + pass + + +######################### Result Aggregation & Combination ######################### +# +def aggregate_results( + relevant_results: Dict, + question: str, + sub_questions: List[str] +) -> Dict: + """ + Combines and summarizes relevant results + + Returns: + Dict containing: + - summary: str + - evidence: List[Dict] + - confidence: float + """ + pass + + +######################### Main Orchestration Workflow ######################### +# +def process_question(question: str, search_params: Dict) -> Dict: + """ + Orchestrates the entire pipeline: + 1. Optionally generate sub-queries (if subquery_generation=True). + 2. Perform web searches for each query (the original and sub-queries). + 3. (Placeholder) Score results for relevance; filter them. + 4. (Placeholder) Aggregate final answer from relevant results. + + Args: + question (str): The user’s original question or query. + search_params (Dict): Dictionary of search parameters (engine, lang, date_range, etc.). + + Returns: + Dict: A dictionary containing all relevant data, including results from each sub-query. + + A dictionary containing parameters for performing a web search and processing the results. + + Parameters: + engine (str): The search engine to use (e.g., "google", "bing", "brave", "duckduckgo", etc.). + content_country (str): The country code for content localization (e.g., "US", "UK", "DE"). + search_lang (str): The language for the search query (e.g., "en" for English). + output_lang (str): The desired language for the search results. + result_count (int): The number of results to return. + date_range (str or None): The time range for the search results (e.g., "y" for the past year). + safesearch (str or None): The safe search setting (e.g., "moderate", "strict", "off"). + site_blacklist (list or None): A list of sites to exclude from the search results. + exactTerms (str or None): Terms that must appear in the search results. + excludeTerms (str or None): Terms that must not appear in the search results. + filter (str or None): Any additional filtering criteria. + geolocation (str or None): Geographical location for search results. + search_result_language (str or None): The language of the search results. + sort_results_by (str or None): Criteria for sorting the results. + subquery_generation (bool): Whether to generate sub-queries. + subquery_generation_api (str): The API to use for sub-query generation (e.g., "openai", "anthropic", "deepseek"). + relevance_analysis_llm (str): The LLM model to use for relevance analysis (e.g., "openai", "anthropic", "deepseek"). + final_answer_llm (str): The LLM model to use for generating the final answer (e.g., "openai", "anthropic", "deepseek"). + + Example: + search_params = { + "engine": "google", + "content_country": "US", + "search_lang": "en", + "output_lang": "en", + "result_count": 10, + "date_range": "y", + "safesearch": "moderate", + "site_blacklist": ["example.com", "spam-site.com"], + "exactTerms": None, + "excludeTerms": None, + "filter": None, + "geolocation": None, + "search_result_language": None, + "sort_results_by": None, + "subquery_generation": True, + "subquery_generation_llm": "openai", + "relevance_analysis_llm": "openai", + "final_answer_llm": "openai" + } + """ + logging.info(f"Starting process_question with query: {question}") + + # 1. Generate sub-queries if requested + if search_params.get("subquery_generation", False): + api_endpoint = search_params.get("subquery_generation_llm", "openai") + sub_query_dict = analyze_question(question, api_endpoint) + sub_queries = sub_query_dict.get("sub_questions", []) + else: + sub_query_dict = { + "main_goal": question, + "sub_questions": [], + "search_queries": [], + "analysis_prompt": None + } + sub_queries = [] + + # Merge original question with sub-queries + all_queries = [question] + sub_queries + + # 2. Perform searches and accumulate all raw results + all_results: List[Dict] = [] + + for q in all_queries: + raw_results = perform_websearch( + search_engine=search_params.get('engine'), + search_query=q, + content_country=search_params.get('content_country'), + search_lang=search_params.get('search_lang'), + output_lang=search_params.get('output_lang'), + result_count=search_params.get('result_count', 10), + date_range=search_params.get('date_range'), + safesearch=search_params.get('safesearch'), + site_blacklist=search_params.get('site_blacklist'), + exactTerms=search_params.get('exactTerms'), + excludeTerms=search_params.get('excludeTerms'), + filter=search_params.get('filter'), + geolocation=search_params.get('geolocation'), + search_result_language=search_params.get('search_result_language'), + sort_results_by=search_params.get('sort_results_by') + ) + + if isinstance(raw_results, dict) and "processing_error" not in raw_results: + # 'raw_results' should be a processed dict from process_web_search_results + # containing a "results" list inside it. + results_list = raw_results.get("results", []) + all_results.extend(results_list) + else: + # If an error string or error dictionary came back, handle it: + logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") + + # 3. Score/filter (placeholder) + relevant_results = {} + for r in all_results: + relevance_score = score_result_relevance( + r, + question, + sub_query_dict['sub_questions'] + ) + if relevance_score > RELEVANCE_THRESHOLD: + # Use URL or another unique field as the key + key = r.get("url") or r.get("title") + relevant_results[key] = { + "result": r, + "relevance_score": relevance_score + } + + # 4. Summarize/aggregate final answer + final_answer = aggregate_results( + relevant_results, + question, + sub_query_dict['sub_questions'] + ) + + return { + "search_engine": search_params.get('engine'), + "original_query": question, + "sub_questions": sub_query_dict.get('sub_questions', []), + "all_queries": all_queries, + "results": all_results, + "relevant_results": relevant_results, + "analysis_prompt": sub_query_dict.get('analysis_prompt'), + "final_answer": final_answer + } + + +# +# End of Orchestration functions +####################################################################################################################### + + +####################################################################################################################### +# +# Search Engine Functions def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): if search_engine.lower() == "baidu": web_search_results = search_web_baidu(search_query, None, None) - process_results = process_web_search_results(web_search_results, "baidu") + processed_results = process_web_search_results(web_search_results, "baidu") elif search_engine.lower() == "bing": web_search_results = search_web_bing(search_query, search_lang, content_country, date_range, result_count) - process_results = process_web_search_results(web_search_results, "bing") + processed_results = process_web_search_results(web_search_results, "bing") elif search_engine.lower() == "brave": web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, site_blacklist, date_range) - process_results = process_web_search_results(web_search_results, "brave") + processed_results = process_web_search_results(web_search_results, "brave") elif search_engine.lower() == "duckduckgo": web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) - process_results = process_web_search_results(web_search_results, "ddg") + processed_results = process_web_search_results(web_search_results, "ddg") elif search_engine.lower() == "google": web_search_results = search_web_google(search_query, result_count, content_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, search_result_language, safesearch, site_blacklist, sort_results_by) - process_results = process_web_search_results(web_search_results, "google") + processed_results = process_web_search_results(web_search_results, "google") elif search_engine.lower() == "kagi": - web_search_results = search_web_kagi(search_query, content_country, search_lang, output_lang, result_count, safesearch, date_range, - site_blacklist) - process_results = process_web_search_results(web_search_results, "kagi") + web_search_results = search_web_kagi(search_query, content_country) + processed_results = process_web_search_results(web_search_results, "kagi") elif search_engine.lower() == "serper": web_search_results = search_web_serper() - process_results = process_web_search_results(web_search_results, "serper") + processed_results = process_web_search_results(web_search_results, "serper") elif search_engine.lower() == "tavily": web_search_results = search_web_tavily(search_query, result_count, site_blacklist) - process_results = process_web_search_results(web_search_results, "tavily") + processed_results = process_web_search_results(web_search_results, "tavily") elif search_engine.lower() == "searx": web_search_results = search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') - process_results = process_web_search_results(web_search_results, "bing") + processed_results = process_web_search_results(web_search_results, "bing") elif search_engine.lower() == "yandex": web_search_results = search_web_yandex() - process_results = process_web_search_results(web_search_results, "bing") + processed_results = process_web_search_results(web_search_results, "bing") else: return f"Error: Invalid Search Engine Name {search_engine}" diff --git a/Docs/Design/Coding_Page.md b/Docs/Design/Coding_Page.md index 061f90337..a1c57e587 100644 --- a/Docs/Design/Coding_Page.md +++ b/Docs/Design/Coding_Page.md @@ -18,6 +18,7 @@ https://pythontutor.com/articles/c-cpp-visualizer.html https://gitingest.com/ https://gitdiagram.com/ https://www.ilograph.com/blog/posts/diagrams-ai-can-and-cannot-generate/#system-diagramming-with-ai +https://github.com/osanseviero/geminicoder diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index 7def74312..fd5957474 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -8,7 +8,7 @@ https://excalidraw.com/ https://arxiv.org/abs/2411.07407 https://arxiv.org/abs/2412.16429 https://huggingface.co/papers/2412.15443 - +https://github.com/thiswillbeyourgithub/AnkiAIUtils diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 1ccbb7fa4..e399a64db 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -135,7 +135,9 @@ https://github.com/memgraph/memgraph https://github.com/rashadphz/farfalle/tree/main/src/backend https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama - +https://github.com/rashadphz/farfalle/blob/main/src/backend/agent_search.py +https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py +https://github.com/stanford-oval/storm/blob/main/examples/storm_examples/run_storm_wiki_claude.py diff --git a/Docs/Design/Summarization.md b/Docs/Design/Summarization.md new file mode 100644 index 000000000..e28d4a543 --- /dev/null +++ b/Docs/Design/Summarization.md @@ -0,0 +1,21 @@ +# Summarization + +## Introduction + +This page is to document the 'summarization' workflow for use within the project. The goal is to create a system that can automatically generate summaries of text, documents, and other content. + + + +### Summarization Goals + + +### Relevant Research + + + +### Link Dump +https://neptune.ai/blog/llm-evaluation-text-summarization +https://phoenix.arize.com/llm-summarization-getting-to-production/ +https://blog.metrostar.com/iteratively-summarize-long-documents-llm +https://arxiv.org/html/2412.15487v1 + diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index ff3b554a9..6ee8cd852 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -52,6 +52,7 @@ https://github.com/smellslikeml/dolla_llama Coqui TTS https://github.com/idiap/coqui-ai-TTS + https://huggingface.co/spaces/coqui/xtts/blob/main/app.py Cartesia https://docs.cartesia.ai/get-started/make-an-api-request diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 733af5b42..26639e40b 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -81,6 +81,9 @@ https://uxdesign.cc/youre-trying-to-turn-an-oil-tanker-288ed30876e4 https://uxdesign.cc/simple-recall-testing-for-b2b-ux-134e18330da9 https://lovable.dev/ https://markwhen.com/ +https://kando.menu/ + + Prompt Engineering page: - 3 Sections diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index e432cf7ee..1196fa120 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -10,18 +10,17 @@ Pipeline: 2. Question is analyzed - Question is analyzed to identify most likely purpose/goal of question, and Sub-questions are generated to support this - User has option of seeing/modifying prompt used for Analysis/sub-question creation -3. Search(es) is/are performed - User toggled +3. Search(es) is/are performed - User gated - Search is performed using the user's question and sub-questions 4. Results are collected, stored, and analyzed - - Results are collected, stored in a temp 'search_results' dict, and analyzed for relevance, based on initial snippet(? or full page?) - - User has the option of seeing all results, or only relevant results - - User has the option to select which results are 'relevant', + - Results are collected, stored in a 'search_results' dict, and analyzed for relevance, based on initial snippet(? or full page?) + - User has the option of seeing all results, removing irrelevant results, and selecting which results are 'relevant' - User also has the option to select which 'relevant' results are used to answer the question -5. Relevant results are added to result dictionary +5. Irrelevant results are removed from the dictionary - Results determined to be relevant are then stored in a 'relevant_results' dictionary, and the process is repeated until all results are analyzed/limit is hit. 6. Once all results are collected, they are then used to answer the user's question/sub-questions - - The relevant results are then used to answer the user's question/sub-questions - - Each result is first abstract summarized, FIXME + - The remaining relevant results are then used to answer the user's question/sub-questions + - Each result is abstractly summarized, and then combined into a single document and supplied to the LLM for final analysis 7. The final answer/'briefing' is then presented to the user 8. User has the option to save the results to the DB 9. User has the option to ask follow-up questions / see potential other questions diff --git a/App_Function_Libraries/Web_Scraping/Search_Prompt.md b/Docs/Prompts/WebSearch/Search_Prompt.md similarity index 90% rename from App_Function_Libraries/Web_Scraping/Search_Prompt.md rename to Docs/Prompts/WebSearch/Search_Prompt.md index b4995f2a0..7deb57e3e 100644 --- a/App_Function_Libraries/Web_Scraping/Search_Prompt.md +++ b/Docs/Prompts/WebSearch/Search_Prompt.md @@ -2,46 +2,7 @@ ######################################################################################################################## -Initial Search Prompts -``` -CHAT_PROMPT = """\ -Generate a comprehensive, well-structured, and informative answer for a given question, -using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). -Use an unbiased, journalistic tone, adapting the level of formality to match the user’s question. - -• Cite your statements using [number] notation, placing citations at the end of the relevant sentence. -• Only cite the most relevant results. If multiple sources support the same point, cite all relevant sources [e.g., 1, 2, 3]. -• If sources conflict, present both perspectives clearly and cite the respective sources. -• If different sources refer to different entities with the same name, provide separate answers. -• Do not add any external or fabricated information. -• Do not include URLs or a reference section; cite inline with [number] format only. -• Do not repeat the question or include unnecessary redundancy. -• Use markdown formatting (e.g., **bold**, bullet points, ## headings) to organize the information. -• If the provided results are insufficient to answer the question, explicitly state what information is missing or unclear. - -Structure your answer like this: -1. **Short introduction**: Briefly summarize the topic (1–2 sentences). -2. **Bulleted points**: Present key details, each with appropriate citations. -3. **Conclusion**: Summarize the findings or restate the core answer (with citations if needed). - -Example: -1. **Short introduction**: This topic explores the impact of climate change on agriculture. -2. **Bulleted points**: - - Rising temperatures have reduced crop yields in some regions [1]. - - Changes in rainfall patterns are affecting irrigation practices [2, 3]. -3. **Conclusion**: Climate change poses significant challenges to global agriculture [1, 2, 3]. - - -{my_context} - ---------------------- -Make sure to match the language of the user's question. - -Question: {my_query} -Answer (in the language of the user's question): -""" -``` @@ -50,132 +11,40 @@ Answer (in the language of the user's question): Sub-Query Generation Prompts ``` -You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. Your goal is to generate queries that are diverse, specific, and highly relevant to the original query, ensuring comprehensive coverage of the topic. +system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. Your goal is to generate queries that are diverse, specific, and highly relevant to the original query, ensuring comprehensive coverage of the topic. Important instructions: -1. Generate between 2 and 6 queries unless a fixed count is specified, while also generating more queries for complex or multifaceted topics and fewer for simple or straightforward ones. +1. Generate between 2 and 6 queries unless a fixed count is specified. Generate more queries for complex or multifaceted topics and fewer for simple or straightforward ones. 2. Ensure the queries are diverse, covering different aspects or perspectives of the original query, while remaining highly relevant to its core intent. 3. Prefer specific queries over general ones, as they are more likely to yield targeted and useful results. 4. If the query involves comparing two topics, generate separate queries for each topic. 5. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. -6. Split searches for each important part of the query to ensure comprehensive coverage. -7. If the original query is broad or ambiguous, generate queries that explore specific subtopics or clarify the intent. -8. If the query is too specific or unclear, generate queries that explore related or broader topics to ensure useful results. -9. Return the queries as a comma-separated list or in a natural language format, depending on the user's needs. - -Example: -For the query "What are the benefits of exercise?", generate queries like "health benefits of physical activity," "mental health benefits of exercise," and "long-term effects of regular exercise." - -Original query: {original_query} -``` - -https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py -``` -search_prompt_system = """ -You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate -and unbiased answers the way a highly informed individual would. -Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. -You must answer in english. -Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. -you are known for your expertise in this field. - - -###Guidelines### -1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. -2- never mention the context or this prompt in your response, just answer the user question. +6. If the original query is broad or ambiguous, generate queries that explore specific subtopics or clarify the intent. +7. If the query is too specific or unclear, generate queries that explore related or broader topics to ensure useful results. +8. Return the queries as a JSON array in the format ["query_1", "query_2", ...]. -###Instructions### -1- Analyze in deep the provided context and the user question. -2- extract relevant information's from the context about the user question. -3- Yassine must take into account the date and time to answer the user question. -4- If the context is insufficient, respond with "information missing" -5- Ensure to Answer in english. -6- Use the response format provided. -7- answer the user question in a way an expert would do. -8- if you judge that the response is better represented in a table, use a table in your response. - - -###Response Format### - -You must use Markdown to format your response. - -Think step by step. -""" - -relevant_prompt_system = """ - you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related - to the user query and contexts provided. - you must keep the questions related to the user query and contexts.don't lose the context in the questions. - - The JSON object must not include special characters. - The JSON schema should include an array of follow-up questions. - - use the schema: - { - "followUp": [ - "string", - "string", - "string" - ] - } -""" -``` - - - -self-improving prompt -``` -Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: - -User's question: "{user_query_short}" - -Scraped Content: -{self.format_scraped_content(scraped_content)} +Examples: +1. For the query "What are the benefits of exercise?", generate queries like: + ["health benefits of physical activity", "mental health benefits of exercise", "long-term effects of regular exercise", "how exercise improves cardiovascular health", "role of exercise in weight management"] -Your task: -1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. -2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. +2. For the query "Compare Python and JavaScript", generate queries like: + ["key features of Python programming language", "advantages of JavaScript for web development", "use cases for Python vs JavaScript", "performance comparison of Python and JavaScript", "ease of learning Python vs JavaScript"] -Respond using EXACTLY this format: -Evaluation: [Your evaluation of the scraped content] -Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] -""" -``` +3. For the query "How does climate change affect biodiversity?", generate queries like: + ["impact of climate change on species extinction", "effects of global warming on ecosystems", "role of climate change in habitat loss", "how rising temperatures affect marine biodiversity", "climate change and its impact on migratory patterns"] +4. For the query "Best practices for remote work", generate queries like: + ["tips for staying productive while working from home", "how to maintain work-life balance in remote work", "tools for effective remote team collaboration", "managing communication in remote teams", "ergonomic setup for home offices"] -Generate Search Queries -``` -system_content = """You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. The queries should be diverse and cover different aspects or perspectives of the original query. Return the queries as a JSON array. - Important instructions: - - 1. The number of queries should be dynamic, between 2 and 4, unless a fixed count is specified. - 2. Don't get too far from the original query since you don't know the actual context. - 3. Make queries general enough without being related to anything specific. - 4. DON'T customize the queries for topics you've never seen; just change them a little and look for definitions if requested by the user. - 5. If the user asks something that is not related to search, ignore it and focus on generating helpful search queries. - 6. Just return the given format ["custom_query_1","custom_query_2",...]. - 7. If you need to use your knowledge first, do so. - 8. When asked about the difference between two things, generate search intents for each topic separately. - 9. ALWAYS at most queries just require one or two queries, only on those cases where the query is simple or you are unsure, generate more than one or two. - 10. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. - 11. ALWAYS split searches for each important part of the query in case you need to gather information but make sure to not get off the rails. In short, don't look for things together, make a search for each important part instead. DONT LOOK FOR THINGS TOGETHER.""" - - messages = [ - {"role": "system", "content": system_content}, - {"role": "user", "content": f"Original query: {original_query}" + (f" (Generate exactly {fixed_count} queries)" if fixed_count else "")} - ] +5. For the query "What is quantum computing?", generate queries like: + ["basic principles of quantum computing", "applications of quantum computing in real-world problems", "difference between classical and quantum computing", "key challenges in developing quantum computers", "future prospects of quantum computing"] - if previous_queries and previous_answer: - messages.append({ - "role": "user", - "content": f"Previous queries: {previous_queries}\nPrevious answer: {previous_answer}\nPlease generate new queries to address any shortcomings in the previous answer." - }) +Original query: {original_query} ``` - ######################################################################################################################## Content Extraction Prompts @@ -217,6 +86,26 @@ messages = [ ``` +self-improving prompt +``` +Evaluate if the following scraped content contains sufficient information to answer the user's question comprehensively: + +User's question: "{user_query_short}" + +Scraped Content: +{self.format_scraped_content(scraped_content)} + +Your task: +1. Determine if the scraped content provides enough relevant and detailed information to answer the user's question thoroughly. +2. If the information is sufficient, decide to 'answer'. If more information or clarification is needed, decide to 'refine' the search. + +Respond using EXACTLY this format: +Evaluation: [Your evaluation of the scraped content] +Decision: [ONLY 'answer' if content is sufficient, or 'refine' if more information is needed] +""" +``` + + Select relevant content ``` Given the following search results for the user's question: "{user_query}" @@ -241,6 +130,46 @@ Reasoning: [Your reasoning for the selections] ######################################################################################################################## Final Answer Generation Prompts +Search Analysis +``` +CHAT_PROMPT = """\ +Generate a comprehensive, well-structured, and informative answer for a given question, +using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). +Use an unbiased, journalistic tone, adapting the level of formality to match the user’s question. + +• Cite your statements using [number] notation, placing citations at the end of the relevant sentence. +• Only cite the most relevant results. If multiple sources support the same point, cite all relevant sources [e.g., 1, 2, 3]. +• If sources conflict, present both perspectives clearly and cite the respective sources. +• If different sources refer to different entities with the same name, provide separate answers. +• Do not add any external or fabricated information. +• Do not include URLs or a reference section; cite inline with [number] format only. +• Do not repeat the question or include unnecessary redundancy. +• Use markdown formatting (e.g., **bold**, bullet points, ## headings) to organize the information. +• If the provided results are insufficient to answer the question, explicitly state what information is missing or unclear. + +Structure your answer like this: +1. **Short introduction**: Briefly summarize the topic (1–2 sentences). +2. **Bulleted points**: Present key details, each with appropriate citations. +3. **Conclusion**: Summarize the findings or restate the core answer (with citations if needed). + +Example: +1. **Short introduction**: This topic explores the impact of climate change on agriculture. +2. **Bulleted points**: + - Rising temperatures have reduced crop yields in some regions [1]. + - Changes in rainfall patterns are affecting irrigation practices [2, 3]. +3. **Conclusion**: Climate change poses significant challenges to global agriculture [1, 2, 3]. + + +{my_context} + +--------------------- + +Make sure to match the language of the user's question. + +Question: {my_query} +Answer (in the language of the user's question): +""" +``` Final-Answer-1 ``` @@ -386,6 +315,59 @@ In summary, [document's conclusion statement][6], highlighting the broader impli +https://github.com/YassKhazzan/openperplex_backend_os/blob/main/prompts.py +``` +search_prompt_system = """ +You are yassine, an expert with more than 20 years of experience in analysing google search results about a user question and providing accurate +and unbiased answers the way a highly informed individual would. +Your task is to analyse the provided contexts and the user question to provide a correct answer in a clear and concise manner. +You must answer in english. +Date and time in the context : {date_today} , Yassine must take into consideration the date and time in the response. +you are known for your expertise in this field. + + +###Guidelines### +1- Accuracy: Provide correct, unbiased answers. be concise and clear. don't be verbose. +2- never mention the context or this prompt in your response, just answer the user question. + +###Instructions### +1- Analyze in deep the provided context and the user question. +2- extract relevant information's from the context about the user question. +3- Yassine must take into account the date and time to answer the user question. +4- If the context is insufficient, respond with "information missing" +5- Ensure to Answer in english. +6- Use the response format provided. +7- answer the user question in a way an expert would do. +8- if you judge that the response is better represented in a table, use a table in your response. + + +###Response Format### + +You must use Markdown to format your response. + +Think step by step. +""" + +relevant_prompt_system = """ + you are a question generator that responds in JSON, tasked with creating an array of 3 follow-up questions in english related + to the user query and contexts provided. + you must keep the questions related to the user query and contexts.don't lose the context in the questions. + + The JSON object must not include special characters. + The JSON schema should include an array of follow-up questions. + + use the schema: + { + "followUp": [ + "string", + "string", + "string" + ] + } +""" +``` + + Perplexity Samples ``` From e1520a572a93ae8fde27ad5c0d434f1f8f4924ee Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 28 Dec 2024 23:10:39 -0800 Subject: [PATCH 33/88] f --- .../Web_Scraping/WebSearch_APIs.py | 248 ++++++++++++++---- Docs/Design/WebSearch.md | 36 +-- Docs/Prompts/WebSearch/Search_Prompt.md | 5 +- 3 files changed, 214 insertions(+), 75 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 296a30a86..e1dd3f34b 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -131,37 +131,173 @@ def analyze_question(question: str, api_endpoint) -> Dict: } -######################### Relevance Scoring ######################### +######################### Relevance Analysis ######################### # -def score_result_relevance( - result: Dict, - original_question: str, - sub_questions: List[str] -) -> float: +def search_result_relevance( + search_results: List[Dict], + original_question: str, + sub_questions: List[str], + api_endpoint: str +) -> Dict[str, Dict]: """ - Scores how relevant a search result is to the question + Evaluate whether each search result is relevant to the original question and sub-questions. + + Args: + search_results (List[Dict]): List of search results to evaluate. + original_question (str): The original question posed by the user. + sub_questions (List[str]): List of sub-questions generated from the original question. + api_endpoint (str): The LLM or API endpoint to use for relevance analysis. + + Returns: + Dict[str, Dict]: A dictionary of relevant results, keyed by a unique ID or index. """ - # Implement scoring logic - pass + relevant_results = {} + + for idx, result in enumerate(search_results): + content = result.get("content", "") + if not content: + logging.error("No Content found in search results array!") + + eval_prompt = f""" + Given the following search results for the user's question: "{original_question}" and the generated sub-questions: {sub_questions}, evaluate the relevance of the search result to the user's question. + Explain your reasoning for selection. + + Search Results: + {content} + + Instructions: + 1. You MUST only answer TRUE or False while providing your reasoning for your answer. + 2. A result is relevant if the result most likely contains comprehensive and relevant information to answer the user's question. + 3. Provide a brief reason for selection. + + You MUST respond using EXACTLY this format and nothing else: + + Selected Answer: [True or False] + Reasoning: [Your reasoning for the selections] + """ + input_data = "Evaluate the relevance of the search result." + + try: + # Perform API call to evaluate relevance + relevancy_result = chat_api_call( + api_endpoint=api_endpoint, + input_data=input_data, + prompt=eval_prompt, + temp=0.7 + ) + + if relevancy_result: + # Extract the selected answer and reasoning via regex + selected_answer_match = re.search( + r"Selected Answer:\s*(True|False)", + relevancy_result, + re.IGNORECASE + ) + reasoning_match = re.search( + r"Reasoning:\s*(.+)", + relevancy_result, + re.IGNORECASE + ) + + if selected_answer_match and reasoning_match: + is_relevant = selected_answer_match.group(1).strip().lower() == "true" + reasoning = reasoning_match.group(1).strip() + + if is_relevant: + # Use the 'id' from the result if available, otherwise use idx + result_id = result.get("id", str(idx)) + relevant_results[result_id] = { + "content": content, + "reasoning": reasoning + } + logging.info(f"Relevant result found: ID={result_id} Reasoning={reasoning}") + else: + logging.info(f"Irrelevant result: {reasoning}") + + else: + logging.warning("Failed to parse the API response for relevance analysis.") + except Exception as e: + logging.error(f"Error during relevance evaluation for result idx={idx}: {e}") + + return relevant_results ######################### Result Aggregation & Combination ######################### # def aggregate_results( - relevant_results: Dict, - question: str, - sub_questions: List[str] + relevant_results: Dict[str, Dict], + question: str, + sub_questions: List[str], + api_endpoint: str ) -> Dict: """ - Combines and summarizes relevant results + Combines and summarizes relevant results into a final answer. + + Args: + relevant_results (Dict[str, Dict]): Dictionary of relevant articles/content. + question (str): Original question. + sub_questions (List[str]): List of sub-questions. + api_endpoint (str): LLM or API endpoint for summarization. Returns: Dict containing: - - summary: str - - evidence: List[Dict] - - confidence: float + - summary (str): Final summarized answer. + - evidence (List[Dict]): List of relevant content items included in the summary. + - confidence (float): A rough confidence score (placeholder). """ - pass + if not relevant_results: + return { + "summary": "No relevant results found. Unable to provide an answer.", + "evidence": [], + "confidence": 0.0 + } + + # Concatenate relevant contents for summarization + concatenated_texts = "\n\n".join( + f"ID: {rid}\nContent: {res['content']}\nReasoning: {res['reasoning']}" + for rid, res in relevant_results.items() + ) + + # Example summarization prompt + summarize_prompt = f""" +Please provide a concise summary that answers the question: "{question}" + +Relevant sub-questions: {sub_questions} + +Here are the relevant excerpts from search results: +{concatenated_texts} + +Instructions: +1. Provide a concise summary that incorporates the key points. +2. Avoid quoting large passages verbatim; aim to synthesize the main ideas. +3. Make sure your answer is coherent, logically consistent, and addresses the question directly. +""" + + input_data = "Summarize the relevant results." + + try: + summary_response = chat_api_call( + api_endpoint=api_endpoint, + api_key=None, + input_data=input_data, + prompt=summarize_prompt, + temp=0.7 + ) + if summary_response: + # You could do further parsing or confidence estimation here + return { + "summary": summary_response, + "evidence": list(relevant_results.values()), + "confidence": 0.9 # Hardcoded or computed as needed + } + except Exception as e: + logging.error(f"Error aggregating results: {e}") + + return { + "summary": "Could not summarize the results due to an error.", + "evidence": list(relevant_results.values()), + "confidence": 0.0 + } ######################### Main Orchestration Workflow ######################### @@ -171,19 +307,20 @@ def process_question(question: str, search_params: Dict) -> Dict: Orchestrates the entire pipeline: 1. Optionally generate sub-queries (if subquery_generation=True). 2. Perform web searches for each query (the original and sub-queries). - 3. (Placeholder) Score results for relevance; filter them. - 4. (Placeholder) Aggregate final answer from relevant results. + 3. Score results for relevance; filter them. + 4. Aggregate final answer from relevant results. Args: - question (str): The user’s original question or query. - search_params (Dict): Dictionary of search parameters (engine, lang, date_range, etc.). + question (str): The user's original question or query. + search_params (Dict): A dictionary containing parameters for performing web searches + and specifying LLM endpoints. Returns: Dict: A dictionary containing all relevant data, including results from each sub-query. A dictionary containing parameters for performing a web search and processing the results. - Parameters: + Dict Parameters: engine (str): The search engine to use (e.g., "google", "bing", "brave", "duckduckgo", etc.). content_country (str): The country code for content localization (e.g., "US", "UK", "DE"). search_lang (str): The language for the search query (e.g., "en" for English). @@ -228,36 +365,35 @@ def process_question(question: str, search_params: Dict) -> Dict: logging.info(f"Starting process_question with query: {question}") # 1. Generate sub-queries if requested + sub_queries = [] + sub_query_dict = { + "main_goal": question, + "sub_questions": [], + "search_queries": [], + "analysis_prompt": None + } + if search_params.get("subquery_generation", False): api_endpoint = search_params.get("subquery_generation_llm", "openai") sub_query_dict = analyze_question(question, api_endpoint) sub_queries = sub_query_dict.get("sub_questions", []) - else: - sub_query_dict = { - "main_goal": question, - "sub_questions": [], - "search_queries": [], - "analysis_prompt": None - } - sub_queries = [] # Merge original question with sub-queries all_queries = [question] + sub_queries # 2. Perform searches and accumulate all raw results all_results: List[Dict] = [] - for q in all_queries: raw_results = perform_websearch( - search_engine=search_params.get('engine'), + search_engine=search_params.get('engine', 'google'), search_query=q, - content_country=search_params.get('content_country'), - search_lang=search_params.get('search_lang'), - output_lang=search_params.get('output_lang'), + content_country=search_params.get('content_country', 'US'), + search_lang=search_params.get('search_lang', 'en'), + output_lang=search_params.get('output_lang', 'en'), result_count=search_params.get('result_count', 10), date_range=search_params.get('date_range'), - safesearch=search_params.get('safesearch'), - site_blacklist=search_params.get('site_blacklist'), + safesearch=search_params.get('safesearch', 'moderate'), + site_blacklist=search_params.get('site_blacklist', []), exactTerms=search_params.get('exactTerms'), excludeTerms=search_params.get('excludeTerms'), filter=search_params.get('filter'), @@ -266,30 +402,32 @@ def process_question(question: str, search_params: Dict) -> Dict: sort_results_by=search_params.get('sort_results_by') ) - if isinstance(raw_results, dict) and "processing_error" not in raw_results: - # 'raw_results' should be a processed dict from process_web_search_results - # containing a "results" list inside it. - results_list = raw_results.get("results", []) - all_results.extend(results_list) - else: - # If an error string or error dictionary came back, handle it: + # Validate raw_results + if not isinstance(raw_results, dict) or "processing_error" in raw_results: logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") + continue + + results_list = raw_results.get("results", []) + all_results.extend(results_list) # 3. Score/filter (placeholder) relevant_results = {} for r in all_results: - relevance_score = score_result_relevance( - r, + # FIXME - Put in proper args / Ensure this works + # search_results: List[Dict], + # original_question: str, + # sub_questions: List[str], + # api_endpoint: str + list_of_relevant_articles = search_result_relevance( + all_results, question, - sub_query_dict['sub_questions'] + sub_query_dict['sub_questions'], + search_params.get('relevance_analysis_llm') ) - if relevance_score > RELEVANCE_THRESHOLD: - # Use URL or another unique field as the key - key = r.get("url") or r.get("title") - relevant_results[key] = { - "result": r, - "relevance_score": relevance_score - } + if list_of_relevant_articles: + relevant_results[r] = list_of_relevant_articles + + # 4. Summarize/aggregate final answer final_answer = aggregate_results( @@ -298,6 +436,8 @@ def process_question(question: str, search_params: Dict) -> Dict: sub_query_dict['sub_questions'] ) + # Return the final data + # FIXME - Return full query details for debugging and analysis return { "search_engine": search_params.get('engine'), "original_query": question, diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 1196fa120..b66aca73e 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -26,7 +26,23 @@ Pipeline: 9. User has the option to ask follow-up questions / see potential other questions - +- **Text Search Workflow** + 1. User inputs a search query + 2. User selects a search engine (Option for default search engine in config file) + 3. The user presses 'Search' + 4. The search query is passed to the selected search engine + 5. The appropriate search engine is used to perform a search via API call + 6. The search results are returned from the search engine's API + 7. Search engine results are then _MODIFIED_ (if necessary/enabled) to fit the user's preferences + - This could include re-ranking, summarization/analysis, or other modifications + 8. The (modified) search results are displayed to the user + 9. Results are then displayed to the user, + - either as titles of pages with dropdown for all info, + - or as a list of links with a briefing/summary of each link + - or as a single briefing/summary of all results + 10. User may then select to save this resulting text to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information + 11. Search results are then saved to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information + 12. This is then searchable via the Media DB ---------------- @@ -155,23 +171,7 @@ Perplexity - Output Metadata (URL, Date, etc) - Word count limit for output (per search result & total) - Output dialect (US, UK, a NY Radio DJ, etc) -- **Text Search Workflow** - 1. User inputs a search query - 2. User selects a search engine (Option for default search engine in config file) - 3. The user presses 'Search' - 4. The search query is passed to the selected search engine - 5. The appropriate search engine is used to perform a search via API call - 6. The search results are returned from the search engine's API - 7. Search engine results are then _MODIFIED_ (if necessary/enabled) to fit the user's preferences - - This could include re-ranking, summarization/analysis, or other modifications - 8. The (modified) search results are displayed to the user - 9. Results are then displayed to the user, - - either as titles of pages with dropdown for all info, - - or as a list of links with a briefing/summary of each link - - or as a single briefing/summary of all results - 10. User may then select to save this resulting text to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information - 11. Search results are then saved to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information - 12. This is then searchable via the Media DB + Results dictionary: ``` diff --git a/Docs/Prompts/WebSearch/Search_Prompt.md b/Docs/Prompts/WebSearch/Search_Prompt.md index 7deb57e3e..d73783d8f 100644 --- a/Docs/Prompts/WebSearch/Search_Prompt.md +++ b/Docs/Prompts/WebSearch/Search_Prompt.md @@ -67,13 +67,12 @@ Below is the provided content: Evaluate Answer ``` -messages = [ - {"role": "system", "content": """You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. +System: You are an AI assistant that evaluates the quality and completeness of its own answer to user queries. Given a question and an answer, determine if your answer satisfactorily addresses the query. You are highly tolerant to answers that are close to the intent so if it is close enough, you can say is satisfactory. Remember, if it's close enough, mark it as satisfactory. Respond with a JSON object containing two fields: 1. "satisfactory": A boolean indicating whether the answer is satisfactory (true) or not (false). 2. "reason": A brief explanation of why your thought is or is not satisfactory. Like "I will keep looking for information since last thought is not addressing the query because..." or "Let look for something different. My last search didn't solve the query. The reason is..." or "I found the right answer so I can ignore this..."."""}, - {"role": "user", "content": f"Query: {query}\nAnswer: {answer}"} +"user": f"Query: {query}\nAnswer: {answer}"} ] ``` Eval best answer From 067826d25d963724b03f3e7c52031eb3af810b97 Mon Sep 17 00:00:00 2001 From: Robert Date: Sat, 28 Dec 2024 23:19:48 -0800 Subject: [PATCH 34/88] f --- .../Web_Scraping/WebSearch_APIs.py | 20 ++++++- Docs/Design/WebSearch.md | 60 +++++++++++-------- 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index e1dd3f34b..dfa8766d8 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -181,6 +181,7 @@ def search_result_relevance( # Perform API call to evaluate relevance relevancy_result = chat_api_call( api_endpoint=api_endpoint, + api_key=None, input_data=input_data, prompt=eval_prompt, temp=0.7 @@ -431,9 +432,16 @@ def process_question(question: str, search_params: Dict) -> Dict: # 4. Summarize/aggregate final answer final_answer = aggregate_results( + # FIXME - Add proper Args / Ensure this works + # relevant_results: Dict[str, Dict], + # question: str, + # sub_questions: List[str], + # api_endpoint: str + # FIXME - Proper datatypes/expectations relevant_results, question, - sub_query_dict['sub_questions'] + sub_query_dict['sub_questions'], + api_endpoint=search_params.get('final_answer_llm') ) # Return the final data @@ -464,45 +472,55 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, if search_engine.lower() == "baidu": web_search_results = search_web_baidu(search_query, None, None) processed_results = process_web_search_results(web_search_results, "baidu") + return processed_results elif search_engine.lower() == "bing": web_search_results = search_web_bing(search_query, search_lang, content_country, date_range, result_count) processed_results = process_web_search_results(web_search_results, "bing") + return processed_results elif search_engine.lower() == "brave": web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, site_blacklist, date_range) processed_results = process_web_search_results(web_search_results, "brave") + return processed_results elif search_engine.lower() == "duckduckgo": web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) processed_results = process_web_search_results(web_search_results, "ddg") + return processed_results elif search_engine.lower() == "google": web_search_results = search_web_google(search_query, result_count, content_country, date_range, exactTerms, excludeTerms, filter, geolocation, output_lang, search_result_language, safesearch, site_blacklist, sort_results_by) processed_results = process_web_search_results(web_search_results, "google") + return processed_results elif search_engine.lower() == "kagi": web_search_results = search_web_kagi(search_query, content_country) processed_results = process_web_search_results(web_search_results, "kagi") + return processed_results elif search_engine.lower() == "serper": web_search_results = search_web_serper() processed_results = process_web_search_results(web_search_results, "serper") + return processed_results elif search_engine.lower() == "tavily": web_search_results = search_web_tavily(search_query, result_count, site_blacklist) processed_results = process_web_search_results(web_search_results, "tavily") + return processed_results elif search_engine.lower() == "searx": web_search_results = search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') processed_results = process_web_search_results(web_search_results, "bing") + return processed_results elif search_engine.lower() == "yandex": web_search_results = search_web_yandex() processed_results = process_web_search_results(web_search_results, "bing") + return processed_results else: return f"Error: Invalid Search Engine Name {search_engine}" diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index b66aca73e..2ff589eb9 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -3,6 +3,8 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. +Search is performed -> Results obtained, each individual item is first analyzed based on snippet, if relevant, entire page is fetched and analyzed, this is then stored in the results dictionary, and the process is repeated until all results are analyzed/limit is hit. +Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. Pipeline: 1. User posts question @@ -47,35 +49,41 @@ Pipeline: ---------------- ### Setting the Stage -- The goal of this module is to provide a simple, easy-to-use interface for searching the web and retrieving results. -- All the web searches are simple HTTP requests to an API or to the direct endpoint and then scraping the results. -- Results are then reviewed for relevancy, if relevant, the full page is fetched and analyzed. -- The results are then stored in a dictionary, and the process is repeated until all results are analyzed/limit is hit. -- Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. -- The goal is to provide a simple, easy-to-use interface for searching the web and retrieving results. -- Other modules are responsible for anything else, this module just performs the search, and delivers the results. -- **Main Function:** +- **Purpose:** + - The goal of this module is to provide a simple, easy-to-use interface for searching the web and retrieving results. + - All the web searches are simple HTTP requests to an API or to the direct endpoint and then scraping the results. + - Results are then reviewed for relevancy, if relevant, the full page is fetched and analyzed. + - The results are then stored in a dictionary, and the process is repeated until all results are analyzed/limit is hit. + - Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. + - The goal is to provide a simple, easy-to-use interface for searching the web and retrieving results. + - Other modules are responsible for anything else, this module just performs the search, and delivers the results. +- **Main Function:** (Start Here) + - `def process_question(question: str, search_params: Dict) -> Dict:` + - `question` - The question to search for + - `search_params` - A dictionary containing the search parameters + + +- **Function Execution Steps:** - `def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None)` - - `search_engine` - The search engine to use for the search - - `search_query` - The query to search for - - `content_country` - The country of the content to search for - - `search_lang` - The language to use for the search - - `output_lang` - The language to use for the output - - `result_count` - The number of results to return - - `date_range` - The date range to search within - - `safesearch` - Whether to enable safe search - - `site_blacklist` - A list of sites to exclude from the search results - - `exactTerms` - Terms that must be in the search results - - `excludeTerms` - Terms that must not be in the search results - - `filter` - A filter to apply to the search results - - `geolocation` - The geolocation to use for the search - - `search_result_language` - The language to use for the search results - - `sort_results_by` - How to sort the search results + - `search_engine` - The search engine to use for the search + - `search_query` - The query to search for + - `content_country` - The country of the content to search for + - `search_lang` - The language to use for the search + - `output_lang` - The language to use for the output + - `result_count` - The number of results to return + - `date_range` - The date range to search within + - `safesearch` - Whether to enable safe search + - `site_blacklist` - A list of sites to exclude from the search results + - `exactTerms` - Terms that must be in the search results + - `excludeTerms` - Terms that must not be in the search results + - `filter` - A filter to apply to the search results + - `geolocation` - The geolocation to use for the search + - `search_result_language` - The language to use for the search results + - `sort_results_by` - How to sort the search results - **Returns:** A list of search results as a dictionary. - FIXME: Define the structure of the dictionary - - Each result should contain the title, URL, content, and metadata of the search result. + - Each result should contain the title, URL, content, and metadata of the search result. + -Search is performed -> Results obtained, each individual item is first analyzed based on snippet, if relevant, entire page is fetched and analyzed, this is then stored in the results dictionary, and the process is repeated until all results are analyzed/limit is hit. -Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. https://github.com/scrapinghub/article-extraction-benchmark From 65f9ef19a4f068da4eb25805c67aebfef14e412c Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 13:01:59 -0800 Subject: [PATCH 35/88] Fix the github workflow error? --- .../Local_LLM/Local_LLM_huggingface.py | 11 + App_Function_Libraries/Utils/Utils.py | 2 +- .../Web_Scraping/WebSearch_APIs.py | 330 +++++++++--------- Docs/Design/3rd-Party.md | 4 + Docs/Design/Education.md | 2 +- Docs/Design/Inference_Engines.md | 6 + Docs/Design/Researcher.md | 1 + Docs/Design/Structured_Outputs.md | 1 + 8 files changed, 186 insertions(+), 171 deletions(-) create mode 100644 Docs/Design/3rd-Party.md create mode 100644 Docs/Design/Inference_Engines.md diff --git a/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py b/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py index 2c17e2452..1ca29b6c7 100644 --- a/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py +++ b/App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py @@ -12,6 +12,17 @@ # # Functions: +# FIXME: This function is not complete +# Setup proper path/configurations for the models +HF_MODELS_DIR = "models" + +# FIXME: This function is not complete +def get_local_models(): + if not os.path.exists(HF_MODELS_DIR): + os.makedirs(HF_MODELS_DIR) + return [d for d in os.listdir(HF_MODELS_DIR) if os.path.isdir(os.path.join(HF_MODELS_DIR, d))] + + def chat_with_transformers(user_message, system_message, model_name=None, model_path=None, max_new_tokens=100): pass diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index fefd1f412..bf5f5dd2d 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -553,7 +553,7 @@ def load_and_log_configs(): # Setup Default API Endpoint loaded_config_data = load_and_log_configs() -default_api_endpoint = loaded_config_data['default_api'] +default_api_endpoint = loaded_config_data.get['default_api', 'openai'] def format_api_name(api): name_mapping = { diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index dfa8766d8..835c8ae90 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -25,23 +25,171 @@ ####################################################################################################################### # # Functions: -# 1. perform_websearch -# 2. search_web_baidu -# 3. search_web_bing -# 4. search_web_brave -# 5. search_web_duckduckgo -# 6. search_web_google -# 7. search_web_kagi -# 8. search_web_serper -# 9. search_web_tavily -# 10. search_web_searx -# 11. search_web_yandex -# 12. parse_html_search_results_generic +# 1. analyze_question # ####################################################################################################################### # # Functions: +######################### Main Orchestration Workflow ######################### +# +def process_question(question: str, search_params: Dict) -> Dict: + """ + Orchestrates the entire pipeline: + 1. Optionally generate sub-queries (if subquery_generation=True). + 2. Perform web searches for each query (the original and sub-queries). + 3. Score results for relevance; filter them. + 4. Aggregate final answer from relevant results. + + Args: + question (str): The user's original question or query. + search_params (Dict): A dictionary containing parameters for performing web searches + and specifying LLM endpoints. + + Returns: + Dict: A dictionary containing all relevant data, including results from each sub-query. + + A dictionary containing parameters for performing a web search and processing the results. + + Dict Parameters: + engine (str): The search engine to use (e.g., "google", "bing", "brave", "duckduckgo", etc.). + content_country (str): The country code for content localization (e.g., "US", "UK", "DE"). + search_lang (str): The language for the search query (e.g., "en" for English). + output_lang (str): The desired language for the search results. + result_count (int): The number of results to return. + date_range (str or None): The time range for the search results (e.g., "y" for the past year). + safesearch (str or None): The safe search setting (e.g., "moderate", "strict", "off"). + site_blacklist (list or None): A list of sites to exclude from the search results. + exactTerms (str or None): Terms that must appear in the search results. + excludeTerms (str or None): Terms that must not appear in the search results. + filter (str or None): Any additional filtering criteria. + geolocation (str or None): Geographical location for search results. + search_result_language (str or None): The language of the search results. + sort_results_by (str or None): Criteria for sorting the results. + subquery_generation (bool): Whether to generate sub-queries. + subquery_generation_api (str): The API to use for sub-query generation (e.g., "openai", "anthropic", "deepseek"). + relevance_analysis_llm (str): The LLM model to use for relevance analysis (e.g., "openai", "anthropic", "deepseek"). + final_answer_llm (str): The LLM model to use for generating the final answer (e.g., "openai", "anthropic", "deepseek"). + + Example: + search_params = { + "engine": "google", + "content_country": "US", + "search_lang": "en", + "output_lang": "en", + "result_count": 10, + "date_range": "y", + "safesearch": "moderate", + "site_blacklist": ["example.com", "spam-site.com"], + "exactTerms": None, + "excludeTerms": None, + "filter": None, + "geolocation": None, + "search_result_language": None, + "sort_results_by": None, + "subquery_generation": True, + "subquery_generation_llm": "openai", + "relevance_analysis_llm": "openai", + "final_answer_llm": "openai" + } + """ + logging.info(f"Starting process_question with query: {question}") + + # 1. Generate sub-queries if requested + sub_queries = [] + sub_query_dict = { + "main_goal": question, + "sub_questions": [], + "search_queries": [], + "analysis_prompt": None + } + + if search_params.get("subquery_generation", False): + api_endpoint = search_params.get("subquery_generation_llm", "openai") + sub_query_dict = analyze_question(question, api_endpoint) + sub_queries = sub_query_dict.get("sub_questions", []) + + # Merge original question with sub-queries + all_queries = [question] + sub_queries + + # 2. Perform searches and accumulate all raw results + all_results: List[Dict] = [] + for q in all_queries: + # FIXME - change raw_results to proper dict format + raw_results = perform_websearch( + search_engine=search_params.get('engine', 'google'), + search_query=q, + content_country=search_params.get('content_country', 'US'), + search_lang=search_params.get('search_lang', 'en'), + output_lang=search_params.get('output_lang', 'en'), + result_count=search_params.get('result_count', 10), + date_range=search_params.get('date_range'), + safesearch=search_params.get('safesearch', 'moderate'), + site_blacklist=search_params.get('site_blacklist', []), + exactTerms=search_params.get('exactTerms'), + excludeTerms=search_params.get('excludeTerms'), + filter=search_params.get('filter'), + geolocation=search_params.get('geolocation'), + search_result_language=search_params.get('search_result_language'), + sort_results_by=search_params.get('sort_results_by') + ) + + # Validate raw_results + # FIXME - Account for proper structure of returned web_search_results_dict dictionary + if not isinstance(raw_results, dict) or "processing_error" in raw_results: + logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") + continue + + results_list = raw_results.get("results", []) + all_results.extend(results_list) + + # 3. Score/filter (placeholder) + relevant_results = {} + for r in all_results: + # FIXME - Put in proper args / Ensure this works + # search_results: List[Dict], + # original_question: str, + # sub_questions: List[str], + # api_endpoint: str + list_of_relevant_articles = search_result_relevance( + all_results, + question, + sub_query_dict['sub_questions'], + search_params.get('relevance_analysis_llm') + ) + if list_of_relevant_articles: + relevant_results[r] = list_of_relevant_articles + + + + # 4. Summarize/aggregate final answer + final_answer = aggregate_results( + # FIXME - Add proper Args / Ensure this works + # relevant_results: Dict[str, Dict], + # question: str, + # sub_questions: List[str], + # api_endpoint: str + # FIXME - Proper datatypes/expectations + relevant_results, + question, + sub_query_dict['sub_questions'], + api_endpoint=search_params.get('final_answer_llm') + ) + + # Return the final data + # FIXME - Return full query details for debugging and analysis + return { + "search_engine": search_params.get('engine'), + "original_query": question, + "sub_questions": sub_query_dict.get('sub_questions', []), + "all_queries": all_queries, + "results": all_results, + "relevant_results": relevant_results, + "analysis_prompt": sub_query_dict.get('analysis_prompt'), + "final_answer": final_answer + } + + ######################### Question Analysis ######################### # # @@ -301,163 +449,6 @@ def aggregate_results( } -######################### Main Orchestration Workflow ######################### -# -def process_question(question: str, search_params: Dict) -> Dict: - """ - Orchestrates the entire pipeline: - 1. Optionally generate sub-queries (if subquery_generation=True). - 2. Perform web searches for each query (the original and sub-queries). - 3. Score results for relevance; filter them. - 4. Aggregate final answer from relevant results. - - Args: - question (str): The user's original question or query. - search_params (Dict): A dictionary containing parameters for performing web searches - and specifying LLM endpoints. - - Returns: - Dict: A dictionary containing all relevant data, including results from each sub-query. - - A dictionary containing parameters for performing a web search and processing the results. - - Dict Parameters: - engine (str): The search engine to use (e.g., "google", "bing", "brave", "duckduckgo", etc.). - content_country (str): The country code for content localization (e.g., "US", "UK", "DE"). - search_lang (str): The language for the search query (e.g., "en" for English). - output_lang (str): The desired language for the search results. - result_count (int): The number of results to return. - date_range (str or None): The time range for the search results (e.g., "y" for the past year). - safesearch (str or None): The safe search setting (e.g., "moderate", "strict", "off"). - site_blacklist (list or None): A list of sites to exclude from the search results. - exactTerms (str or None): Terms that must appear in the search results. - excludeTerms (str or None): Terms that must not appear in the search results. - filter (str or None): Any additional filtering criteria. - geolocation (str or None): Geographical location for search results. - search_result_language (str or None): The language of the search results. - sort_results_by (str or None): Criteria for sorting the results. - subquery_generation (bool): Whether to generate sub-queries. - subquery_generation_api (str): The API to use for sub-query generation (e.g., "openai", "anthropic", "deepseek"). - relevance_analysis_llm (str): The LLM model to use for relevance analysis (e.g., "openai", "anthropic", "deepseek"). - final_answer_llm (str): The LLM model to use for generating the final answer (e.g., "openai", "anthropic", "deepseek"). - - Example: - search_params = { - "engine": "google", - "content_country": "US", - "search_lang": "en", - "output_lang": "en", - "result_count": 10, - "date_range": "y", - "safesearch": "moderate", - "site_blacklist": ["example.com", "spam-site.com"], - "exactTerms": None, - "excludeTerms": None, - "filter": None, - "geolocation": None, - "search_result_language": None, - "sort_results_by": None, - "subquery_generation": True, - "subquery_generation_llm": "openai", - "relevance_analysis_llm": "openai", - "final_answer_llm": "openai" - } - """ - logging.info(f"Starting process_question with query: {question}") - - # 1. Generate sub-queries if requested - sub_queries = [] - sub_query_dict = { - "main_goal": question, - "sub_questions": [], - "search_queries": [], - "analysis_prompt": None - } - - if search_params.get("subquery_generation", False): - api_endpoint = search_params.get("subquery_generation_llm", "openai") - sub_query_dict = analyze_question(question, api_endpoint) - sub_queries = sub_query_dict.get("sub_questions", []) - - # Merge original question with sub-queries - all_queries = [question] + sub_queries - - # 2. Perform searches and accumulate all raw results - all_results: List[Dict] = [] - for q in all_queries: - raw_results = perform_websearch( - search_engine=search_params.get('engine', 'google'), - search_query=q, - content_country=search_params.get('content_country', 'US'), - search_lang=search_params.get('search_lang', 'en'), - output_lang=search_params.get('output_lang', 'en'), - result_count=search_params.get('result_count', 10), - date_range=search_params.get('date_range'), - safesearch=search_params.get('safesearch', 'moderate'), - site_blacklist=search_params.get('site_blacklist', []), - exactTerms=search_params.get('exactTerms'), - excludeTerms=search_params.get('excludeTerms'), - filter=search_params.get('filter'), - geolocation=search_params.get('geolocation'), - search_result_language=search_params.get('search_result_language'), - sort_results_by=search_params.get('sort_results_by') - ) - - # Validate raw_results - if not isinstance(raw_results, dict) or "processing_error" in raw_results: - logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") - continue - - results_list = raw_results.get("results", []) - all_results.extend(results_list) - - # 3. Score/filter (placeholder) - relevant_results = {} - for r in all_results: - # FIXME - Put in proper args / Ensure this works - # search_results: List[Dict], - # original_question: str, - # sub_questions: List[str], - # api_endpoint: str - list_of_relevant_articles = search_result_relevance( - all_results, - question, - sub_query_dict['sub_questions'], - search_params.get('relevance_analysis_llm') - ) - if list_of_relevant_articles: - relevant_results[r] = list_of_relevant_articles - - - - # 4. Summarize/aggregate final answer - final_answer = aggregate_results( - # FIXME - Add proper Args / Ensure this works - # relevant_results: Dict[str, Dict], - # question: str, - # sub_questions: List[str], - # api_endpoint: str - # FIXME - Proper datatypes/expectations - relevant_results, - question, - sub_query_dict['sub_questions'], - api_endpoint=search_params.get('final_answer_llm') - ) - - # Return the final data - # FIXME - Return full query details for debugging and analysis - return { - "search_engine": search_params.get('engine'), - "original_query": question, - "sub_questions": sub_query_dict.get('sub_questions', []), - "all_queries": all_queries, - "results": all_results, - "relevant_results": relevant_results, - "analysis_prompt": sub_query_dict.get('analysis_prompt'), - "final_answer": final_answer - } - - # # End of Orchestration functions ####################################################################################################################### @@ -467,6 +458,7 @@ def process_question(question: str, search_params: Dict) -> Dict: # # Search Engine Functions +# FIXME def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): if search_engine.lower() == "baidu": diff --git a/Docs/Design/3rd-Party.md b/Docs/Design/3rd-Party.md new file mode 100644 index 000000000..1834a4b42 --- /dev/null +++ b/Docs/Design/3rd-Party.md @@ -0,0 +1,4 @@ +# 3rd Party Stuff + + +https://github.com/cohere-ai/quick-start-connectors \ No newline at end of file diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index fd5957474..fbe3d99d0 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -9,7 +9,7 @@ https://arxiv.org/abs/2411.07407 https://arxiv.org/abs/2412.16429 https://huggingface.co/papers/2412.15443 https://github.com/thiswillbeyourgithub/AnkiAIUtils - +https://news.ycombinator.com/item?id=42534931 diff --git a/Docs/Design/Inference_Engines.md b/Docs/Design/Inference_Engines.md new file mode 100644 index 000000000..25cfb781e --- /dev/null +++ b/Docs/Design/Inference_Engines.md @@ -0,0 +1,6 @@ +# Inference Engines + + + +### Link Dump +https://github.com/intel/neural-compressor/tree/v3.2 \ No newline at end of file diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index e399a64db..0e3c9f5ef 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -138,6 +138,7 @@ https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama https://github.com/rashadphz/farfalle/blob/main/src/backend/agent_search.py https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py https://github.com/stanford-oval/storm/blob/main/examples/storm_examples/run_storm_wiki_claude.py +https://learning.google.com/experiments/learn-about diff --git a/Docs/Design/Structured_Outputs.md b/Docs/Design/Structured_Outputs.md index 7643ac475..01a158bac 100644 --- a/Docs/Design/Structured_Outputs.md +++ b/Docs/Design/Structured_Outputs.md @@ -41,6 +41,7 @@ Tools https://github.com/outlines-dev/outlines https://github.com/Dan-wanna-M/formatron/tree/master https://github.com/whyhow-ai/knowledge-table + https://github.com/guardrails-ai/guardrails Examples https://github.com/dottxt-ai/demos/tree/main/lore-generator From b4a74bc105d61fb6c14b1d58e122b02acf8ff7a5 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 14:21:52 -0800 Subject: [PATCH 36/88] ? --- App_Function_Libraries/Utils/Utils.py | 2 +- Docs/Design/TTS_STT.md | 24 +++++++++++++++++++++++- Tests/ChromaDB/test_chromadb.py | 6 ++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index bf5f5dd2d..fefd1f412 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -553,7 +553,7 @@ def load_and_log_configs(): # Setup Default API Endpoint loaded_config_data = load_and_log_configs() -default_api_endpoint = loaded_config_data.get['default_api', 'openai'] +default_api_endpoint = loaded_config_data['default_api'] def format_api_name(api): name_mapping = { diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 6ee8cd852..090b65720 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -366,4 +366,26 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) -``` \ No newline at end of file +``` + + +### GPT-SoVITS + +- [GPT-SoVITS](f) + +https://tts.x86.st/ +Finetuning is very quick (about 5 minutes). Captioning of audio was automated with faster-whisper (it is required that the audio is captioned). +With the default batch size of 12, training takes 9.5~ GB. + +- **101** + - F +- **Setup** + - F +- **Training** + - F +- **Inference** + - F +- **Fine-Tuning** + - F + + diff --git a/Tests/ChromaDB/test_chromadb.py b/Tests/ChromaDB/test_chromadb.py index f3526f6b4..3ed7e20f8 100644 --- a/Tests/ChromaDB/test_chromadb.py +++ b/Tests/ChromaDB/test_chromadb.py @@ -7,6 +7,9 @@ from unittest.mock import patch, MagicMock # Third-party library imports import pytest + +from App_Function_Libraries.Utils.Utils import load_and_log_configs + # #################################################################################################### # @@ -29,6 +32,9 @@ # Fixtures for Reusable Mocking and Setup ############################################ +loaded_config_data = load_and_log_configs() +default_api_endpoint = loaded_config_data['default_api'] + # Fixture to mock a ChromaDB collection @pytest.fixture def mock_collection(): From 49c47f23ec4bd620d25e8c58b8e1c69ca63a37e8 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 15:19:05 -0800 Subject: [PATCH 37/88] Update test_chromadb.py --- Tests/ChromaDB/test_chromadb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/ChromaDB/test_chromadb.py b/Tests/ChromaDB/test_chromadb.py index 3ed7e20f8..2a45a611c 100644 --- a/Tests/ChromaDB/test_chromadb.py +++ b/Tests/ChromaDB/test_chromadb.py @@ -8,7 +8,7 @@ # Third-party library imports import pytest -from App_Function_Libraries.Utils.Utils import load_and_log_configs + # #################################################################################################### @@ -24,9 +24,9 @@ # Local Imports from App_Function_Libraries.RAG.ChromaDB_Library import ( process_and_store_content, check_embedding_status, - reset_chroma_collection, vector_search, store_in_chroma, batched, situate_context, schedule_embedding, - embedding_api_url + reset_chroma_collection, vector_search, store_in_chroma, batched, embedding_api_url ) +from App_Function_Libraries.Utils.Utils import load_and_log_configs # ############################################ # Fixtures for Reusable Mocking and Setup From c029244ac9a88bd6d18277217b6b91b97d820971 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 15:55:23 -0800 Subject: [PATCH 38/88] Update test_chromadb.py --- Tests/ChromaDB/test_chromadb.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tests/ChromaDB/test_chromadb.py b/Tests/ChromaDB/test_chromadb.py index 2a45a611c..2235360d5 100644 --- a/Tests/ChromaDB/test_chromadb.py +++ b/Tests/ChromaDB/test_chromadb.py @@ -33,8 +33,12 @@ ############################################ loaded_config_data = load_and_log_configs() +if loaded_config_data is None: + raise ValueError("Failed to load configuration data") + default_api_endpoint = loaded_config_data['default_api'] + # Fixture to mock a ChromaDB collection @pytest.fixture def mock_collection(): From af28e7bc510b82d0aa159d74a651a5550d675ae7 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 16:27:20 -0800 Subject: [PATCH 39/88] Update test_chromadb.py --- Tests/ChromaDB/test_chromadb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/ChromaDB/test_chromadb.py b/Tests/ChromaDB/test_chromadb.py index 2235360d5..17be2708c 100644 --- a/Tests/ChromaDB/test_chromadb.py +++ b/Tests/ChromaDB/test_chromadb.py @@ -22,11 +22,12 @@ print(f"Project root added to sys.path: {project_root}") # Local Imports +from App_Function_Libraries.Utils.Utils import load_and_log_configs from App_Function_Libraries.RAG.ChromaDB_Library import ( process_and_store_content, check_embedding_status, reset_chroma_collection, vector_search, store_in_chroma, batched, embedding_api_url ) -from App_Function_Libraries.Utils.Utils import load_and_log_configs + # ############################################ # Fixtures for Reusable Mocking and Setup From f1f7d3a2a0457a4b956610ce44a581034a628ddb Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 16:39:31 -0800 Subject: [PATCH 40/88] Update Utils.py --- App_Function_Libraries/Utils/Utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index fefd1f412..d21722c8c 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -552,8 +552,13 @@ def load_and_log_configs(): openai_tts_voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] # Setup Default API Endpoint -loaded_config_data = load_and_log_configs() -default_api_endpoint = loaded_config_data['default_api'] +try: + loaded_config_data = load_and_log_configs() + default_api_endpoint = loaded_config_data['default_api'] + print(f"Default API Endpoint: {default_api_endpoint}") +except Exception as e: + logging.error(f"Error loading default API endpoint: {str(e)}") + default_api_endpoint = "openai" def format_api_name(api): name_mapping = { @@ -577,9 +582,6 @@ def format_api_name(api): } return name_mapping.get(api, api.title()) - -print(f"Default API Endpoint: {default_api_endpoint}") - # # End of Config loading ####################################################################################################################### From e4a8a54dcaa30ea00afde2a0247b1ecb0353e873 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 16:49:42 -0800 Subject: [PATCH 41/88] Update python-app.yml --- .github/workflows/python-app.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 7e4f1ad98..affad105c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -33,8 +33,7 @@ jobs: - name: Test ChromaDB lib functions with pytest run: | pwd - cd ./Tests/ChromaDB - pytest test_chromadb.py + pytest ./Tests/ChromaDBtest_chromadb.py - name: Test RAG lib functions with pytest run: | From 88d8c67901d8d9039703f76ba60e9f013d789859 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 16:54:10 -0800 Subject: [PATCH 42/88] Update python-app.yml --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index affad105c..cc139c8d2 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -33,7 +33,7 @@ jobs: - name: Test ChromaDB lib functions with pytest run: | pwd - pytest ./Tests/ChromaDBtest_chromadb.py + pytest ./Tests/ChromaDB/test_chromadb.py - name: Test RAG lib functions with pytest run: | From 5cc8181d9f2417e3c476d0d1a21ab6b51af66a6e Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 17:05:26 -0800 Subject: [PATCH 43/88] Update python-app.yml --- .github/workflows/python-app.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index cc139c8d2..e373d2698 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -33,7 +33,10 @@ jobs: - name: Test ChromaDB lib functions with pytest run: | pwd - pytest ./Tests/ChromaDB/test_chromadb.py + ls + ls ./Config_Files + cd ./Tests/ChromaDB/ + pytest ./test_chromadb.py - name: Test RAG lib functions with pytest run: | From 0c26bfd4092eafbd847dd9da40d9b3be226fd6b3 Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 17:30:19 -0800 Subject: [PATCH 44/88] amithatretarded --- Docs/Design/Education.md | 2 +- Tests/ChromaDB/test_chromadb.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index fbe3d99d0..b72a4a1d9 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -10,7 +10,7 @@ https://arxiv.org/abs/2412.16429 https://huggingface.co/papers/2412.15443 https://github.com/thiswillbeyourgithub/AnkiAIUtils https://news.ycombinator.com/item?id=42534931 - +https://ankiweb.net/shared/info/1531888719 diff --git a/Tests/ChromaDB/test_chromadb.py b/Tests/ChromaDB/test_chromadb.py index 17be2708c..3bcd0ccef 100644 --- a/Tests/ChromaDB/test_chromadb.py +++ b/Tests/ChromaDB/test_chromadb.py @@ -22,7 +22,7 @@ print(f"Project root added to sys.path: {project_root}") # Local Imports -from App_Function_Libraries.Utils.Utils import load_and_log_configs +#from App_Function_Libraries.Utils.Utils import load_and_log_configs from App_Function_Libraries.RAG.ChromaDB_Library import ( process_and_store_content, check_embedding_status, reset_chroma_collection, vector_search, store_in_chroma, batched, embedding_api_url @@ -33,11 +33,7 @@ # Fixtures for Reusable Mocking and Setup ############################################ -loaded_config_data = load_and_log_configs() -if loaded_config_data is None: - raise ValueError("Failed to load configuration data") - -default_api_endpoint = loaded_config_data['default_api'] +default_api_endpoint = "openai" # Fixture to mock a ChromaDB collection From c4ecdef36a88bf5cf52c84da1622b014204810de Mon Sep 17 00:00:00 2001 From: Robert Date: Sun, 29 Dec 2024 18:13:32 -0800 Subject: [PATCH 45/88] Wow. I'll take that as a yes. --- App_Function_Libraries/Utils/Utils.py | 2 + .../Web_Scraping/WebSearch_APIs.py | 101 ++++++---- Docs/Design/WebSearch.md | 177 ++++++++---------- Docs/Issues/Evaluation_Plans.md | 1 + 4 files changed, 146 insertions(+), 135 deletions(-) diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index d21722c8c..0c3cef517 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -551,6 +551,7 @@ def load_and_log_configs(): openai_tts_voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] + # Setup Default API Endpoint try: loaded_config_data = load_and_log_configs() @@ -560,6 +561,7 @@ def load_and_log_configs(): logging.error(f"Error loading default API endpoint: {str(e)}") default_api_endpoint = "openai" + def format_api_name(api): name_mapping = { "openai": "OpenAI", diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 835c8ae90..b88fbaa98 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -160,34 +160,21 @@ def process_question(question: str, search_params: Dict) -> Dict: if list_of_relevant_articles: relevant_results[r] = list_of_relevant_articles - - # 4. Summarize/aggregate final answer final_answer = aggregate_results( # FIXME - Add proper Args / Ensure this works - # relevant_results: Dict[str, Dict], - # question: str, - # sub_questions: List[str], - # api_endpoint: str + # web_search_results_dict: Dict, + # report_language: str, + # aggregation_api_endpoint: str # FIXME - Proper datatypes/expectations - relevant_results, - question, - sub_query_dict['sub_questions'], + web_search_results_dict, + report_language=search_params.get('output_lang', 'en'), api_endpoint=search_params.get('final_answer_llm') ) # Return the final data # FIXME - Return full query details for debugging and analysis - return { - "search_engine": search_params.get('engine'), - "original_query": question, - "sub_questions": sub_query_dict.get('sub_questions', []), - "all_queries": all_queries, - "results": all_results, - "relevant_results": relevant_results, - "analysis_prompt": sub_query_dict.get('analysis_prompt'), - "final_answer": final_answer - } + return web_search_results_dict ######################### Question Analysis ######################### @@ -532,6 +519,42 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict Returns: Dict: A dictionary containing the processed search results in the specified structure. + + web_search_results_dict = { + "search_engine": search_engine, + "search_query": search_results.get("search_query", ""), + "content_country": search_results.get("content_country", ""), + "search_lang": search_results.get("search_lang", ""), + "output_lang": search_results.get("output_lang", ""), + "result_count": search_results.get("result_count", 0), + "date_range": search_results.get("date_range", None), + "safesearch": search_results.get("safesearch", None), + "site_blacklist": search_results.get("site_blacklist", None), + "exactTerms": search_results.get("exactTerms", None), + "excludeTerms": search_results.get("excludeTerms", None), + "filter": search_results.get("filter", None), + "geolocation": search_results.get("geolocation", None), + "search_result_language": search_results.get("search_result_language", None), + "sort_results_by": search_results.get("sort_results_by", None), + "results": [ + { + "title": str, + "url": str, + "content": str, + "metadata": { + "date_published": Optional[str], + "author": Optional[str], + "source": Optional[str], + "language": Optional[str], + "relevance_score": Optional[float], + "snippet": Optional[str] + } + }, + "total_results_found": search_results.get("total_results_found", 0), + "search_time": search_results.get("search_time", 0.0), + "error": search_results.get("error", None), + "processing_error": None + } """ # Initialize the output dictionary with default values web_search_results_dict = { @@ -550,7 +573,21 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict "geolocation": search_results.get("geolocation", None), "search_result_language": search_results.get("search_result_language", None), "sort_results_by": search_results.get("sort_results_by", None), - "results": [], + "results": [ + { + "title": str, + "url": str, + "content": str, + "metadata": { + "date_published": Optional[str], + "author": Optional[str], + "source": Optional[str], + "language": Optional[str], + "relevance_score": Optional[float], + "snippet": Optional[str] + } + }, + ], "total_results_found": search_results.get("total_results_found", 0), "search_time": search_results.get("search_time", 0.0), "error": search_results.get("error", None), @@ -560,40 +597,32 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict if search_engine.lower() == "baidu": pass elif search_engine.lower() == "bing": - parse_bing_results(search_results, web_search_results_dict) - pass + parsed_results = parse_bing_results(search_results, web_search_results_dict) elif search_engine.lower() == "brave": parse_brave_results(search_results, web_search_results_dict) pass elif search_engine.lower() == "duckduckgo": - parse_duckduckgo_results(search_results, web_search_results_dict) - pass + parsed_results = parse_duckduckgo_results(search_results, web_search_results_dict) elif search_engine.lower() == "google": - parse_google_results(search_results, web_search_results_dict) - pass + parsed_results = parse_google_results(search_results, web_search_results_dict) elif search_engine.lower() == "kagi": - parse_kagi_results(search_results, web_search_results_dict) - pass + parsed_results = parse_kagi_results(search_results, web_search_results_dict) elif search_engine.lower() == "serper": - parse_serper_results(search_results, web_search_results_dict) - pass + parsed_results = parse_serper_results(search_results, web_search_results_dict) elif search_engine.lower() == "tavily": - parse_tavily_results(search_results, web_search_results_dict) - pass + parsed_results = parse_tavily_results(search_results, web_search_results_dict) elif search_engine.lower() == "searx": - parse_searx_results(search_results, web_search_results_dict) - pass + parsed_results = parse_searx_results(search_results, web_search_results_dict) elif search_engine.lower() == "yandex": - parse_yandex_results(search_results, web_search_results_dict) - pass + parsed_results = parse_yandex_results(search_results, web_search_results_dict) else: web_search_results_dict["processing_error"] = f"Error: Invalid Search Engine Name {search_engine}" diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 2ff589eb9..928352ff8 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -2,65 +2,46 @@ ## Introduction This page serves as documentation regarding the web search functionality within tldw and provides context/justification for the decisions made within the module. - -Search is performed -> Results obtained, each individual item is first analyzed based on snippet, if relevant, entire page is fetched and analyzed, this is then stored in the results dictionary, and the process is repeated until all results are analyzed/limit is hit. -Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. - -Pipeline: -1. User posts question - - Gradio/UI/API -2. Question is analyzed - - Question is analyzed to identify most likely purpose/goal of question, and Sub-questions are generated to support this - - User has option of seeing/modifying prompt used for Analysis/sub-question creation -3. Search(es) is/are performed - User gated - - Search is performed using the user's question and sub-questions -4. Results are collected, stored, and analyzed - - Results are collected, stored in a 'search_results' dict, and analyzed for relevance, based on initial snippet(? or full page?) - - User has the option of seeing all results, removing irrelevant results, and selecting which results are 'relevant' - - User also has the option to select which 'relevant' results are used to answer the question -5. Irrelevant results are removed from the dictionary - - Results determined to be relevant are then stored in a 'relevant_results' dictionary, and the process is repeated until all results are analyzed/limit is hit. -6. Once all results are collected, they are then used to answer the user's question/sub-questions - - The remaining relevant results are then used to answer the user's question/sub-questions - - Each result is abstractly summarized, and then combined into a single document and supplied to the LLM for final analysis -7. The final answer/'briefing' is then presented to the user -8. User has the option to save the results to the DB -9. User has the option to ask follow-up questions / see potential other questions - - -- **Text Search Workflow** - 1. User inputs a search query - 2. User selects a search engine (Option for default search engine in config file) - 3. The user presses 'Search' - 4. The search query is passed to the selected search engine - 5. The appropriate search engine is used to perform a search via API call - 6. The search results are returned from the search engine's API - 7. Search engine results are then _MODIFIED_ (if necessary/enabled) to fit the user's preferences - - This could include re-ranking, summarization/analysis, or other modifications - 8. The (modified) search results are displayed to the user - 9. Results are then displayed to the user, - - either as titles of pages with dropdown for all info, - - or as a list of links with a briefing/summary of each link - - or as a single briefing/summary of all results - 10. User may then select to save this resulting text to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information - 11. Search results are then saved to the DB as a plaintext entry, with metadata containing the search query, search engine, and any other relevant information - 12. This is then searchable via the Media DB +- **High-Level Workflow** + 1. User inputs a search query. + 2. User selects a search engine (Option for default search engine + default query options in config file). + 3. User may select `Advanced Search` for additional search parameters (Language, Date Range, etc). + 4. The user presses 'Search'. + 5. Search is performed -> Results obtained, + 6. Each individual item is first analyzed based on snippet, if relevant, entire page is fetched and analyzed, this is then stored in the results dictionary, and the process is repeated until all results are analyzed/limit is hit. + 7. Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. + 8. The final product is then passed back to the UI for display to the user. ---------------- ### Setting the Stage -- **Purpose:** - - The goal of this module is to provide a simple, easy-to-use interface for searching the web and retrieving results. - - All the web searches are simple HTTP requests to an API or to the direct endpoint and then scraping the results. - - Results are then reviewed for relevancy, if relevant, the full page is fetched and analyzed. - - The results are then stored in a dictionary, and the process is repeated until all results are analyzed/limit is hit. - - Once all results are collected, they are then operated on, being used to create whatever final product is desired by the user. - - The goal is to provide a simple, easy-to-use interface for searching the web and retrieving results. - - Other modules are responsible for anything else, this module just performs the search, and delivers the results. -- **Main Function:** (Start Here) - - `def process_question(question: str, search_params: Dict) -> Dict:` - - `question` - The question to search for - - `search_params` - A dictionary containing the search parameters + +- **Text Search Workflow** + 1. User inputs a search query. + 2. User selects a search engine + Query options (Option for default search engine + default query options in config file). + 3. The user presses 'Search'. + - Gradio UI Function makes a call to `process_question` with the search query + search parameters as a dictionary + 4. `process_question()` checks the search params dict to see if sub-query creation is enabled, if so, it creates a list of sub-queries based on the search query with a call to `analyze_question()`. + 5. `analyze_question()` takes the search query and generates a list of sub-queries based on the search query, attempts this 3 times, making a call to the LLM API, and then returns the list of sub-queries if successful. + 6. once back in `process_question()`, all queries are combined into a single query list `#L113`. + 7. `process_question()` then iterates through the query list, making a call to `perform_websearch()` with each query in the list, and the matching search parameters. + 8. `perform_websearch()` makes a call to the selected search engine's API with the query and search parameters, and returns the results. + - This function is a `sink' for all search engine API calls, and is responsible for handling calling the appropriate search engine API call + 9. `process_web_search_results()` then takes the results from the search engine, and processes them, converting them into a dictionary of results in the `web_search_results_dict` dictionary. + - FIXME - this is where I lose track of what's happening, need to re-read the code + - This function returns a filled `web_search_results_dict` dictionary + 10. `process_question()` then takes the `web_search_results_dict` and processes it, checking to make sure it is valid and contains results. + - FIXME - Verify this is correct + 11. FIXME - Make it optional to display the results to the user, and allow them to select which results are relevant before continuing processing + 12. `process_question()` then iterates through each search result, checking if it is relevant, and if so, adds it to the `relevant_results_dict` + - FIXME - The results should be added back to the `web_search_results_dict` if they are relevant. + 13. `process_question()` then calls into `aggregate_results()` function with the `web_search_results_dict` + 14. `aggregate_results()` then takes the `web_search_results_dict` and processes it, combining all the results into a single document + - FIXME - This is not implemented yet and also want various options available for this. + 15. `process_question()` then returns the `web_search_results_dict` to the calling function. + 15. The calling function then takes the `web_search_results_dict` and processes it, extracting the final results/aggregated report and presenting it to the user + 16. The user then has the option to save the results to the DB, or ask follow-up questions, etc. + 17. The user can also select which results are relevant, and which are not, and remove irrelevant results from the `web_search_results_dict` - **Function Execution Steps:** @@ -85,7 +66,26 @@ Pipeline: -https://github.com/scrapinghub/article-extraction-benchmark +---------------- +### Web Search API +- TBD +```def perform_websearch(search_engine, + search_query, + content_country, + search_lang, + output_lang, + result_count, + date_range=None, + safesearch=None, + site_blacklist=None, + exactTerms=None, + excludeTerms=None, + filter=None, + geolocation=None, + search_result_language=None, + sort_results_by=None +``` + ask.py https://github.com/pengfeng/ask.py 1. User inputs a search query @@ -183,22 +183,22 @@ Perplexity Results dictionary: ``` -{ - "search_engine": str, - "search_query": str, - "content_country": str, - "search_lang": str, - "output_lang": str, - "result_count": int, - "date_range": Optional[str], - "safesearch": Optional[bool], - "site_blacklist": Optional[List[str]], - "exactTerms": Optional[List[str]], - "excludeTerms": Optional[List[str]], - "filter": Optional[str], - "geolocation": Optional[str], - "search_result_language": Optional[str], - "sort_results_by": Optional[str], +web_search_results_dict = { + "search_engine": search_engine, + "search_query": search_results.get("search_query", ""), + "content_country": search_results.get("content_country", ""), + "search_lang": search_results.get("search_lang", ""), + "output_lang": search_results.get("output_lang", ""), + "result_count": search_results.get("result_count", 0), + "date_range": search_results.get("date_range", None), + "safesearch": search_results.get("safesearch", None), + "site_blacklist": search_results.get("site_blacklist", None), + "exactTerms": search_results.get("exactTerms", None), + "excludeTerms": search_results.get("excludeTerms", None), + "filter": search_results.get("filter", None), + "geolocation": search_results.get("geolocation", None), + "search_result_language": search_results.get("search_result_language", None), + "sort_results_by": search_results.get("sort_results_by", None), "results": [ { "title": str, @@ -213,11 +213,11 @@ Results dictionary: "snippet": Optional[str] } }, - # ... more results ... ], - "total_results_found": int, - "search_time": float, - "error": Optional[str] + "total_results_found": search_results.get("total_results_found", 0), + "search_time": search_results.get("search_time", 0.0), + "error": search_results.get("error", None), + "processing_error": None } ``` @@ -226,6 +226,7 @@ Results dictionary: #### Baidu Search - [Baidu Search API](https://www.baidu.com/) +- Baidu doens't have an official english API, so we'll have to scrape the results or use Serper #### Bing Search @@ -236,6 +237,7 @@ Results dictionary: - If for some reason you're doing modifications to the code(Fuck MS), be aware: https://github.com/Azure-Samples/cognitive-services-REST-api-samples/issues/139 - Perform searches using Bing! + #### Brave Search Two APIs, 1 for 'AI' the other for 'regular' search - [Brave Search API](https://brave.com/search/api/) @@ -282,26 +284,3 @@ Structure/approach taken from https://github.com/deedy5/duckduckgo_search #### PubMedCentral Search - https://www.ncbi.nlm.nih.gov/home/develop/api/ - - - ----------------- -### Web Search API -- TBD -```def perform_websearch(search_engine, - search_query, - content_country, - search_lang, - output_lang, - result_count, - date_range=None, - safesearch=None, - site_blacklist=None, - exactTerms=None, - excludeTerms=None, - filter=None, - geolocation=None, - search_result_language=None, - sort_results_by=None -``` - diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 289f9715a..14a993d64 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -48,6 +48,7 @@ https://arxiv.org/abs/2412.17758 https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard https://www.atla-ai.com/post/evaluating-the-evaluator https://hamel.dev/blog/posts/llm-judge/ +https://github.com/scrapinghub/article-extraction-benchmark Have LLMs play Social deception games From 7a9e25ad840786549f9ec47ad891318859452de7 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 00:07:41 -0800 Subject: [PATCH 46/88] more progress on perplexity --- .../Web_Scraping/WebSearch_APIs.py | 387 +++++++++++------- Docs/Design/Education.md | 2 +- Docs/Design/TTS_STT.md | 27 +- 3 files changed, 261 insertions(+), 155 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index b88fbaa98..2c3e9e53f 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -31,6 +31,33 @@ # # Functions: +# FIXME - Add Logging + +def test_process_question(): + results = process_question("What is the capital of France?", { + "engine": "google", + "content_country": "US", + "search_lang": "en", + "output_lang": "en", + "result_count": 10, + "date_range": "y", + "safesearch": "moderate", + "site_blacklist": ["example.com", "spam-site.com"], + "exactTerms": None, + "excludeTerms": None, + "filter": None, + "geolocation": None, + "search_result_language": None, + "sort_results_by": None, + "subquery_generation": True, + "subquery_generation_llm": "openai", + "relevance_analysis_llm": "openai", + "final_answer_llm": "openai" + }) + print(results) + pass + + ######################### Main Orchestration Workflow ######################### # def process_question(question: str, search_params: Dict) -> Dict: @@ -95,6 +122,12 @@ def process_question(question: str, search_params: Dict) -> Dict: """ logging.info(f"Starting process_question with query: {question}") + # Validate input parameters + if not question or not isinstance(question, str): + raise ValueError("Invalid question parameter") + if not search_params or not isinstance(search_params, dict): + raise ValueError("Invalid search_params parameter") + # 1. Generate sub-queries if requested sub_queries = [] sub_query_dict = { @@ -112,10 +145,32 @@ def process_question(question: str, search_params: Dict) -> Dict: # Merge original question with sub-queries all_queries = [question] + sub_queries - # 2. Perform searches and accumulate all raw results - all_results: List[Dict] = [] + # 2. Initialize a single web_search_results_dict + web_search_results_dict = { + "search_engine": search_params.get('engine', 'google'), + "search_query": question, + "content_country": search_params.get('content_country', 'US'), + "search_lang": search_params.get('search_lang', 'en'), + "output_lang": search_params.get('output_lang', 'en'), + "result_count": 0, # Will be updated as results are added + "date_range": search_params.get('date_range'), + "safesearch": search_params.get('safesearch', 'moderate'), + "site_blacklist": search_params.get('site_blacklist', []), + "exactTerms": search_params.get('exactTerms'), + "excludeTerms": search_params.get('excludeTerms'), + "filter": search_params.get('filter'), + "geolocation": search_params.get('geolocation'), + "search_result_language": search_params.get('search_result_language'), + "sort_results_by": search_params.get('sort_results_by'), + "results": [], + "total_results_found": 0, + "search_time": 0.0, + "error": None, + "processing_error": None + } + + # 3. Perform searches and accumulate all raw results for q in all_queries: - # FIXME - change raw_results to proper dict format raw_results = perform_websearch( search_engine=search_params.get('engine', 'google'), search_query=q, @@ -134,47 +189,43 @@ def process_question(question: str, search_params: Dict) -> Dict: sort_results_by=search_params.get('sort_results_by') ) - # Validate raw_results - # FIXME - Account for proper structure of returned web_search_results_dict dictionary if not isinstance(raw_results, dict) or "processing_error" in raw_results: logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") continue - results_list = raw_results.get("results", []) - all_results.extend(results_list) + # Append results to the single web_search_results_dict + web_search_results_dict["results"].extend(raw_results.get("results", [])) + web_search_results_dict["total_results_found"] += raw_results.get("total_results_found", 0) + web_search_results_dict["search_time"] += raw_results.get("search_time", 0.0) + - # 3. Score/filter (placeholder) - relevant_results = {} - for r in all_results: - # FIXME - Put in proper args / Ensure this works - # search_results: List[Dict], - # original_question: str, - # sub_questions: List[str], - # api_endpoint: str - list_of_relevant_articles = search_result_relevance( - all_results, - question, - sub_query_dict['sub_questions'], - search_params.get('relevance_analysis_llm') - ) - if list_of_relevant_articles: - relevant_results[r] = list_of_relevant_articles - # 4. Summarize/aggregate final answer + # 4. Score/filter results + relevant_results = search_result_relevance( + web_search_results_dict["results"], + question, + sub_query_dict['sub_questions'], + search_params.get('relevance_analysis_llm') + ) + + # 5. Allow user to review and select relevant results + if search_params.get("user_review", False): + relevant_results = review_and_select_results({"results": list(relevant_results.values())}) + + # 6. Summarize/aggregate final answer final_answer = aggregate_results( - # FIXME - Add proper Args / Ensure this works - # web_search_results_dict: Dict, - # report_language: str, - # aggregation_api_endpoint: str - # FIXME - Proper datatypes/expectations - web_search_results_dict, - report_language=search_params.get('output_lang', 'en'), - api_endpoint=search_params.get('final_answer_llm') + relevant_results, + question, + sub_query_dict['sub_questions'], + search_params.get('final_answer_llm') ) - # Return the final data - # FIXME - Return full query details for debugging and analysis - return web_search_results_dict + # 7. Return the final data, including the single web_search_results_dict + return { + "final_answer": final_answer, + "relevant_results": relevant_results, + "web_search_results_dict": web_search_results_dict # Pass the single dict back + } ######################### Question Analysis ######################### @@ -228,8 +279,8 @@ def analyze_question(question: str, api_endpoint) -> Dict: input_data = "Follow the above instructions." sub_questions: List[str] = [] - try: - for attempt in range(3): + for attempt in range(3): + try: logging.info(f"Generating sub-questions (attempt {attempt + 1})") response = chat_api_call(api_endpoint, None, input_data, sub_question_generation_prompt, temp=0.7) if response: @@ -249,12 +300,12 @@ def analyze_question(question: str, api_endpoint) -> Dict: logging.info("Successfully extracted sub-questions using regex") break - # If still no sub-questions, log an error or handle appropriately - if not sub_questions: - logging.error("Failed to extract sub-questions from API response after all attempts.") + except Exception as e: + logging.error(f"Error generating sub-questions: {str(e)}") - except Exception as e: - logging.error(f"Error generating sub-questions: {str(e)}") + if not sub_questions: + logging.error("Failed to extract sub-questions from API response after all attempts.") + sub_questions = [original_query] # Fallback to the original query # Construct and return the result dictionary logging.info("Sub-questions generated successfully") @@ -268,6 +319,7 @@ def analyze_question(question: str, api_endpoint) -> Dict: ######################### Relevance Analysis ######################### # +# FIXME - Ensure edge cases are handled properly / Structured outputs? def search_result_relevance( search_results: List[Dict], original_question: str, @@ -292,6 +344,7 @@ def search_result_relevance( content = result.get("content", "") if not content: logging.error("No Content found in search results array!") + continue eval_prompt = f""" Given the following search results for the user's question: "{original_question}" and the generated sub-questions: {sub_questions}, evaluate the relevance of the search result to the user's question. @@ -383,46 +436,72 @@ def aggregate_results( """ if not relevant_results: return { - "summary": "No relevant results found. Unable to provide an answer.", + "Report": "No relevant results found. Unable to provide an answer.", "evidence": [], "confidence": 0.0 } + # FIXME - Validate and test thoroughly, also structured generation # Concatenate relevant contents for summarization concatenated_texts = "\n\n".join( f"ID: {rid}\nContent: {res['content']}\nReasoning: {res['reasoning']}" for rid, res in relevant_results.items() ) - # Example summarization prompt - summarize_prompt = f""" -Please provide a concise summary that answers the question: "{question}" - -Relevant sub-questions: {sub_questions} - -Here are the relevant excerpts from search results: -{concatenated_texts} - -Instructions: -1. Provide a concise summary that incorporates the key points. -2. Avoid quoting large passages verbatim; aim to synthesize the main ideas. -3. Make sure your answer is coherent, logically consistent, and addresses the question directly. -""" + # Example analysis prompt + # FIXME - Add to config.txt/utils.py + analyze_search_results_prompt = f""" + Generate a comprehensive, well-structured, and informative answer for a given question, + using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). + Use an unbiased, journalistic tone, adapting the level of formality to match the user’s question. + + • Cite your statements using [number] notation, placing citations at the end of the relevant sentence. + • Only cite the most relevant results. If multiple sources support the same point, cite all relevant sources [e.g., 1, 2, 3]. + • If sources conflict, present both perspectives clearly and cite the respective sources. + • If different sources refer to different entities with the same name, provide separate answers. + • Do not add any external or fabricated information. + • Do not include URLs or a reference section; cite inline with [number] format only. + • Do not repeat the question or include unnecessary redundancy. + • Use markdown formatting (e.g., **bold**, bullet points, ## headings) to organize the information. + • If the provided results are insufficient to answer the question, explicitly state what information is missing or unclear. + + Structure your answer like this: + 1. **Short introduction**: Briefly summarize the topic (1–2 sentences). + 2. **Bulleted points**: Present key details, each with appropriate citations. + 3. **Conclusion**: Summarize the findings or restate the core answer (with citations if needed). + + Example: + 1. **Short introduction**: This topic explores the impact of climate change on agriculture. + 2. **Bulleted points**: + - Rising temperatures have reduced crop yields in some regions [1]. + - Changes in rainfall patterns are affecting irrigation practices [2, 3]. + 3. **Conclusion**: Climate change poses significant challenges to global agriculture [1, 2, 3]. + + + {concatenated_texts} + + --------------------- + + Make sure to match the language of the user's question. + + Question: {question} + Answer (in the language of the user's question): + """ - input_data = "Summarize the relevant results." + input_data = "Follow the above instructions." try: - summary_response = chat_api_call( + returned_response = chat_api_call( api_endpoint=api_endpoint, api_key=None, input_data=input_data, - prompt=summarize_prompt, + prompt=analyze_search_results_prompt, temp=0.7 ) - if summary_response: + if returned_response: # You could do further parsing or confidence estimation here return { - "summary": summary_response, + "Report": returned_response, "evidence": list(relevant_results.values()), "confidence": 0.9 # Hardcoded or computed as needed } @@ -430,12 +509,35 @@ def aggregate_results( logging.error(f"Error aggregating results: {e}") return { - "summary": "Could not summarize the results due to an error.", + "summary": "Could not create the report due to an error.", "evidence": list(relevant_results.values()), "confidence": 0.0 } +def review_and_select_results(web_search_results_dict: Dict) -> Dict: + """ + Allows the user to review and select relevant results from the search results. + + Args: + web_search_results_dict (Dict): The dictionary containing all search results. + + Returns: + Dict: A dictionary containing only the user-selected relevant results. + """ + relevant_results = {} + print("Review the search results and select the relevant ones:") + for idx, result in enumerate(web_search_results_dict["results"]): + print(f"\nResult {idx + 1}:") + print(f"Title: {result['title']}") + print(f"URL: {result['url']}") + print(f"Content: {result['content'][:200]}...") # Show a preview of the content + user_input = input("Is this result relevant? (y/n): ").strip().lower() + if user_input == 'y': + relevant_results[str(idx)] = result + + return relevant_results + # # End of Orchestration functions ####################################################################################################################### @@ -448,62 +550,49 @@ def aggregate_results( # FIXME def perform_websearch(search_engine, search_query, content_country, search_lang, output_lang, result_count, date_range=None, safesearch=None, site_blacklist=None, exactTerms=None, excludeTerms=None, filter=None, geolocation=None, search_result_language=None, sort_results_by=None): - if search_engine.lower() == "baidu": - web_search_results = search_web_baidu(search_query, None, None) - processed_results = process_web_search_results(web_search_results, "baidu") - return processed_results - - elif search_engine.lower() == "bing": - web_search_results = search_web_bing(search_query, search_lang, content_country, date_range, result_count) - processed_results = process_web_search_results(web_search_results, "bing") - return processed_results - - elif search_engine.lower() == "brave": - web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, - site_blacklist, date_range) - processed_results = process_web_search_results(web_search_results, "brave") - return processed_results - - elif search_engine.lower() == "duckduckgo": - web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) - processed_results = process_web_search_results(web_search_results, "ddg") - return processed_results - - elif search_engine.lower() == "google": - web_search_results = search_web_google(search_query, result_count, content_country, date_range, exactTerms, - excludeTerms, filter, geolocation, output_lang, - search_result_language, safesearch, site_blacklist, sort_results_by) - processed_results = process_web_search_results(web_search_results, "google") - return processed_results - - elif search_engine.lower() == "kagi": - web_search_results = search_web_kagi(search_query, content_country) - processed_results = process_web_search_results(web_search_results, "kagi") - return processed_results - - elif search_engine.lower() == "serper": - web_search_results = search_web_serper() - processed_results = process_web_search_results(web_search_results, "serper") - return processed_results - - elif search_engine.lower() == "tavily": - web_search_results = search_web_tavily(search_query, result_count, site_blacklist) - processed_results = process_web_search_results(web_search_results, "tavily") - return processed_results - - elif search_engine.lower() == "searx": - web_search_results = search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') - processed_results = process_web_search_results(web_search_results, "bing") - return processed_results - - elif search_engine.lower() == "yandex": - web_search_results = search_web_yandex() - processed_results = process_web_search_results(web_search_results, "bing") - return processed_results + try: + if search_engine.lower() == "baidu": + web_search_results = search_web_baidu(search_query, None, None) - else: - return f"Error: Invalid Search Engine Name {search_engine}" + elif search_engine.lower() == "bing": + web_search_results = search_web_bing(search_query, search_lang, content_country, date_range, result_count) + elif search_engine.lower() == "brave": + web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, + site_blacklist, date_range) + + elif search_engine.lower() == "duckduckgo": + web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) + + elif search_engine.lower() == "google": + web_search_results = search_web_google(search_query, result_count, content_country, date_range, exactTerms, + excludeTerms, filter, geolocation, output_lang, + search_result_language, safesearch, site_blacklist, sort_results_by) + + elif search_engine.lower() == "kagi": + web_search_results = search_web_kagi(search_query, content_country) + + elif search_engine.lower() == "serper": + web_search_results = search_web_serper() + + elif search_engine.lower() == "tavily": + web_search_results = search_web_tavily(search_query, result_count, site_blacklist) + + elif search_engine.lower() == "searx": + web_search_results = search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general') + + elif search_engine.lower() == "yandex": + web_search_results = search_web_yandex() + + else: + return f"Error: Invalid Search Engine Name {search_engine}" + + # Process the raw search results + web_search_results_dict = process_web_search_results(web_search_results, search_engine) + return web_search_results_dict + + except Exception as e: + return {"processing_error": f"Error performing web search: {str(e)}"} # ######################### Search Result Parsing ################################################################## @@ -556,6 +645,10 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict "processing_error": None } """ + # Validate input parameters + if not isinstance(search_results, dict): + raise TypeError("search_results must be a dictionary") + # Initialize the output dictionary with default values web_search_results_dict = { "search_engine": search_engine, @@ -594,59 +687,50 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict "processing_error": None } try: + # Parse results based on the search engine if search_engine.lower() == "baidu": - pass + pass # Placeholder for Baidu-specific parsing elif search_engine.lower() == "bing": parsed_results = parse_bing_results(search_results, web_search_results_dict) - elif search_engine.lower() == "brave": - parse_brave_results(search_results, web_search_results_dict) - pass - + parsed_results = parse_brave_results(search_results, web_search_results_dict) elif search_engine.lower() == "duckduckgo": parsed_results = parse_duckduckgo_results(search_results, web_search_results_dict) - elif search_engine.lower() == "google": parsed_results = parse_google_results(search_results, web_search_results_dict) - elif search_engine.lower() == "kagi": parsed_results = parse_kagi_results(search_results, web_search_results_dict) - elif search_engine.lower() == "serper": parsed_results = parse_serper_results(search_results, web_search_results_dict) - elif search_engine.lower() == "tavily": parsed_results = parse_tavily_results(search_results, web_search_results_dict) - elif search_engine.lower() == "searx": parsed_results = parse_searx_results(search_results, web_search_results_dict) - elif search_engine.lower() == "yandex": parsed_results = parse_yandex_results(search_results, web_search_results_dict) - else: - web_search_results_dict["processing_error"] = f"Error: Invalid Search Engine Name {search_engine}" raise ValueError(f"Error: Invalid Search Engine Name {search_engine}") - except Exception as e: - web_search_results_dict["processing_error"] = f"Error processing search results: {str(e)}" - raise - # Process individual search results - for result in search_results.get("results", []): - processed_result = { - "title": result.get("title", ""), - "url": result.get("url", ""), - "content": result.get("content", ""), - "metadata": { - "date_published": result.get("metadata", {}).get("date_published", None), - "author": result.get("metadata", {}).get("author", None), - "source": result.get("metadata", {}).get("source", None), - "language": result.get("metadata", {}).get("language", None), - "relevance_score": result.get("metadata", {}).get("relevance_score", None), - "snippet": result.get("metadata", {}).get("snippet", None) + # Process individual search results + for result in search_results.get("results", []): + processed_result = { + "title": result.get("title", ""), + "url": result.get("url", ""), + "content": result.get("content", ""), + "metadata": { + "date_published": result.get("metadata", {}).get("date_published", None), + "author": result.get("metadata", {}).get("author", None), + "source": result.get("metadata", {}).get("source", None), + "language": result.get("metadata", {}).get("language", None), + "relevance_score": result.get("metadata", {}).get("relevance_score", None), + "snippet": result.get("metadata", {}).get("snippet", None) + } } - } - web_search_results_dict["results"].append(processed_result) + web_search_results_dict["results"].append(processed_result) + + except Exception as e: + web_search_results_dict["processing_error"] = f"Error processing search results: {str(e)}" + logging.error(f"Error in process_web_search_results: {str(e)}") return web_search_results_dict @@ -1284,7 +1368,7 @@ def test_search_google(): results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation,ui_language, search_result_language, safesearch, site_blacklist, sort_results_by) print(result) - + return result # FIXME - untested def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: @@ -1368,6 +1452,11 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: output_dict["processing_error"] = f"Error processing Google results: {str(e)}" def test_parse_google_results(): + parsed_results = {} + raw_results = {} + raw_results = test_search_google() + parse_google_results(raw_results, parsed_results) + print(parsed_results) pass diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index b72a4a1d9..96323c5ed 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -11,7 +11,7 @@ https://huggingface.co/papers/2412.15443 https://github.com/thiswillbeyourgithub/AnkiAIUtils https://news.ycombinator.com/item?id=42534931 https://ankiweb.net/shared/info/1531888719 - +https://bbycroft.net/llm one2manny diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 090b65720..03a2945bd 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -49,6 +49,15 @@ https://github.com/aedocw/epub2tts https://github.com/microsoft/SpeechT5 https://github.com/smellslikeml/dolla_llama +Bark +https://github.com/suno-ai/bark + + +ChatTTS +https://huggingface.co/2Noise/ChatTTS +https://chattts.com/#Demo + + Coqui TTS https://github.com/idiap/coqui-ai-TTS @@ -60,6 +69,9 @@ Cartesia F5 TTS https://github.com/SWivid/F5-TTS +lina TTS +https://github.com/theodorblackbird/lina-speech/blob/main/InferenceLina.ipynb +https://github.com/theodorblackbird/lina-speech Podcastfy https://github.com/souzatharsis/podcastfy/blob/main/podcastfy/tts/base.py @@ -90,11 +102,7 @@ Models Merging Audio https://github.com/jiaaro/pydub -GPT-SoviTTS - https://levelup.gitconnected.com/great-api-design-comprehensive-guide-from-basics-to-best-practices-9b4e0b613a44?source=home---------56-1--------------------0fc48da7_5ce6_48ca_92d2_260680a20318-------3 - https://rentry.org/GPT-SoVITS-guide - https://github.com/RVC-Boss/GPT-SoVITS - https://github.com/cpumaxx/sovits-ff-plugin + MaskGCT https://maskgct.github.io/#emotion-samples @@ -372,6 +380,15 @@ if __name__ == "__main__": ### GPT-SoVITS - [GPT-SoVITS](f) +- [GPT-SoVITS-guide rentry.org](https://rentry.org/GPT-SoVITS-guide) +- Setup Guide: https://ai-hub-docs.vercel.app/tts/gpt-sovits/ + + +GPT-SoviTTS + https://levelup.gitconnected.com/great-api-design-comprehensive-guide-from-basics-to-best-practices-9b4e0b613a44?source=home---------56-1--------------------0fc48da7_5ce6_48ca_92d2_260680a20318-------3 + https://rentry.org/GPT-SoVITS-guide + https://github.com/RVC-Boss/GPT-SoVITS + https://github.com/cpumaxx/sovits-ff-plugin https://tts.x86.st/ Finetuning is very quick (about 5 minutes). Captioning of audio was automated with faster-whisper (it is required that the audio is captioned). From d09c8aed21251977f410d14571f9b9f3f9c9f65e Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 17:34:29 -0800 Subject: [PATCH 47/88] checkpoint --- App_Function_Libraries/Utils/Utils.py | 2 + .../Web_Scraping/WebSearch_APIs.py | 271 +++++++++++++----- Docs/Design/Researcher.md | 41 ++- Docs/Design/Search.md | 2 +- Docs/Design/TTS_STT.md | 45 +++ Docs/Design/WebSearch.md | 9 + Docs/RAG_Notes.md | 3 +- setup.py | 23 ++ 8 files changed, 314 insertions(+), 82 deletions(-) create mode 100644 setup.py diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 0c3cef517..da418f29b 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -373,6 +373,7 @@ def load_and_log_configs(): google_search_engine_id = config.get('Search-Engines', 'search_engine_id_google', fallback='') google_simp_trad_chinese = config.get('Search-Engines', 'enable_traditional_chinese', fallback='0') limit_google_search_to_country = config.get('Search-Engines', 'limit_google_search_to_country', fallback='0') + google_search_country = config.get('Search-Engines', 'google_search_country', fallback='us') google_search_country_code = config.get('Search-Engines', 'google_search_country_code', fallback='us') google_filter_setting = config.get('Search-Engines', 'google_filter_setting', fallback='1') google_user_geolocation = config.get('Search-Engines', 'google_user_geolocation', fallback='') @@ -521,6 +522,7 @@ def load_and_log_configs(): 'google_search_engine_id': google_search_engine_id, 'google_simp_trad_chinese': google_simp_trad_chinese, 'limit_google_search_to_country': limit_google_search_to_country, + 'google_search_country': google_search_country, 'google_search_country_code': google_search_country_code, 'google_search_filter_setting': google_filter_setting, 'google_user_geolocation': google_user_geolocation, diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 2c3e9e53f..1c1dae69a 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -4,7 +4,9 @@ # Imports import json import logging +import random import re +import time from html import unescape from typing import Optional, Dict, Any, List from urllib.parse import urlparse, urlencode, unquote @@ -121,7 +123,6 @@ def process_question(question: str, search_params: Dict) -> Dict: } """ logging.info(f"Starting process_question with query: {question}") - # Validate input parameters if not question or not isinstance(question, str): raise ValueError("Invalid question parameter") @@ -129,6 +130,7 @@ def process_question(question: str, search_params: Dict) -> Dict: raise ValueError("Invalid search_params parameter") # 1. Generate sub-queries if requested + logging.info(f"Generating sub-queries for the query: {question}") sub_queries = [] sub_query_dict = { "main_goal": question, @@ -138,11 +140,13 @@ def process_question(question: str, search_params: Dict) -> Dict: } if search_params.get("subquery_generation", False): + logging.info("Sub-query generation enabled") api_endpoint = search_params.get("subquery_generation_llm", "openai") sub_query_dict = analyze_question(question, api_endpoint) sub_queries = sub_query_dict.get("sub_questions", []) # Merge original question with sub-queries + logging.info(f"Sub-queries generated: {sub_queries}") all_queries = [question] + sub_queries # 2. Initialize a single web_search_results_dict @@ -171,8 +175,9 @@ def process_question(question: str, search_params: Dict) -> Dict: # 3. Perform searches and accumulate all raw results for q in all_queries: + logging.info(f"Performing web search for query: {q}") raw_results = perform_websearch( - search_engine=search_params.get('engine', 'google'), + search_engine=search_params.get('engine'), search_query=q, content_country=search_params.get('content_country', 'US'), search_lang=search_params.get('search_lang', 'en'), @@ -192,15 +197,18 @@ def process_question(question: str, search_params: Dict) -> Dict: if not isinstance(raw_results, dict) or "processing_error" in raw_results: logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") continue + logging.info(f"Search results found for query '{q}': {len(raw_results.get('results', []))}") # Append results to the single web_search_results_dict web_search_results_dict["results"].extend(raw_results.get("results", [])) web_search_results_dict["total_results_found"] += raw_results.get("total_results_found", 0) web_search_results_dict["search_time"] += raw_results.get("search_time", 0.0) - + logging.info(f"Total results found so far: {len(web_search_results_dict['results'])}") # 4. Score/filter results + # FIXME - allow for user interaction at this point as well + logging.info("Scoring and filtering search results") relevant_results = search_result_relevance( web_search_results_dict["results"], question, @@ -208,9 +216,19 @@ def process_question(question: str, search_params: Dict) -> Dict: search_params.get('relevance_analysis_llm') ) - # 5. Allow user to review and select relevant results + # 5. Allow user to review and select relevant results (if enabled) or perform further searches/query refinement + logging.info("Reviewing and selecting relevant results") if search_params.get("user_review", False): + logging.info("User review not enabled") + relevant_results = review_and_select_results({"results": list(relevant_results.values())}) + elif search_params.get("user_review", True): + logging.info("User review enabled") relevant_results = review_and_select_results({"results": list(relevant_results.values())}) + # FIXME - Add return logic here so the function returns and another function picks up + # FIXME - the user can then decide to continue or repeat the process/stop + return { + "web_search_results_dict": web_search_results_dict # Pass the single dict back + } # 6. Summarize/aggregate final answer final_answer = aggregate_results( @@ -221,6 +239,7 @@ def process_question(question: str, search_params: Dict) -> Dict: ) # 7. Return the final data, including the single web_search_results_dict + logging.info("Returning final websearch results") return { "final_answer": final_answer, "relevant_results": relevant_results, @@ -411,6 +430,29 @@ def search_result_relevance( return relevant_results +def review_and_select_results(web_search_results_dict: Dict) -> Dict: + """ + Allows the user to review and select relevant results from the search results. + + Args: + web_search_results_dict (Dict): The dictionary containing all search results. + + Returns: + Dict: A dictionary containing only the user-selected relevant results. + """ + relevant_results = {} + print("Review the search results and select the relevant ones:") + for idx, result in enumerate(web_search_results_dict["results"]): + print(f"\nResult {idx + 1}:") + print(f"Title: {result['title']}") + print(f"URL: {result['url']}") + print(f"Content: {result['content'][:200]}...") # Show a preview of the content + user_input = input("Is this result relevant? (y/n): ").strip().lower() + if user_input == 'y': + relevant_results[str(idx)] = result + + return relevant_results + ######################### Result Aggregation & Combination ######################### # def aggregate_results( @@ -514,30 +556,6 @@ def aggregate_results( "confidence": 0.0 } - -def review_and_select_results(web_search_results_dict: Dict) -> Dict: - """ - Allows the user to review and select relevant results from the search results. - - Args: - web_search_results_dict (Dict): The dictionary containing all search results. - - Returns: - Dict: A dictionary containing only the user-selected relevant results. - """ - relevant_results = {} - print("Review the search results and select the relevant ones:") - for idx, result in enumerate(web_search_results_dict["results"]): - print(f"\nResult {idx + 1}:") - print(f"Title: {result['title']}") - print(f"URL: {result['url']}") - print(f"Content: {result['content'][:200]}...") # Show a preview of the content - user_input = input("Is this result relevant? (y/n): ").strip().lower() - if user_input == 'y': - relevant_results[str(idx)] = result - - return relevant_results - # # End of Orchestration functions ####################################################################################################################### @@ -594,6 +612,11 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, except Exception as e: return {"processing_error": f"Error performing web search: {str(e)}"} +def test_perform_websearch(): + results = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="moderate", site_blacklist=["example.com", "spam-site.com"]) + print(results) + pass + # ######################### Search Result Parsing ################################################################## # @@ -835,16 +858,21 @@ def search_web_bing(search_query, bing_lang, bing_country, result_count=None, bi def test_search_web_bing(): search_query = "How can I get started learning machine learning?" bing_lang = "en" - bing_country = "US" + bing_country = "US" result_count = 10 bing_api_key = None date_range = None result = search_web_bing(search_query, bing_lang, bing_country, result_count, bing_api_key, date_range) + # Unparsed results print("Bing Search Results:") print(result) + # Parsed results + output_dict = {"results": []} + parse_bing_results(result, output_dict) + print("Parsed Bing Results:") + print(json.dumps(output_dict, indent=2)) -# FIXME - untested def parse_bing_results(raw_results: Dict, output_dict: Dict) -> None: """ Parse Bing search results and update the output dictionary @@ -853,7 +881,12 @@ def parse_bing_results(raw_results: Dict, output_dict: Dict) -> None: raw_results (Dict): Raw Bing API response output_dict (Dict): Dictionary to store processed results """ + logging.info(f"Raw Bing results received: {json.dumps(raw_results, indent=2)}") try: + # Initialize results list if not present + if "results" not in output_dict: + output_dict["results"] = [] + # Extract web pages results if "webPages" in raw_results: web_pages = raw_results["webPages"] @@ -905,13 +938,10 @@ def parse_bing_results(raw_results: Dict, output_dict: Dict) -> None: ] except Exception as e: + logging.error(f"Error processing Bing results: {str(e)}") output_dict["processing_error"] = f"Error processing Bing results: {str(e)}" -def test_parse_bing_results(): - pass - - ######################### Brave Search ######################### # # https://brave.com/search/api/ @@ -919,7 +949,7 @@ def test_parse_bing_results(): def search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch="moderate", brave_api_key=None, result_filter=None, search_type="ai", date_range=None): search_url = "https://api.search.brave.com/res/v1/web/search" - if not brave_api_key: + if not brave_api_key and search_type == "web": # load key from config file brave_api_key = loaded_config_data['search_engines']['brave_search_api_key'] if not brave_api_key: @@ -939,8 +969,9 @@ def search_web_brave(search_term, country, search_lang, ui_lang, result_count, s if not result_filter: result_filter = "webpages" if search_type == "ai": - # FIXME - Option for switching between AI/Regular search - pass + brave_api_key = loaded_config_data['search_engines']['brave_search_ai_api_key'] + else: + raise ValueError("Invalid search type. Please choose 'ai' or 'web'.") headers = {"Accept": "application/json", "Accept-Encoding": "gzip", "X-Subscription-Token": brave_api_key} @@ -966,12 +997,16 @@ def test_search_brave(): date_range = None result_filter = None result = search_web_brave(search_term, country, search_lang, ui_lang, result_count, safesearch, date_range, - result_filter) + result_filter) + print("Brave Search Results:") print(result) - return result + + output_dict = {"results": []} + parse_brave_results(result, output_dict) + print("Parsed Brave Results:") + print(json.dumps(output_dict, indent=2)) -# FIXME - untested def parse_brave_results(raw_results: Dict, output_dict: Dict) -> None: """ Parse Brave search results and update the output dictionary @@ -981,6 +1016,10 @@ def parse_brave_results(raw_results: Dict, output_dict: Dict) -> None: output_dict (Dict): Dictionary to store processed results """ try: + # Initialize results list if not present + if "results" not in output_dict: + output_dict["results"] = [] + # Extract query information if "query" in raw_results: query_info = raw_results["query"] @@ -1023,8 +1062,8 @@ def parse_brave_results(raw_results: Dict, output_dict: Dict) -> None: output_dict["family_friendly"] = raw_results.get("family_friendly", True) except Exception as e: + logging.error(f"Error processing Brave results: {str(e)}") output_dict["processing_error"] = f"Error processing Brave results: {str(e)}" - raise def test_parse_brave_results(): pass @@ -1122,6 +1161,7 @@ def _normalize(raw_html: str) -> str: return results + def test_search_duckduckgo(): try: results = search_web_duckduckgo( @@ -1137,6 +1177,12 @@ def test_search_duckduckgo(): print(f"Snippet: {result['body']}") print("---") + # Parse the results + output_dict = {"results": []} + parse_duckduckgo_results({"results": results}, output_dict) + print("Parsed DuckDuckGo Results:") + print(json.dumps(output_dict, indent=2)) + except ValueError as e: print(f"Invalid input: {str(e)}") except requests.RequestException as e: @@ -1152,26 +1198,28 @@ def parse_duckduckgo_results(raw_results: Dict, output_dict: Dict) -> None: output_dict (Dict): Dictionary to store processed results """ try: - # DuckDuckGo results appear to be in a simple list format - # Each result is separated by "---" + # Initialize results list if not present + if "results" not in output_dict: + output_dict["results"] = [] + + # DuckDuckGo results are in a list of dictionaries results = raw_results.get("results", []) for result in results: - # Extract information using the consistent format in results - title = "" - url = "" - snippet = "" - - # Parse the result text - lines = result.split('\n') - for line in lines: - if line.startswith("Title: "): - title = line.replace("Title: ", "").strip() - elif line.startswith("URL: "): - url = line.replace("URL: ", "").strip() - elif line.startswith("Snippet: "): - snippet = line.replace("Snippet: ", "").strip() - + # Extract information directly from the dictionary + title = result.get("title", "") + url = result.get("href", "") + snippet = result.get("body", "") + + # Log warnings for missing data + if not title: + logging.warning("Missing title in result") + if not url: + logging.warning("Missing URL in result") + if not snippet: + logging.warning("Missing snippet in result") + + # Add the processed result to the output dictionary processed_result = { "title": title, "url": url, @@ -1192,6 +1240,7 @@ def parse_duckduckgo_results(raw_results: Dict, output_dict: Dict) -> None: output_dict["total_results_found"] = len(output_dict["results"]) except Exception as e: + logging.error(f"Error processing DuckDuckGo results: {str(e)}") output_dict["processing_error"] = f"Error processing DuckDuckGo results: {str(e)}" @@ -1213,6 +1262,7 @@ def extract_domain(url: str) -> str: except: return url + def test_parse_duckduckgo_results(): pass @@ -1364,13 +1414,27 @@ def test_search_google(): safesearch = "off" site_blacklist = None sort_results_by = None - result = search_web_google(search_query, google_search_api_key, google_search_engine_id, result_count, c2coff, - results_origin_country, date_range, exactTerms, excludeTerms, filter, geolocation,ui_language, - search_result_language, safesearch, site_blacklist, sort_results_by) + result = search_web_google(search_query, + google_search_api_key, + google_search_engine_id, + result_count, + c2coff, + results_origin_country, + date_range, + exactTerms, + excludeTerms, + filter, + geolocation, + ui_language, + search_result_language, + safesearch, + site_blacklist, + sort_results_by + ) print(result) return result -# FIXME - untested + def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: """ Parse Google Custom Search API results and update the output dictionary @@ -1379,12 +1443,17 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: raw_results (Dict): Raw Google API response output_dict (Dict): Dictionary to store processed results """ + logging.info(f"Raw results received: {json.dumps(raw_results, indent=2)}") try: + # Initialize results list if not present + if "results" not in output_dict: + output_dict["results"] = [] + # Extract search information if "searchInformation" in raw_results: search_info = raw_results["searchInformation"] output_dict["total_results_found"] = int(search_info.get("totalResults", "0")) - output_dict["search_time"] = search_info.get("searchTime", 0.0) + output_dict["search_time"] = float(search_info.get("searchTime", 0.0)) # Extract spelling suggestions if "spelling" in raw_results: @@ -1414,8 +1483,7 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: "url": item.get("link", ""), "content": item.get("snippet", ""), "metadata": { - "date_published": item.get("pagemap", {}).get("metatags", [{}])[0].get( - "article:published_time"), + "date_published": item.get("pagemap", {}).get("metatags", [{}])[0].get("article:published_time"), "author": item.get("pagemap", {}).get("metatags", [{}])[0].get("article:author"), "source": item.get("displayLink", None), "language": item.get("language", None), @@ -1433,8 +1501,7 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: if "metatags" in pagemap and pagemap["metatags"]: metatags = pagemap["metatags"][0] processed_result["metadata"].update({ - "description": metatags.get("og:description", - metatags.get("description")), + "description": metatags.get("og:description", metatags.get("description")), "keywords": metatags.get("keywords"), "site_name": metatags.get("og:site_name") }) @@ -1449,6 +1516,7 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: } except Exception as e: + logging.error(f"Error processing Google results: {str(e)}") output_dict["processing_error"] = f"Error processing Google results: {str(e)}" def test_parse_google_results(): @@ -1557,11 +1625,43 @@ def test_parse_kagi_results(): # # https://searx.space # https://searx.github.io/searx/dev/search_api.html -def search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general'): - # Check if API URL is configured - searx_url = loaded_config_data['search_engines']['searx_search_api_url'] +def searx_create_session() -> requests.Session: + """ + Create a requests session with retry logic. + """ + session = requests.Session() + retries = Retry( + total=3, # Maximum number of retries + backoff_factor=1, # Exponential backoff factor + status_forcelist=[429, 500, 502, 503, 504], # Retry on these status codes + allowed_methods=["GET"] # Only retry on GET requests + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session + +def search_web_searx(search_query, language='auto', time_range='', safesearch=0, pageno=1, categories='general', searx_url=None): + """ + Perform a search using a Searx instance. + + Args: + search_query (str): The search query. + language (str): Language for the search results. + time_range (str): Time range for the search results. + safesearch (int): Safe search level (0=off, 1=moderate, 2=strict). + pageno (int): Page number of the results. + categories (str): Categories to search in (e.g., 'general', 'news'). + searx_url (str): Custom Searx instance URL (optional). + + Returns: + str: JSON string containing the search results or an error message. + """ + # Use the provided Searx URL or fall back to the configured one + if not searx_url: + searx_url = loaded_config_data['search_engines']['searx_search_api_url'] if not searx_url: - return "SearX Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet." + return json.dumps({"error": "SearX Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet."}) # Validate and construct URL try: @@ -1575,16 +1675,28 @@ def search_web_searx(search_query, language='auto', time_range='', safesearch=0, 'categories': categories } search_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}?{urlencode(params)}" + logging.info(f"Search URL: {search_url}") except Exception as e: - return f"Search is disabled and no content was found. Invalid URL configuration: {str(e)}" + return json.dumps({"error": f"Invalid URL configuration: {str(e)}"}) # Perform the search request try: + # Mimic browser headers headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0' + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Referer': 'https://www.google.com/', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1' } - response = requests.get(search_url, headers=headers) + # Add a random delay to mimic human behavior + delay = random.uniform(2, 5) # Random delay between 2 and 5 seconds + time.sleep(delay) + + session = searx_create_session() + response = session.get(search_url, headers=headers) response.raise_for_status() # Check if the response is JSON @@ -1608,18 +1720,19 @@ def search_web_searx(search_query, language='auto', time_range='', safesearch=0, }) if not data: - return "No information was found online for the search query." + return json.dumps({"error": "No information was found online for the search query."}) return json.dumps(data) except requests.exceptions.RequestException as e: - return f"There was an error searching for content. {str(e)}" - + logging.error(f"Error searching for content: {str(e)}") + return json.dumps({"error": f"There was an error searching for content. {str(e)}"}) def test_search_searx(): - result = search_web_searx("How can I bake a cherry cake?") + # Use a different Searx instance to avoid rate limiting + searx_url = "https://searx.be" # Example of a different Searx instance + result = search_web_searx("What goes into making a cherry cake?", searx_url=searx_url) print(result) - pass def parse_searx_results(searx_search_results, web_search_results_dict): pass diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 0e3c9f5ef..330bae150 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -186,4 +186,43 @@ Current step to execute: {current_step} Your search queries based: """ -``` \ No newline at end of file +``` + + + +I use NotebookLM daily and find it incredibly helpful. However, I've noticed a potential improvement for the audio creation feature. Currently, when generating audio from a source, it primarily focuses on the provided text. I propose enhancing this by adding a "deep research" component that runs in the background during audio generation. + +Imagine this: you provide NotebookLM with a news article about a new AI tool. When you click "create audio," instead of just reading the article, NotebookLM would: + Analyze the Source: Understand the core topic, key terms, and context of the provided source. + Conduct Background Research: Leverage Google's powerful search and knowledge graph to gather additional information related to the topic. This could include: + Official documentation or websites for tools. + Related news articles, blog posts, and research papers. + Expert opinions and analyses. + Relevant historical context. + Integrate Findings: Seamlessly weave the researched information into the audio output, creating a more comprehensive and insightful experience. This could be done by: + Adding explanatory segments or summaries. + Providing context and background information. + Highlighting different perspectives or opinions. + Offering definitions of key terms. +Example: +If the source is an article about "LaMDA," NotebookLM could research: + Google AI's official information on LaMDA. + Recent advancements in large language models. + Ethical considerations surrounding AI language models. + Comparisons to other similar models. +This would result in an audio output that not only summarizes the original article but also provides valuable context and deeper understanding. +Benefits: + More Comprehensive Content: Audio outputs become more informative and valuable for users. + Saves User Time: Users don't have to conduct their own research to get the full picture. + Enhanced Learning Experience: Provides a richer and more engaging way to consume information. + Positions NotebookLM as an Expert Resource: By providing in-depth information, NotebookLM becomes a go-to tool for learning about various topics. +Suggested Implementation Details: + Leverage Google's Existing Tools: Utilize Google Search, Knowledge Graph, and potentially the "deep research" module already present within Google's ecosystem. This would ensure seamless integration and efficient use of existing resources. + Clear User Controls: Provide options for users to customize the depth of research (e.g., "basic," "moderate," "in-depth"). This gives users control over the process and prevents information overload. + Citation and Source Linking: Include links to the researched sources within the NotebookLM document associated with the audio, providing transparency and allowing users to verify information. + Integration with Google Lens: If an image is part of the source, use Google Lens to extract text and context, further enhancing the research capabilities. +Additional Features: + Option to Exclude Research: Allow users to disable background research if they only want a direct reading of the source. + Customizable Research Focus: Allow users to specify keywords or areas of focus for the background research, allowing for more targeted results. + Multilingual Research: Expand research capabilities to multiple languages, making the feature more globally accessible. +By implementing this feature, NotebookLM can become an even more powerful tool for learning and understanding complex topics, providing users with comprehensive and insightful audio experiences. \ No newline at end of file diff --git a/Docs/Design/Search.md b/Docs/Design/Search.md index 65c7fd0ec..744fd95d3 100644 --- a/Docs/Design/Search.md +++ b/Docs/Design/Search.md @@ -4,6 +4,6 @@ ### Link Dump https://pub.towardsai.net/hnsw-small-world-yes-but-how-in-the-world-is-it-navigable-77701ed37e20 https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1 - +https://github.com/quickwit-oss/tantivy diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 03a2945bd..43db3d601 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -49,6 +49,22 @@ https://github.com/aedocw/epub2tts https://github.com/microsoft/SpeechT5 https://github.com/smellslikeml/dolla_llama + +STT + https://github.com/KoljaB/RealtimeSTT + +TTS + https://github.com/KoljaB/RealtimeTTS + +101 + https://www.inferless.com/learn/comparing-different-text-to-speech---tts--models-for-different-use-cases + https://clideo.com/resources/what-is-tts + RVC 101 + https://gudgud96.github.io/2024/09/26/annotated-rvc/ + +Datasets(?) + https://voice-models.com/ + Bark https://github.com/suno-ai/bark @@ -115,6 +131,10 @@ Parler https://github.com/huggingface/parler-tts +YourTTS + https://github.com/Edresson/YourTTS + + TTS Pipeline https://www.astramind.ai/post/auralis @@ -389,11 +409,33 @@ GPT-SoviTTS https://rentry.org/GPT-SoVITS-guide https://github.com/RVC-Boss/GPT-SoVITS https://github.com/cpumaxx/sovits-ff-plugin + https://github.com/HanxSmile/Simplify-GPT-SoVITS + https://github.com/lrxwisdom001/GPT-SoVITS-Novels/tree/main/voice_synthesis + openneurosama - https://github.com/JarodMica/open-neruosama/blob/master/main.py + https://tts.x86.st/ Finetuning is very quick (about 5 minutes). Captioning of audio was automated with faster-whisper (it is required that the audio is captioned). With the default batch size of 12, training takes 9.5~ GB. +Inference + https://github.com/RVC-Boss/GPT-SoVITS/blob/main/GPT_SoVITS/inference_cli.py + No WebUI Inference on Colab: https://colab.research.google.com/drive/1gC1lRxuOh4qW8Yz5TA10BEUPR28nJ3VR + Training on Colab: https://colab.research.google.com/drive/1NQGKXYxJcJyTPnHsSyusTdD0l4IdMS37#scrollTo=nhyKqVwcPnvz + No WebUI Training on Colab: https://colab.research.google.com/drive/1LmeM8yUyT9MTYF8OXc-NiBonvdh6hII6 + +Datasets + https://ai-hub-docs.vercel.app/rvc/resources/datasets/ + https://mvsep.com/en + +API + https://github.com/cpumaxx/sovits-ff-plugin + +Comfyui integration + https://github.com/heshengtao/comfyui_LLM_party + + + - **101** - F - **Setup** @@ -405,4 +447,7 @@ With the default batch size of 12, training takes 9.5~ GB. - **Fine-Tuning** - F +### Dataset Creation/Curation + + diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 928352ff8..fa1846fbc 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -13,6 +13,15 @@ This page serves as documentation regarding the web search functionality within 8. The final product is then passed back to the UI for display to the user. +### Current Status +- Bing, Brave, DDG, Google work. +- Baidu, SearX, Serper, Tavily, Yandex are not implemented yet. +- Kagi & SearX are implemented but not working (Kagi because API access and SearX because I'm not sure why) +- Parsing works for Bing, Brave, DDG, Google. +- Currently no use of Structured outputs (to-do...) +- + + ---------------- ### Setting the Stage diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 36a9df810..1059a2bf5 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -74,7 +74,8 @@ GraphRAG https://towardsdatascience.com/how-to-query-a-knowledge-graph-with-llms-using-grag-38bfac47a322 https://arxiv.org/abs/2408.04187 https://pub.towardsai.net/building-a-knowledge-graph-from-unstructured-text-data-a-step-by-step-guide-c14c926c2229 - + https://www.youtube.com/watch?v=g6xBklAIrsA + https://neuml.hashnode.dev/advanced-rag-with-graph-path-traversal ### Links diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..ebae4f552 --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +from setuptools import setup, find_packages + +setup( + name="tldw", + version="0.1.0", + author="Robert Musser", + author_email="contact@tldwproject.com", + description="A short description of your project", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="https://github.com/rmusser01/tldw", + packages=find_packages(), + install_requires=[ + "numpy", + "requests", + ], + classifiers=[ # Classifiers for PyPI (optional) + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache 2.0 License", + "Operating System :: OS Independent", + ], + python_requires=">=3.9", +) \ No newline at end of file From a31bf68f4ef7ed74a79f2c3142eea528e9d0a0c7 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 18:03:58 -0800 Subject: [PATCH 48/88] Update WebSearch_APIs.py --- .../Web_Scraping/WebSearch_APIs.py | 148 ++++++++++++++++-- 1 file changed, 139 insertions(+), 9 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 1c1dae69a..0c7d71fc6 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -573,19 +573,61 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, web_search_results = search_web_baidu(search_query, None, None) elif search_engine.lower() == "bing": - web_search_results = search_web_bing(search_query, search_lang, content_country, date_range, result_count) + # Prepare the arguments for search_web_bing + bing_args = { + "search_query": search_query, + "bing_lang": search_lang, + "bing_country": content_country, + "result_count": result_count, + "bing_api_key": loaded_config_data['search_engines'].get('bing_api_key'), # Fetch Bing API key from config + "date_range": date_range, + } + + # Call the search_web_bing function with the prepared arguments + web_search_results = search_web_bing(**bing_args) elif search_engine.lower() == "brave": - web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, - site_blacklist, date_range) + web_search_results = search_web_brave(search_query, content_country, search_lang, output_lang, result_count, safesearch, + site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) elif search_engine.lower() == "google": - web_search_results = search_web_google(search_query, result_count, content_country, date_range, exactTerms, - excludeTerms, filter, geolocation, output_lang, - search_result_language, safesearch, site_blacklist, sort_results_by) + # Convert site_blacklist list to a comma-separated string + if site_blacklist and isinstance(site_blacklist, list): + site_blacklist = ",".join(site_blacklist) + + # Prepare the arguments for search_web_google + google_args = { + "search_query": search_query, + "google_search_api_key": loaded_config_data['search_engines']['google_search_api_key'], + "google_search_engine_id": loaded_config_data['search_engines']['google_search_engine_id'], + "result_count": result_count, + "c2coff": "1", # Default value + "results_origin_country": content_country, + "ui_language": output_lang, + "search_result_language": search_result_language or "lang_en", # Default value + "geolocation": geolocation or "us", # Default value + "safesearch": safesearch or "off", # Default value + } + + # Add optional parameters only if they are provided + if date_range: + google_args["date_range"] = date_range + if exactTerms: + google_args["exactTerms"] = exactTerms + if excludeTerms: + google_args["excludeTerms"] = excludeTerms + if filter: + google_args["filter"] = filter + if site_blacklist: + google_args["site_blacklist"] = site_blacklist + if sort_results_by: + google_args["sort_results_by"] = sort_results_by + + # Call the search_web_google function with the prepared arguments + web_search_results = search_web_google(**google_args) elif search_engine.lower() == "kagi": web_search_results = search_web_kagi(search_query, content_country) @@ -612,9 +654,97 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, except Exception as e: return {"processing_error": f"Error performing web search: {str(e)}"} -def test_perform_websearch(): - results = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="moderate", site_blacklist=["example.com", "spam-site.com"]) - print(results) + +def test_perform_websearch_google(): + # Google Searches + try: + test_1 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 1: {test_1}") + # FIXME - Fails. Need to fix arg formatting + test_2 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="moderate", site_blacklist=["example.com", "spam-site.com"]) + print(f"Test 2: {test_2}") + test_3 = results = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 3: {test_3}") + except Exception as e: + print(f"Error performing google searches: {str(e)}") + pass + + +def test_perform_websearch_bing(): + # Bing Searches + try: + test_4 = perform_websearch("bing", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 4: {test_4}") + test_5 = perform_websearch("bing", "What is the capital of France?", "US", "en", "en", 10, date_range="y") + print(f"Test 5: {test_5}") + except Exception as e: + print(f"Error performing bing searches: {str(e)}") + + +def test_perform_websearch_brave(): + # Brave Searches + try: + test_7 = perform_websearch("brave", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 7: {test_7}") + except Exception as e: + print(f"Error performing brave searches: {str(e)}") + + +def test_perform_websearch_ddg(): + # DuckDuckGo Searches + try: + test_6 = perform_websearch("duckduckgo", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 6: {test_6}") + except Exception as e: + print(f"Error performing duckduckgo searches: {str(e)}") + + +# FIXME +def test_perform_websearch_kagi(): + # Kagi Searches + try: + test_8 = perform_websearch("kagi", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 8: {test_8}") + except Exception as e: + print(f"Error performing kagi searches: {str(e)}") + +# FIXME +def test_perform_websearch_serper(): + # Serper Searches + try: + test_9 = perform_websearch("serper", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 9: {test_9}") + except Exception as e: + print(f"Error performing serper searches: {str(e)}") + +# FIXME +def test_perform_websearch_tavily(): + # Tavily Searches + try: + test_10 = perform_websearch("tavily", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 10: {test_10}") + except Exception as e: + print(f"Error performing tavily searches: {str(e)}") + + +# FIXME +def test_perform_websearch_searx(): + # Searx Searches + try: + test_11 = perform_websearch("searx", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 11: {test_11}") + except Exception as e: + print(f"Error performing searx searches: {str(e)}") + + +# FIXME +def test_perform_websearch_yandex(): + #Yandex Searches + try: + test_12 = perform_websearch("yandex", "What is the capital of France?", "US", "en", "en", 10) + print(f"Test 12: {test_12}") + except Exception as e: + print(f"Error performing yandex searches: {str(e)}") pass # From 7b29c1e99f24b2ea655f875ff49196cdc73cf674 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 18:21:13 -0800 Subject: [PATCH 49/88] DDG works --- .../Web_Scraping/WebSearch_APIs.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 0c7d71fc6..3c0f18186 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -591,7 +591,19 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, site_blacklist, date_range) elif search_engine.lower() == "duckduckgo": - web_search_results = search_web_duckduckgo(search_query, content_country, date_range, result_count) + # Prepare the arguments for search_web_duckduckgo + ddg_args = { + "keywords": search_query, + "region": f"{content_country.lower()}-{search_lang.lower()}", # Format: "us-en" + "timelimit": date_range[0] if date_range else None, # Use first character of date_range (e.g., "y" -> "y") + "max_results": result_count, + } + + # Call the search_web_duckduckgo function with the prepared arguments + ddg_results = search_web_duckduckgo(**ddg_args) + + # Wrap the results in a dictionary to match the expected format + web_search_results = {"results": ddg_results} elif search_engine.lower() == "google": # Convert site_blacklist list to a comma-separated string @@ -609,7 +621,7 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, "ui_language": output_lang, "search_result_language": search_result_language or "lang_en", # Default value "geolocation": geolocation or "us", # Default value - "safesearch": safesearch or "off", # Default value + "safesearch": safesearch or "off", # Default value, } # Add optional parameters only if they are provided @@ -695,6 +707,8 @@ def test_perform_websearch_ddg(): try: test_6 = perform_websearch("duckduckgo", "What is the capital of France?", "US", "en", "en", 10) print(f"Test 6: {test_6}") + test_7 = perform_websearch("duckduckgo", "What is the capital of France?", "US", "en", "en", 10, date_range="y") + print(f"Test 7: {test_7}") except Exception as e: print(f"Error performing duckduckgo searches: {str(e)}") From 55328ba1bc8226cc6c21021dc65621608fd24c5d Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 21:45:13 -0800 Subject: [PATCH 50/88] Fuck Yea it works. Perplexity pipeline is in. --- App_Function_Libraries/LLM_API_Calls.py | 2 +- .../Web_Scraping/WebSearch_APIs.py | 264 ++++++++++-------- Docs/Design/TTA.md | 9 + 3 files changed, 151 insertions(+), 124 deletions(-) create mode 100644 Docs/Design/TTA.md diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 148c1e476..7569e8631 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -202,7 +202,7 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m ], "max_tokens": 4096, "temperature": temp, - "streaming": streaming + "stream": streaming } if streaming: logging.debug("OpenAI: Posting request (streaming") diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 3c0f18186..0751efd25 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -33,42 +33,48 @@ # # Functions: -# FIXME - Add Logging - -def test_process_question(): - results = process_question("What is the capital of France?", { - "engine": "google", - "content_country": "US", - "search_lang": "en", - "output_lang": "en", - "result_count": 10, - "date_range": "y", - "safesearch": "moderate", - "site_blacklist": ["example.com", "spam-site.com"], - "exactTerms": None, - "excludeTerms": None, - "filter": None, - "geolocation": None, - "search_result_language": None, - "sort_results_by": None, - "subquery_generation": True, - "subquery_generation_llm": "openai", - "relevance_analysis_llm": "openai", - "final_answer_llm": "openai" - }) - print(results) - pass ######################### Main Orchestration Workflow ######################### # -def process_question(question: str, search_params: Dict) -> Dict: +# FIXME - Add Logging + +def initialize_web_search_results_dict(search_params: Dict) -> Dict: """ - Orchestrates the entire pipeline: - 1. Optionally generate sub-queries (if subquery_generation=True). - 2. Perform web searches for each query (the original and sub-queries). - 3. Score results for relevance; filter them. - 4. Aggregate final answer from relevant results. + Initializes and returns a dictionary for storing web search results and metadata. + + Args: + search_params (Dict): A dictionary containing search parameters. + + Returns: + Dict: A dictionary initialized with search metadata. + """ + return { + "search_engine": search_params.get('engine', 'google'), + "search_query": "", + "content_country": search_params.get('content_country', 'US'), + "search_lang": search_params.get('search_lang', 'en'), + "output_lang": search_params.get('output_lang', 'en'), + "result_count": 0, + "date_range": search_params.get('date_range'), + "safesearch": search_params.get('safesearch', 'moderate'), + "site_blacklist": search_params.get('site_blacklist', []), + "exactTerms": search_params.get('exactTerms'), + "excludeTerms": search_params.get('excludeTerms'), + "filter": search_params.get('filter'), + "geolocation": search_params.get('geolocation'), + "search_result_language": search_params.get('search_result_language'), + "sort_results_by": search_params.get('sort_results_by'), + "results": [], + "total_results_found": 0, + "search_time": 0.0, + "error": None, + "processing_error": None + } + +def generate_and_search(question: str, search_params: Dict) -> Dict: + """ + Generates sub-queries (if enabled) and performs web searches for each query. Args: question (str): The user's original question or query. @@ -76,59 +82,25 @@ def process_question(question: str, search_params: Dict) -> Dict: and specifying LLM endpoints. Returns: - Dict: A dictionary containing all relevant data, including results from each sub-query. - - A dictionary containing parameters for performing a web search and processing the results. - - Dict Parameters: - engine (str): The search engine to use (e.g., "google", "bing", "brave", "duckduckgo", etc.). - content_country (str): The country code for content localization (e.g., "US", "UK", "DE"). - search_lang (str): The language for the search query (e.g., "en" for English). - output_lang (str): The desired language for the search results. - result_count (int): The number of results to return. - date_range (str or None): The time range for the search results (e.g., "y" for the past year). - safesearch (str or None): The safe search setting (e.g., "moderate", "strict", "off"). - site_blacklist (list or None): A list of sites to exclude from the search results. - exactTerms (str or None): Terms that must appear in the search results. - excludeTerms (str or None): Terms that must not appear in the search results. - filter (str or None): Any additional filtering criteria. - geolocation (str or None): Geographical location for search results. - search_result_language (str or None): The language of the search results. - sort_results_by (str or None): Criteria for sorting the results. - subquery_generation (bool): Whether to generate sub-queries. - subquery_generation_api (str): The API to use for sub-query generation (e.g., "openai", "anthropic", "deepseek"). - relevance_analysis_llm (str): The LLM model to use for relevance analysis (e.g., "openai", "anthropic", "deepseek"). - final_answer_llm (str): The LLM model to use for generating the final answer (e.g., "openai", "anthropic", "deepseek"). + Dict: A dictionary containing all search results and related metadata. - Example: - search_params = { - "engine": "google", - "content_country": "US", - "search_lang": "en", - "output_lang": "en", - "result_count": 10, - "date_range": "y", - "safesearch": "moderate", - "site_blacklist": ["example.com", "spam-site.com"], - "exactTerms": None, - "excludeTerms": None, - "filter": None, - "geolocation": None, - "search_result_language": None, - "sort_results_by": None, - "subquery_generation": True, - "subquery_generation_llm": "openai", - "relevance_analysis_llm": "openai", - "final_answer_llm": "openai" - } + Raises: + ValueError: If the input parameters are invalid. """ - logging.info(f"Starting process_question with query: {question}") + logging.info(f"Starting generate_and_search with query: {question}") + # Validate input parameters if not question or not isinstance(question, str): raise ValueError("Invalid question parameter") if not search_params or not isinstance(search_params, dict): raise ValueError("Invalid search_params parameter") + # Check for required keys in search_params + required_keys = ["engine", "content_country", "search_lang", "output_lang", "result_count"] + for key in required_keys: + if key not in search_params: + raise ValueError(f"Missing required key in search_params: {key}") + # 1. Generate sub-queries if requested logging.info(f"Generating sub-queries for the query: {question}") sub_queries = [] @@ -143,35 +115,15 @@ def process_question(question: str, search_params: Dict) -> Dict: logging.info("Sub-query generation enabled") api_endpoint = search_params.get("subquery_generation_llm", "openai") sub_query_dict = analyze_question(question, api_endpoint) - sub_queries = sub_query_dict.get("sub_questions", []) # Merge original question with sub-queries + sub_queries = sub_query_dict.get("sub_questions", []) logging.info(f"Sub-queries generated: {sub_queries}") all_queries = [question] + sub_queries # 2. Initialize a single web_search_results_dict - web_search_results_dict = { - "search_engine": search_params.get('engine', 'google'), - "search_query": question, - "content_country": search_params.get('content_country', 'US'), - "search_lang": search_params.get('search_lang', 'en'), - "output_lang": search_params.get('output_lang', 'en'), - "result_count": 0, # Will be updated as results are added - "date_range": search_params.get('date_range'), - "safesearch": search_params.get('safesearch', 'moderate'), - "site_blacklist": search_params.get('site_blacklist', []), - "exactTerms": search_params.get('exactTerms'), - "excludeTerms": search_params.get('excludeTerms'), - "filter": search_params.get('filter'), - "geolocation": search_params.get('geolocation'), - "search_result_language": search_params.get('search_result_language'), - "sort_results_by": search_params.get('sort_results_by'), - "results": [], - "total_results_found": 0, - "search_time": 0.0, - "error": None, - "processing_error": None - } + web_search_results_dict = initialize_web_search_results_dict(search_params) + web_search_results_dict["search_query"] = question # 3. Perform searches and accumulate all raw results for q in all_queries: @@ -195,8 +147,9 @@ def process_question(question: str, search_params: Dict) -> Dict: ) if not isinstance(raw_results, dict) or "processing_error" in raw_results: - logging.warning(f"Error or invalid data returned for query '{q}': {raw_results}") + logging.error(f"Error or invalid data returned for query '{q}': {raw_results}") continue + logging.info(f"Search results found for query '{q}': {len(raw_results.get('results', []))}") # Append results to the single web_search_results_dict @@ -205,46 +158,93 @@ def process_question(question: str, search_params: Dict) -> Dict: web_search_results_dict["search_time"] += raw_results.get("search_time", 0.0) logging.info(f"Total results found so far: {len(web_search_results_dict['results'])}") + return { + "web_search_results_dict": web_search_results_dict, + "sub_query_dict": sub_query_dict + } + + +def analyze_and_aggregate(web_search_results_dict: Dict, sub_query_dict: Dict, search_params: Dict) -> Dict: + """ + Analyzes the relevance of search results and aggregates the final answer. + + Args: + web_search_results_dict (Dict): A dictionary containing all search results and related metadata. + sub_query_dict (Dict): A dictionary containing sub-queries and related metadata. + search_params (Dict): A dictionary containing parameters for performing web searches + and specifying LLM endpoints. + + Returns: + Dict: A dictionary containing: + - final_answer: The aggregated final answer. + - relevant_results: The filtered relevant results. + - web_search_results_dict: The original search results and metadata. + """ + logging.info("Starting analyze_and_aggregate") # 4. Score/filter results - # FIXME - allow for user interaction at this point as well logging.info("Scoring and filtering search results") + sub_questions = sub_query_dict.get("sub_questions", []) relevant_results = search_result_relevance( web_search_results_dict["results"], - question, - sub_query_dict['sub_questions'], + sub_query_dict["main_goal"], + sub_questions, search_params.get('relevance_analysis_llm') ) + # FIXME + print("Relevant results returned by search_result_relevance:") + print(json.dumps(relevant_results, indent=2)) - # 5. Allow user to review and select relevant results (if enabled) or perform further searches/query refinement + # 5. Allow user to review and select relevant results (if enabled) logging.info("Reviewing and selecting relevant results") if search_params.get("user_review", False): - logging.info("User review not enabled") - relevant_results = review_and_select_results({"results": list(relevant_results.values())}) - elif search_params.get("user_review", True): logging.info("User review enabled") relevant_results = review_and_select_results({"results": list(relevant_results.values())}) - # FIXME - Add return logic here so the function returns and another function picks up - # FIXME - the user can then decide to continue or repeat the process/stop - return { - "web_search_results_dict": web_search_results_dict # Pass the single dict back - } # 6. Summarize/aggregate final answer final_answer = aggregate_results( relevant_results, - question, - sub_query_dict['sub_questions'], + sub_query_dict["main_goal"], + sub_questions, search_params.get('final_answer_llm') ) - # 7. Return the final data, including the single web_search_results_dict + # 7. Return the final data logging.info("Returning final websearch results") return { "final_answer": final_answer, "relevant_results": relevant_results, - "web_search_results_dict": web_search_results_dict # Pass the single dict back + "web_search_results_dict": web_search_results_dict + } + + +def test_perplexity_pipeline(): + # Phase 1: Generate sub-queries and perform web searches + search_params = { + "engine": "google", + "content_country": "countryUS", + "search_lang": "en", + "output_lang": "en", + "result_count": 10, + "date_range": None, + "safesearch": "active", + "site_blacklist": ["spam-site.com"], + "exactTerms": None, + "excludeTerms": None, + "filter": None, + "geolocation": None, + "search_result_language": None, + "sort_results_by": None, + "subquery_generation": True, + "subquery_generation_llm": "openai", + "relevance_analysis_llm": "openai", + "final_answer_llm": "openai" } + phase1_results = generate_and_search("What is the capital of France?", search_params) + # Review the results here if needed + # Phase 2: Analyze relevance and aggregate final answer + phase2_results = analyze_and_aggregate(phase1_results["web_search_results_dict"], phase1_results["sub_query_dict"], search_params) + print(phase2_results["final_answer"]) ######################### Question Analysis ######################### @@ -358,7 +358,7 @@ def search_result_relevance( Dict[str, Dict]: A dictionary of relevant results, keyed by a unique ID or index. """ relevant_results = {} - + print("WHAT THE FUCK") for idx, result in enumerate(search_results): content = result.get("content", "") if not content: @@ -385,6 +385,7 @@ def search_result_relevance( input_data = "Evaluate the relevance of the search result." try: + print("WHAT THE FUCK") # Perform API call to evaluate relevance relevancy_result = chat_api_call( api_endpoint=api_endpoint, @@ -394,8 +395,12 @@ def search_result_relevance( temp=0.7 ) + # FIXME + print(f"[DEBUG] Relevancy LLM response for index {idx}:\n{relevancy_result}\n---") + if relevancy_result: # Extract the selected answer and reasoning via regex + print(f"LLM Relevancy Response for item {idx}:", relevancy_result) selected_answer_match = re.search( r"Selected Answer:\s*(True|False)", relevancy_result, @@ -624,6 +629,16 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, "safesearch": safesearch or "off", # Default value, } + # If site_blacklist has multiple domains, do not use siteSearch + if site_blacklist and len(site_blacklist) == 1: + google_args["siteSearch"] = site_blacklist[0] + google_args["siteSearchFilter"] = "e" + else: + # Do not use siteSearch for multiple domains + # Either skip it entirely or see Option 2 below + google_args.pop("siteSearch", None) + google_args.pop("siteSearchFilter", None) + # Add optional parameters only if they are provided if date_range: google_args["date_range"] = date_range @@ -661,6 +676,9 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, # Process the raw search results web_search_results_dict = process_web_search_results(web_search_results, search_engine) + # FIXME + print("After process_web_search_results:") + print(json.dumps(web_search_results_dict, indent=2)) return web_search_results_dict except Exception as e: @@ -673,7 +691,7 @@ def test_perform_websearch_google(): test_1 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10) print(f"Test 1: {test_1}") # FIXME - Fails. Need to fix arg formatting - test_2 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="moderate", site_blacklist=["example.com", "spam-site.com"]) + test_2 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="moderate", site_blacklist=["spam-site.com"]) print(f"Test 2: {test_2}") test_3 = results = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10) print(f"Test 3: {test_3}") @@ -1450,7 +1468,7 @@ def search_web_google( :param ui_language: Language of the user interface :param search_result_language: Language of search results :param safesearch: Safe search setting - :param site_blacklist: Sites to exclude from search + :param site_blacklist: Single Site to exclude from search :param sort_results_by: Sorting criteria for results :return: JSON response from Google Search API """ @@ -1511,9 +1529,6 @@ def search_web_google( safesearch = loaded_config_data['search_engines']['google_safe_search'] if safesearch: params["safe"] = safesearch - if site_blacklist: - params["siteSearch"] = site_blacklist - params["siteSearchFilter"] = "e" # Exclude these sites if sort_results_by: params["sort"] = sort_results_by @@ -1547,7 +1562,7 @@ def test_search_google(): google_search_engine_id = loaded_config_data['search_engines']['google_search_engine_id'] result_count = 10 c2coff = "1" - results_origin_country = "US" + results_origin_country = "countryUS" date_range = None exactTerms = None excludeTerms = None @@ -1588,6 +1603,9 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: output_dict (Dict): Dictionary to store processed results """ logging.info(f"Raw results received: {json.dumps(raw_results, indent=2)}") + # FIXME + print("Raw web_search_results from Google:") + print(json.dumps(raw_results, indent=2)) try: # Initialize results list if not present if "results" not in output_dict: diff --git a/Docs/Design/TTA.md b/Docs/Design/TTA.md new file mode 100644 index 000000000..157bd000d --- /dev/null +++ b/Docs/Design/TTA.md @@ -0,0 +1,9 @@ +# Text-to-Audio (TTA) + +## Introduction + + +### Link Dump: +https://github.com/declare-lab/Tangoflux +https://tangoflux.github.io/ + From a9a5ceb34704d756934f17a600ef89f8987bc381 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 22:33:11 -0800 Subject: [PATCH 51/88] sweet now its working --- .../Web_Scraping/WebSearch_APIs.py | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 0751efd25..888fb377d 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -146,14 +146,19 @@ def generate_and_search(question: str, search_params: Dict) -> Dict: sort_results_by=search_params.get('sort_results_by') ) - if not isinstance(raw_results, dict) or "processing_error" in raw_results: + # Debug: Inspect raw results + logging.debug(f"Raw results for query '{q}': {raw_results}") + + # Check for errors or invalid data + if not isinstance(raw_results, dict) or raw_results.get("processing_error"): logging.error(f"Error or invalid data returned for query '{q}': {raw_results}") + print(f"Error or invalid data returned for query '{q}': {raw_results}") continue logging.info(f"Search results found for query '{q}': {len(raw_results.get('results', []))}") # Append results to the single web_search_results_dict - web_search_results_dict["results"].extend(raw_results.get("results", [])) + web_search_results_dict["results"].extend(raw_results["results"]) web_search_results_dict["total_results_found"] += raw_results.get("total_results_found", 0) web_search_results_dict["search_time"] += raw_results.get("search_time", 0.0) logging.info(f"Total results found so far: {len(web_search_results_dict['results'])}") @@ -358,7 +363,7 @@ def search_result_relevance( Dict[str, Dict]: A dictionary of relevant results, keyed by a unique ID or index. """ relevant_results = {} - print("WHAT THE FUCK") + print("WHAT THE FUCK #1") for idx, result in enumerate(search_results): content = result.get("content", "") if not content: @@ -385,7 +390,7 @@ def search_result_relevance( input_data = "Evaluate the relevance of the search result." try: - print("WHAT THE FUCK") + print("WHAT THE FUCK #2") # Perform API call to evaluate relevance relevancy_result = chat_api_call( api_endpoint=api_endpoint, @@ -654,7 +659,9 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, google_args["sort_results_by"] = sort_results_by # Call the search_web_google function with the prepared arguments - web_search_results = search_web_google(**google_args) + web_search_results = search_web_google(**google_args) # raw JSON + web_search_results_dict = process_web_search_results(web_search_results, "google") + return web_search_results_dict elif search_engine.lower() == "kagi": web_search_results = search_web_kagi(search_query, content_country) @@ -851,21 +858,7 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict "geolocation": search_results.get("geolocation", None), "search_result_language": search_results.get("search_result_language", None), "sort_results_by": search_results.get("sort_results_by", None), - "results": [ - { - "title": str, - "url": str, - "content": str, - "metadata": { - "date_published": Optional[str], - "author": Optional[str], - "source": Optional[str], - "language": Optional[str], - "relevance_score": Optional[float], - "snippet": Optional[str] - } - }, - ], + "results": [], "total_results_found": search_results.get("total_results_found", 0), "search_time": search_results.get("search_time", 0.0), "error": search_results.get("error", None), @@ -897,21 +890,22 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict raise ValueError(f"Error: Invalid Search Engine Name {search_engine}") # Process individual search results - for result in search_results.get("results", []): - processed_result = { - "title": result.get("title", ""), - "url": result.get("url", ""), - "content": result.get("content", ""), - "metadata": { - "date_published": result.get("metadata", {}).get("date_published", None), - "author": result.get("metadata", {}).get("author", None), - "source": result.get("metadata", {}).get("source", None), - "language": result.get("metadata", {}).get("language", None), - "relevance_score": result.get("metadata", {}).get("relevance_score", None), - "snippet": result.get("metadata", {}).get("snippet", None) - } - } - web_search_results_dict["results"].append(processed_result) + # DELETEME? This breaks the structure of the results + # for result in search_results.get("results", []): + # processed_result = { + # "title": result.get("title", ""), + # "url": result.get("url", ""), + # "content": result.get("content", ""), + # "metadata": { + # "date_published": result.get("metadata", {}).get("date_published", None), + # "author": result.get("metadata", {}).get("author", None), + # "source": result.get("metadata", {}).get("source", None), + # "language": result.get("metadata", {}).get("language", None), + # "relevance_score": result.get("metadata", {}).get("relevance_score", None), + # "snippet": result.get("metadata", {}).get("snippet", None) + # } + # } + # web_search_results_dict["results"].append(processed_result) except Exception as e: web_search_results_dict["processing_error"] = f"Error processing search results: {str(e)}" @@ -1596,14 +1590,14 @@ def test_search_google(): def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: """ - Parse Google Custom Search API results and update the output dictionary + Parse Google Custom Search API results and update the output dictionary. Args: - raw_results (Dict): Raw Google API response - output_dict (Dict): Dictionary to store processed results + raw_results (Dict): Raw Google API response. + output_dict (Dict): Dictionary to store processed results. """ logging.info(f"Raw results received: {json.dumps(raw_results, indent=2)}") - # FIXME + # For debugging only FIXME print("Raw web_search_results from Google:") print(json.dumps(raw_results, indent=2)) try: @@ -1643,10 +1637,15 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: processed_result = { "title": item.get("title", ""), "url": item.get("link", ""), + # IMPORTANT: 'snippet' is used as 'content' "content": item.get("snippet", ""), "metadata": { - "date_published": item.get("pagemap", {}).get("metatags", [{}])[0].get("article:published_time"), - "author": item.get("pagemap", {}).get("metatags", [{}])[0].get("article:author"), + "date_published": item.get("pagemap", {}) + .get("metatags", [{}])[0] + .get("article:published_time"), + "author": item.get("pagemap", {}) + .get("metatags", [{}])[0] + .get("article:author"), "source": item.get("displayLink", None), "language": item.get("language", None), "relevance_score": None, # Google doesn't provide this directly @@ -1663,7 +1662,8 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: if "metatags" in pagemap and pagemap["metatags"]: metatags = pagemap["metatags"][0] processed_result["metadata"].update({ - "description": metatags.get("og:description", metatags.get("description")), + "description": metatags.get("og:description", + metatags.get("description")), "keywords": metatags.get("keywords"), "site_name": metatags.get("og:site_name") }) @@ -1674,19 +1674,22 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: output_dict["pagination"] = { "has_next": "nextPage" in raw_results.get("queries", {}), "has_previous": "previousPage" in raw_results.get("queries", {}), - "current_page": raw_results.get("queries", {}).get("request", [{}])[0].get("startIndex", 1) + "current_page": raw_results.get("queries", {}) + .get("request", [{}])[0] + .get("startIndex", 1) } except Exception as e: logging.error(f"Error processing Google results: {str(e)}") output_dict["processing_error"] = f"Error processing Google results: {str(e)}" + def test_parse_google_results(): parsed_results = {} raw_results = {} raw_results = test_search_google() parse_google_results(raw_results, parsed_results) - print(parsed_results) + print(f"Parsed search results: {parsed_results}") pass From dd8ec0dc6fa17c205fb1a5c8e28979c92b80e052 Mon Sep 17 00:00:00 2001 From: Robert Date: Mon, 30 Dec 2024 22:35:35 -0800 Subject: [PATCH 52/88] Update WebSearch_APIs.py --- .../Web_Scraping/WebSearch_APIs.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 888fb377d..0f95a08fa 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -363,7 +363,6 @@ def search_result_relevance( Dict[str, Dict]: A dictionary of relevant results, keyed by a unique ID or index. """ relevant_results = {} - print("WHAT THE FUCK #1") for idx, result in enumerate(search_results): content = result.get("content", "") if not content: @@ -390,7 +389,6 @@ def search_result_relevance( input_data = "Evaluate the relevance of the search result." try: - print("WHAT THE FUCK #2") # Perform API call to evaluate relevance relevancy_result = chat_api_call( api_endpoint=api_endpoint, @@ -401,11 +399,11 @@ def search_result_relevance( ) # FIXME - print(f"[DEBUG] Relevancy LLM response for index {idx}:\n{relevancy_result}\n---") + logging.debug(f"[DEBUG] Relevancy LLM response for index {idx}:\n{relevancy_result}\n---") if relevancy_result: # Extract the selected answer and reasoning via regex - print(f"LLM Relevancy Response for item {idx}:", relevancy_result) + logging.debug(f"LLM Relevancy Response for item {idx}:", relevancy_result) selected_answer_match = re.search( r"Selected Answer:\s*(True|False)", relevancy_result, @@ -684,8 +682,8 @@ def perform_websearch(search_engine, search_query, content_country, search_lang, # Process the raw search results web_search_results_dict = process_web_search_results(web_search_results, search_engine) # FIXME - print("After process_web_search_results:") - print(json.dumps(web_search_results_dict, indent=2)) + #logging.debug("After process_web_search_results:") + #logging.debug(json.dumps(web_search_results_dict, indent=2)) return web_search_results_dict except Exception as e: @@ -1598,8 +1596,8 @@ def parse_google_results(raw_results: Dict, output_dict: Dict) -> None: """ logging.info(f"Raw results received: {json.dumps(raw_results, indent=2)}") # For debugging only FIXME - print("Raw web_search_results from Google:") - print(json.dumps(raw_results, indent=2)) + logging.debug("Raw web_search_results from Google:") + logging.debug(json.dumps(raw_results, indent=2)) try: # Initialize results list if not present if "results" not in output_dict: @@ -1717,7 +1715,7 @@ def search_web_kagi(query: str, limit: int = 10) -> Dict: response = requests.get(endpoint, headers=headers, params=params) response.raise_for_status() - print(response.json()) + logging.debug(response.json()) return response.json() From f46aa0e50210b2913157c8ec87e4793ed5c95351 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 10:15:21 -0800 Subject: [PATCH 53/88] f --- .../Web_Scraping/WebSearch_APIs.py | 21 +---- Docs/Design/Inference_Engines.md | 6 +- Docs/Design/Prompts.md | 78 ++++++++++++++++++- Docs/Design/RSS_Ranking.md | 3 +- Docs/Design/TTS_STT.md | 4 +- Docs/Design/UX.md | 2 +- Docs/Design/Visual_Video_Analysis.md | 2 +- Docs/Issues/Evaluation_Plans.md | 2 +- Docs/RAG_Notes.md | 8 ++ 9 files changed, 100 insertions(+), 26 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 0f95a08fa..2d2fa264e 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -403,7 +403,7 @@ def search_result_relevance( if relevancy_result: # Extract the selected answer and reasoning via regex - logging.debug(f"LLM Relevancy Response for item {idx}:", relevancy_result) + logging.debug(f"LLM Relevancy Response for item:", relevancy_result) selected_answer_match = re.search( r"Selected Answer:\s*(True|False)", relevancy_result, @@ -461,6 +461,7 @@ def review_and_select_results(web_search_results_dict: Dict) -> Dict: return relevant_results + ######################### Result Aggregation & Combination ######################### # def aggregate_results( @@ -887,24 +888,6 @@ def process_web_search_results(search_results: Dict, search_engine: str) -> Dict else: raise ValueError(f"Error: Invalid Search Engine Name {search_engine}") - # Process individual search results - # DELETEME? This breaks the structure of the results - # for result in search_results.get("results", []): - # processed_result = { - # "title": result.get("title", ""), - # "url": result.get("url", ""), - # "content": result.get("content", ""), - # "metadata": { - # "date_published": result.get("metadata", {}).get("date_published", None), - # "author": result.get("metadata", {}).get("author", None), - # "source": result.get("metadata", {}).get("source", None), - # "language": result.get("metadata", {}).get("language", None), - # "relevance_score": result.get("metadata", {}).get("relevance_score", None), - # "snippet": result.get("metadata", {}).get("snippet", None) - # } - # } - # web_search_results_dict["results"].append(processed_result) - except Exception as e: web_search_results_dict["processing_error"] = f"Error processing search results: {str(e)}" logging.error(f"Error in process_web_search_results: {str(e)}") diff --git a/Docs/Design/Inference_Engines.md b/Docs/Design/Inference_Engines.md index 25cfb781e..893f339b3 100644 --- a/Docs/Design/Inference_Engines.md +++ b/Docs/Design/Inference_Engines.md @@ -3,4 +3,8 @@ ### Link Dump -https://github.com/intel/neural-compressor/tree/v3.2 \ No newline at end of file +https://github.com/intel/neural-compressor/tree/v3.2 + +https://magazine.sebastianraschka.com/p/practical-tips-for-finetuning-llms?utm_medium=email +https://magazine.sebastianraschka.com/p/lora-and-dora-from-scratch?utm_medium=email +https://arxiv.org/abs/2405.09673 \ No newline at end of file diff --git a/Docs/Design/Prompts.md b/Docs/Design/Prompts.md index 2a396ba2c..a64f6f6b3 100644 --- a/Docs/Design/Prompts.md +++ b/Docs/Design/Prompts.md @@ -8,4 +8,80 @@ https://github.com/LouisShark/chatgpt_system_prompt https://github.com/microsoft/PromptWizard -https://medium.com/@camauger/crafting-effective-chatgpt-prompts-for-tabletop-roleplaying-games-a-step-by-step-guide-part-1-b81a791d278d \ No newline at end of file +https://medium.com/@camauger/crafting-effective-chatgpt-prompts-for-tabletop-roleplaying-games-a-step-by-step-guide-part-1-b81a791d278d + + + + + + + + + + + + + + + +```https://gist.githubusercontent.com/dsartori/35de7f2ed879d5a5e50f6362dea2281b/raw/fb45b3ebbed46ebd99cd4a8d7083112ada596090/rag_prompt.txt +You are an expert assistant trained to retrieve and generate detailed information **only** from a curated dataset. Your primary goal is to answer natural-language queries accurately and concisely by extracting and synthesizing information explicitly available in the dataset. You are prohibited from making assumptions, inferences, or providing information that cannot be directly traced back to the dataset. The topics you specialize in are: + + +- policies and priorities +- organizational structure +- programs and operations +- key partnerships +- challenges +- history and legislation + + +### Guidelines for Responses: +1. **Source-Dependence**: + - Only provide answers based on explicit information in the dataset. + - Avoid making assumptions, synthesizing unrelated data, or inferring conclusions not directly supported by the dataset. + - If the requested information is not found, respond transparently with: *"This information is not available in the dataset."* + + +2. **Explicit Citations**: + - For every response, reference the specific chunk(s) or metadata field(s) that support your answer (e.g., "According to chunk 1-4, ..."). + - If multiple chunks are used, list all relevant sources to improve transparency. + + +3. **Clarification**: + - If a query is ambiguous or lacks sufficient context, ask clarifying questions before proceeding. + + +4. **Language Consistency**: + - Respond exclusively in the user’s language. Do not switch languages or interpret unless explicitly requested. + + +5. **Accuracy First**: + - Prioritize accuracy by strictly adhering to the dataset. Avoid providing speculative or generalized answers. + + +6. **General Before Specific**: + - Begin with a concise general overview of the relevant topic, based entirely on the dataset. + - Provide detailed insights, examples, or elaborations only upon follow-up or explicit request. + + +7. **Iterative Engagement**: + - Encourage the user to refine or expand their queries to enable more precise responses. + + +### Response Structure: +1. **General Overview**: Provide a high-level summary of the relevant information available in the dataset. +2. **Detailed Insights (If Requested)**: Offer specific details or examples directly sourced from the dataset, explicitly citing the source. +3. **Unavailable Information**: If the dataset lacks information for a query, respond with: *"This information is not available in the dataset."* +4. **Next Steps**: Suggest follow-up queries or related topics the user might explore. + + +### Key Instructions: +- **Do Not Hallucinate**: Never provide information that is not explicitly present in the dataset. If uncertain, state clearly that the information is unavailable. +- **Transparency**: Reference specific chunks, sections, or metadata fields for every detail provided. +- **Avoid Inference**: Refrain from combining or interpreting unrelated information unless explicitly connected within the dataset. +- **Focus on Relevance**: Ensure answers are concise, precise, and directly address the user’s query. + + +Adapt to the user's needs by maintaining strict adherence to the dataset while offering actionable and transparent insights. +``` \ No newline at end of file diff --git a/Docs/Design/RSS_Ranking.md b/Docs/Design/RSS_Ranking.md index 7e47157e0..d73a9c6bb 100644 --- a/Docs/Design/RSS_Ranking.md +++ b/Docs/Design/RSS_Ranking.md @@ -17,7 +17,8 @@ https://www.dogesec.com/blog/full_text_rss_atom_blog_feeds/ https://arxiv.org/abs/2411.19352 https://arxiv.org/abs/2412.18082 - +https://arxiv.org/abs/2412.08604 +https://arxiv.org/abs/2411.18814 https://blog.badsectorlabs.com/files/blogs.txt diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 43db3d601..ca68684b4 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -168,13 +168,15 @@ https://github.com/pipecat-ai/pipecat/tree/a367a038f1a3967292b5de5b43b8600a82a73 https://github.com/lamm-mit/PDF2Audio https://github.com/Purfview/whisper-standalone-win https://github.com/ictnlp/LLaMA-Omni - +https://levelup.gitconnected.com/build-a-real-time-ai-voice-and-video-chat-app-with-function-calling-by-gemini-2-0-49599a48fbe9?gi=c894f6c092be +https://github.com/agituts/gemini-2-podcast https://github.com/SWivid/F5-TTS https://github.com/matatonic/openedai-speech https://github.com/RVC-Boss/GPT-SoVITS +https://www.bilibili.com/video/BV11iiNegEGP/ https://github.com/RVC-Boss/GPT-SoVITS/wiki/GPT%E2%80%90SoVITS%E2%80%90v2%E2%80%90features-(%E6%96%B0%E7%89%B9%E6%80%A7) https://rentry.org/GPT-SoVITS-guide https://rentry.org/GPT-SoVITS-guide diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 26639e40b..8c11ad893 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -82,7 +82,7 @@ https://uxdesign.cc/simple-recall-testing-for-b2b-ux-134e18330da9 https://lovable.dev/ https://markwhen.com/ https://kando.menu/ - +https://deepseek-artifacts.vercel.app/ Prompt Engineering page: diff --git a/Docs/Design/Visual_Video_Analysis.md b/Docs/Design/Visual_Video_Analysis.md index cd8059fab..09f50be28 100644 --- a/Docs/Design/Visual_Video_Analysis.md +++ b/Docs/Design/Visual_Video_Analysis.md @@ -9,7 +9,7 @@ ### Link Dump: https://github.com/SamurAIGPT/AI-Youtube-Shorts-Generator - +https://arxiv.org/abs/2412.19238 diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index 14a993d64..ee5e1d0b1 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -49,7 +49,7 @@ https://huggingface.co/spaces/bigcode/bigcodebench-leaderboard https://www.atla-ai.com/post/evaluating-the-evaluator https://hamel.dev/blog/posts/llm-judge/ https://github.com/scrapinghub/article-extraction-benchmark - +https://github.com/Zhe-Young/SelfCorrectDecompose Have LLMs play Social deception games Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index 1059a2bf5..b163f3ec6 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -42,6 +42,10 @@ https://ai.gopubby.com/agentic-rags-extending-rags-to-sql-databases-1509b25ca3e7 https://ai.plainenglish.io/a-deep-dive-into-golden-retriever-eea3396af3b4 https://github.com/AhmedAl93/multimodal-agentic-RAG https://hub.athina.ai/athina-originals/end-to-end-implementation-of-unstructured-rag/ +https://medium.com/@eliot64/bridging-legal-ai-and-trust-how-we-won-the-llm-x-law-hackathon-45081a8681d9 +https://arxiv.org/abs/2412.15563 +https://arxiv.org/pdf/2410.10293v1 +https://arxiv.org/pdf/2409.02098v1 GraphRAG @@ -76,6 +80,10 @@ GraphRAG https://pub.towardsai.net/building-a-knowledge-graph-from-unstructured-text-data-a-step-by-step-guide-c14c926c2229 https://www.youtube.com/watch?v=g6xBklAIrsA https://neuml.hashnode.dev/advanced-rag-with-graph-path-traversal +https://volodymyrpavlyshyn.medium.com/unified-knowledge-graph-model-rdf-rdf-vs-lpg-the-end-of-war-a7c14d6ac76f + + + ### Links From e6f5695b5ed4eece78f65d53e200145dc55db205 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 10:32:41 -0800 Subject: [PATCH 54/88] sweet, it works. Now to add rate limiting... Yep, got the perplexity pipeline implemented and working. Now need to handle rate limiting and making a placeholder gui for it. --- .../Web_Scraping/WebSearch_APIs.py | 37 ++++++++----------- setup.py | 5 ++- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 2d2fa264e..3f16637d4 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -2,12 +2,14 @@ # Description: This file contains the functions that are used for performing queries against various Search Engine APIs # # Imports +import asyncio import json import logging +from html import unescape +import pytest import random import re import time -from html import unescape from typing import Optional, Dict, Any, List from urllib.parse import urlparse, urlencode, unquote # @@ -23,6 +25,9 @@ # # Local Imports from App_Function_Libraries.Utils.Utils import loaded_config_data +from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_article + + # ####################################################################################################################### # @@ -169,28 +174,13 @@ def generate_and_search(question: str, search_params: Dict) -> Dict: } -def analyze_and_aggregate(web_search_results_dict: Dict, sub_query_dict: Dict, search_params: Dict) -> Dict: - """ - Analyzes the relevance of search results and aggregates the final answer. - - Args: - web_search_results_dict (Dict): A dictionary containing all search results and related metadata. - sub_query_dict (Dict): A dictionary containing sub-queries and related metadata. - search_params (Dict): A dictionary containing parameters for performing web searches - and specifying LLM endpoints. - - Returns: - Dict: A dictionary containing: - - final_answer: The aggregated final answer. - - relevant_results: The filtered relevant results. - - web_search_results_dict: The original search results and metadata. - """ +async def analyze_and_aggregate(web_search_results_dict: Dict, sub_query_dict: Dict, search_params: Dict) -> Dict: logging.info("Starting analyze_and_aggregate") # 4. Score/filter results logging.info("Scoring and filtering search results") sub_questions = sub_query_dict.get("sub_questions", []) - relevant_results = search_result_relevance( + relevant_results = await search_result_relevance( web_search_results_dict["results"], sub_query_dict["main_goal"], sub_questions, @@ -223,7 +213,8 @@ def analyze_and_aggregate(web_search_results_dict: Dict, sub_query_dict: Dict, s } -def test_perplexity_pipeline(): +@pytest.mark.asyncio +async def test_perplexity_pipeline(): # Phase 1: Generate sub-queries and perform web searches search_params = { "engine": "google", @@ -248,7 +239,7 @@ def test_perplexity_pipeline(): phase1_results = generate_and_search("What is the capital of France?", search_params) # Review the results here if needed # Phase 2: Analyze relevance and aggregate final answer - phase2_results = analyze_and_aggregate(phase1_results["web_search_results_dict"], phase1_results["sub_query_dict"], search_params) + phase2_results = await analyze_and_aggregate(phase1_results["web_search_results_dict"], phase1_results["sub_query_dict"], search_params) print(phase2_results["final_answer"]) @@ -344,7 +335,7 @@ def analyze_question(question: str, api_endpoint) -> Dict: ######################### Relevance Analysis ######################### # # FIXME - Ensure edge cases are handled properly / Structured outputs? -def search_result_relevance( +async def search_result_relevance( search_results: List[Dict], original_question: str, sub_questions: List[str], @@ -422,8 +413,10 @@ def search_result_relevance( if is_relevant: # Use the 'id' from the result if available, otherwise use idx result_id = result.get("id", str(idx)) + # Scrape the content of the relevant result + scraped_content = await scrape_article(result['url']) relevant_results[result_id] = { - "content": content, + "content": scraped_content['content'], "reasoning": reasoning } logging.info(f"Relevant result found: ID={result_id} Reasoning={reasoning}") diff --git a/setup.py b/setup.py index ebae4f552..9e081f195 100644 --- a/setup.py +++ b/setup.py @@ -20,4 +20,7 @@ "Operating System :: OS Independent", ], python_requires=">=3.9", -) \ No newline at end of file +) + +# Dev +pytest-asyncio \ No newline at end of file From b24c8b2ef79150516d4b47388965107b18153e34 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 10:45:51 -0800 Subject: [PATCH 55/88] websearch docs --- Docs/Design/Education.md | 1 + Docs/Design/WebSearch.md | 22 ++++++++++++++++++++-- README.md | 2 ++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index 96323c5ed..778425aec 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -12,6 +12,7 @@ https://github.com/thiswillbeyourgithub/AnkiAIUtils https://news.ycombinator.com/item?id=42534931 https://ankiweb.net/shared/info/1531888719 https://bbycroft.net/llm +https://github.com/met4citizen/TalkingHead one2manny diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index fa1846fbc..c76d64705 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -14,12 +14,30 @@ This page serves as documentation regarding the web search functionality within ### Current Status -- Bing, Brave, DDG, Google work. +- Bing, Brave, DDG, Google work for simple searches. Advanced search options are not fully working yet. + - Brave: https://api.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters + - Bing: https://docs.microsoft.com/en-us/rest/api/cognitiveservices-bingsearch/bing-web-api-v7-reference + - Google: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list - Baidu, SearX, Serper, Tavily, Yandex are not implemented yet. - Kagi & SearX are implemented but not working (Kagi because API access and SearX because I'm not sure why) - Parsing works for Bing, Brave, DDG, Google. - Currently no use of Structured outputs (to-do...) -- +- Full Pipeline works. + 1. User enters query + Search options + 2. Query is processed, sub-queries are generated(if specified) + 3. Each query is sent to the search engine API + 4. Search results are returned + 5. Each search result is analyzed, if relevant, the full page is fetched and stored in the results dict + 6. Results are then aggregated and presented to the user +- **To Do:** + 1. Implement rate limiting for search engines/LLM API calls + 2. Implement the saving options (Web Search DB - allow for saving search results in DB, File, Clipboard, Notes DB - allow for saving search report + citations in a note) + 2. User can also select which results are relevant, and which are not, and remove irrelevant results from the `web_search_results_dict` + 3. Implement the remaining search engines (Baidu, SearX, Serper, Tavily, Yandex) + 4. Implement the advanced search options + 5. Implement the structured outputs + 6. Implement the various output options (Style/Format) / Allow the user to customize the aggregation prompt + 7. Provide the user with follow-up questions + potential other questions, if these are selected, the content is added to the 'ongoing' document ---------------- diff --git a/README.md b/README.md index 911b788b3..860c16924 100644 --- a/README.md +++ b/README.md @@ -611,6 +611,8 @@ None of these companies exist to provide AI services in 2024. They’re only doi * https://www.getcoralai.com/ * https://getcahier.com/#features * https://msty.app/ + * https://afforai.com + * https://penno.io/ ------------ ### Credits From 9c6c7f6296cebd82733306953c8cd904d9ac1ae2 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 15:42:27 -0800 Subject: [PATCH 56/88] added 0.3-> 1.5 second pauses in requests. --- .../Web_Scraping/WebSearch_APIs.py | 14 ++++++-------- Docs/Design/WebSearch.md | 1 - 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 3f16637d4..538c087f4 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -20,14 +20,11 @@ from requests import RequestException from requests.adapters import HTTPAdapter from urllib3 import Retry - -from App_Function_Libraries.Chat.Chat_Functions import chat_api_call # # Local Imports from App_Function_Libraries.Utils.Utils import loaded_config_data from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_article - - +from App_Function_Libraries.Chat.Chat_Functions import chat_api_call # ####################################################################################################################### # @@ -38,8 +35,6 @@ # # Functions: - - ######################### Main Orchestration Workflow ######################### # # FIXME - Add Logging @@ -77,6 +72,7 @@ def initialize_web_search_results_dict(search_params: Dict) -> Dict: "processing_error": None } + def generate_and_search(question: str, search_params: Dict) -> Dict: """ Generates sub-queries (if enabled) and performs web searches for each query. @@ -132,6 +128,7 @@ def generate_and_search(question: str, search_params: Dict) -> Dict: # 3. Perform searches and accumulate all raw results for q in all_queries: + sleep_time = random.uniform(1, 1.5) # Add a random delay to avoid rate limiting logging.info(f"Performing web search for query: {q}") raw_results = perform_websearch( search_engine=search_params.get('engine'), @@ -187,8 +184,8 @@ async def analyze_and_aggregate(web_search_results_dict: Dict, sub_query_dict: D search_params.get('relevance_analysis_llm') ) # FIXME - print("Relevant results returned by search_result_relevance:") - print(json.dumps(relevant_results, indent=2)) + logging.debug("Relevant results returned by search_result_relevance:") + logging.debug(json.dumps(relevant_results, indent=2)) # 5. Allow user to review and select relevant results (if enabled) logging.info("Reviewing and selecting relevant results") @@ -381,6 +378,7 @@ async def search_result_relevance( try: # Perform API call to evaluate relevance + sleep_time = random.uniform(0.2, 0.6) # Add a random delay to avoid rate limiting relevancy_result = chat_api_call( api_endpoint=api_endpoint, api_key=None, diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index c76d64705..22b68a807 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -30,7 +30,6 @@ This page serves as documentation regarding the web search functionality within 5. Each search result is analyzed, if relevant, the full page is fetched and stored in the results dict 6. Results are then aggregated and presented to the user - **To Do:** - 1. Implement rate limiting for search engines/LLM API calls 2. Implement the saving options (Web Search DB - allow for saving search results in DB, File, Clipboard, Notes DB - allow for saving search report + citations in a note) 2. User can also select which results are relevant, and which are not, and remove irrelevant results from the `web_search_results_dict` 3. Implement the remaining search engines (Baidu, SearX, Serper, Tavily, Yandex) From 63dbc1f73fcdee22b4f8c943a204d7e6c33549f9 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 17:49:43 -0800 Subject: [PATCH 57/88] well config is 'broken' but better, will break it more though. Also streaming is validated as being in with at least openai Need to update the gradio ui to accommodate it though. --- App_Function_Libraries/LLM_API_Calls.py | 17 +- App_Function_Libraries/Utils/Utils.py | 314 +++++++++++++++++------- 2 files changed, 240 insertions(+), 91 deletions(-) diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 7569e8631..7c3ef6fd1 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -71,7 +71,7 @@ def get_openai_embeddings(input_data: str, model: str) -> List[float]: List[float]: The embeddings generated by the API. """ loaded_config_data = load_and_log_configs() - api_key = loaded_config_data['api_keys']['openai'] + api_key = loaded_config_data['openai_api']['api_key'] if not api_key: logging.error("OpenAI: API key not found or is empty") @@ -124,7 +124,7 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m if not openai_api_key: logging.info("OpenAI: API key not provided as parameter") logging.info("OpenAI: Attempting to use API key from config file") - openai_api_key = loaded_config_data['api_keys']['openai'] + openai_api_key = loaded_config_data['openai_api']['api_key'] if not openai_api_key: logging.error("OpenAI: API key not found or is empty") @@ -132,6 +132,16 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m logging.debug(f"OpenAI: Using API Key: {openai_api_key[:5]}...{openai_api_key[-5:]}") + streaming = loaded_config_data['openai_api']['streaming'] + if streaming == "true" or "True": + streaming = True + else: + streaming = False + if streaming: + logging.debug("OpenAI: Streaming mode enabled") + else: + logging.debug("OpenAI: Streaming mode disabled") + # Input data handling logging.debug(f"OpenAI: Raw input data type: {type(input_data)}") logging.debug(f"OpenAI: Raw input data (first 500 chars): {str(input_data)[:500]}...") @@ -177,7 +187,7 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m logging.debug(f"OpenAI: Extracted text (first 500 chars): {text[:500]}...") logging.debug(f"OpenAI: Custom prompt: {custom_prompt_arg}") - openai_model = loaded_config_data['models']['openai'] or "gpt-4o" + openai_model = loaded_config_data['openai_api']['model'] or "gpt-4o" logging.debug(f"OpenAI: Using model: {openai_model}") headers = { @@ -212,6 +222,7 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m json=data, stream=True ) + logging.debug(f"OpenAI: Response text: {response.text}") response.raise_for_status() def stream_generator(): diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index da418f29b..0e23bf66e 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -236,6 +236,44 @@ def load_and_log_configs(): mistral_model = config.get('API', 'mistral_model', fallback='mistral-large-latest') google_model = config.get('API', 'google_model', fallback='gemini-1.5-pro') + # LLM API Settings - streaming / temperature / top_p / min_p + anthropic_streaming = config.get('API', 'anthropic_streaming', fallback='False') + anthropic_temperature = config.get('API', 'anthropic_temperature', fallback='0.7') + anthropic_top_p = config.get('API', 'anthropic_top_p', fallback='0.95') + anthropic_min_p = config.get('API', 'anthropic_min_p', fallback='0.05') + cohore_streaming = config.get('API', 'cohere_streaming', fallback='False') + cohore_temperature = config.get('API', 'cohere_temperature', fallback='0.7') + cohore_top_p = config.get('API', 'cohere_top_p', fallback='0.95') + cohore_min_p = config.get('API', 'cohere_min_p', fallback='0.05') + groq_streaming = config.get('API', 'groq_streaming', fallback='False') + groq_temperature = config.get('API', 'groq_temperature', fallback='0.7') + groq_top_p = config.get('API', 'groq_top_p', fallback='0.95') + groq_min_p = config.get('API', 'groq_min_p', fallback='0.05') + openai_streaming = config.get('API', 'openai_streaming', fallback='False') + openai_temperature = config.get('API', 'openai_temperature', fallback='0.7') + openai_top_p = config.get('API', 'openai_top_p', fallback='0.95') + openai_min_p = config.get('API', 'openai_min_p', fallback='0.05') + huggingface_streaming = config.get('API', 'huggingface_streaming', fallback='False') + huggingface_temperature = config.get('API', 'huggingface_temperature', fallback='0.7') + huggingface_top_p = config.get('API', 'huggingface_top_p', fallback='0.95') + huggingface_min_p = config.get('API', 'huggingface_min_p', fallback='0.05') + openrouter_streaming = config.get('API', 'openrouter_streaming', fallback='False') + openrouter_temperature = config.get('API', 'openrouter_temperature', fallback='0.7') + openrouter_top_p = config.get('API', 'openrouter_top_p', fallback='0.95') + openrouter_min_p = config.get('API', 'openrouter_min_p', fallback='0.05') + deepseek_streaming = config.get('API', 'deepseek_streaming', fallback='False') + deepseek_temperature = config.get('API', 'deepseek_temperature', fallback='0.7') + deepseek_top_p = config.get('API', 'deepseek_top_p', fallback='0.95') + deepseek_min_p = config.get('API', 'deepseek_min_p', fallback='0.05') + mistral_streaming = config.get('API', 'mistral_streaming', fallback='False') + mistral_temperature = config.get('API', 'mistral_temperature', fallback='0.7') + mistral_top_p = config.get('API', 'mistral_top_p', fallback='0.95') + mistral_min_p = config.get('API', 'mistral_min_p', fallback='0.05') + google_streaming = config.get('API', 'google_streaming', fallback='False') + google_temperature = config.get('API', 'google_temperature', fallback='0.7') + google_top_p = config.get('API', 'google_top_p', fallback='0.95') + google_min_p = config.get('API', 'google_min_p', fallback='0.05') + logging.debug(f"Loaded Anthropic Model: {anthropic_model}") logging.debug(f"Loaded Cohere Model: {cohere_model}") logging.debug(f"Loaded Groq Model: {groq_model}") @@ -249,12 +287,25 @@ def load_and_log_configs(): kobold_api_ip = config.get('Local-API', 'kobold_api_IP', fallback='http://127.0.0.1:5000/api/v1/generate') kobold_openai_api_IP = config.get('Local-API', 'kobold_openai_api_IP', fallback='http://127.0.0.1:5001/v1/chat/completions') kobold_api_key = config.get('Local-API', 'kobold_api_key', fallback='') + kobold_streaming = config.get('Local-API', 'kobold_streaming', fallback='False') + kobold_temperature = config.get('Local-API', 'kobold_temperature', fallback='0.7') + kobold_top_p = config.get('Local-API', 'kobold_top_p', fallback='0.95') + kobold_min_p = config.get('Local-API', 'kobold_min_p', fallback='0.05') + llama_api_IP = config.get('Local-API', 'llama_api_IP', fallback='http://127.0.0.1:8080/v1/chat/completions') llama_api_key = config.get('Local-API', 'llama_api_key', fallback='') + llama_streaming = config.get('Local-API', 'llama_streaming', fallback='False') + llama_temperature = config.get('Local-API', 'llama_temperature', fallback='0.7') + llama_top_p = config.get('Local-API', 'llama_top_p', fallback='0.95') + llama_min_p = config.get('Local-API', 'llama_min_p', fallback='0.05') ooba_api_IP = config.get('Local-API', 'ooba_api_IP', fallback='http://127.0.0.1:5000/v1/chat/completions') ooba_api_key = config.get('Local-API', 'ooba_api_key', fallback='') + ooba_streaming = config.get('Local-API', 'ooba_streaming', fallback='False') + ooba_temperature = config.get('Local-API', 'ooba_temperature', fallback='0.7') + ooba_top_p = config.get('Local-API', 'ooba_top_p', fallback='0.95') + ooba_min_p = config.get('Local-API', 'ooba_min_p', fallback='0.05') tabby_api_IP = config.get('Local-API', 'tabby_api_IP', fallback='http://127.0.0.1:5000/api/v1/generate') tabby_api_key = config.get('Local-API', 'tabby_api_key', fallback=None) @@ -275,6 +326,8 @@ def load_and_log_configs(): custom_openai_api_url = config.get('API', 'custom_openai_url', fallback=None) logging.debug( f"Loaded Custom openai-like endpoint API Key: {custom_openai_api_key[:5]}...{custom_openai_api_key[-5:] if custom_openai_api_key else None}") + custom_openai_api_streaming = config.get('API', 'custom_openai_streaming', fallback='False') + custom_openai_api_temperature = config.get('API', 'custom_openai_temperature', fallback='0.7') logging.debug(f"Loaded Kobold API IP: {kobold_api_ip}") logging.debug(f"Loaded Llama API IP: {llama_api_IP}") @@ -285,6 +338,10 @@ def load_and_log_configs(): # Retrieve default API choices from the configuration file default_api = config.get('API', 'default_api', fallback='openai') + # Retrieve LLM API settings from the configuration file + local_api_retries = config.get('Local-API', 'Settings', fallback='3') + local_api_retry_delay = config.get('Local-API', 'local_api_retry_delay', fallback='5') + # Retrieve output paths from the configuration file output_path = config.get('Paths', 'output_path', fallback='results') logging.debug(f"Output path set to: {output_path}") @@ -397,51 +454,132 @@ def load_and_log_configs(): return { - 'api_keys': { - 'anthropic': anthropic_api_key, - 'cohere': cohere_api_key, - 'groq': groq_api_key, - 'openai': openai_api_key, - 'huggingface': huggingface_api_key, - 'openrouter': openrouter_api_key, - 'deepseek': deepseek_api_key, - 'mistral': mistral_api_key, - 'google': google_api_key, - 'kobold': kobold_api_key, - 'llama': llama_api_key, - 'ooba': ooba_api_key, - 'tabby': tabby_api_key, - 'vllm': vllm_api_key, - 'ollama': ollama_api_key, - 'aphrodite': aphrodite_api_key, - 'elevenlabs': elevenlabs_api_key, - 'custom_openai_api_key': custom_openai_api_key + 'anthropic_api': { + 'api_key': anthropic_api_key, + 'model': anthropic_model, + 'streaming': anthropic_streaming, + 'temperature': anthropic_temperature, + 'top_p': anthropic_top_p, + 'min_p': anthropic_min_p }, - 'models': { - 'anthropic': anthropic_model, - 'cohere': cohere_model, - 'groq': groq_model, - 'openai': openai_model, - 'huggingface': huggingface_model, - 'openrouter': openrouter_model, - 'deepseek': deepseek_model, - 'mistral': mistral_model, - 'google': google_model, - 'vllm': vllm_model, - 'tabby': tabby_model, - 'ollama': ollama_model - + 'cohere_api': { + 'api_key': cohere_api_key, + 'model': cohere_model, + 'streaming': cohore_streaming, + 'temperature': cohore_temperature, + 'top_p': cohore_top_p, + 'min_p': cohore_min_p + }, + 'groq_api': { + 'api_key': groq_api_key, + 'model': groq_model, + 'streaming': groq_streaming, + 'temperature': groq_temperature, + 'top_p': groq_top_p, + 'min_p': groq_min_p + }, + 'openai_api': { + 'api_key': openai_api_key, + 'model': openai_model, + 'streaming': openai_streaming, + 'temperature': openai_temperature, + 'top_p': openai_top_p, + 'min_p': openai_min_p + }, + 'huggingface_api': { + 'api_key': huggingface_api_key, + 'model': huggingface_model, + 'streaming': huggingface_streaming, + }, + 'openrouter_api': { + 'api_key': openrouter_api_key, + 'model': openrouter_model, + 'streaming': openrouter_streaming, + 'temperature': openrouter_temperature, + 'top_p': openrouter_top_p, + 'min_p': openrouter_min_p + }, + 'deepseek_api': { + 'api_key': deepseek_api_key, + 'model': deepseek_model, + 'streaming': deepseek_streaming, + 'temperature': deepseek_temperature, + 'top_p': deepseek_top_p, + 'min_p': deepseek_min_p + }, + 'mistral_api': { + 'api_key': mistral_api_key, + 'model': mistral_model, + 'streaming': mistral_streaming, + 'temperature': mistral_temperature, + 'top_p': mistral_top_p, + 'min_p': mistral_min_p + }, + 'google_api': { + 'api_key': google_api_key, + 'model': google_model, + 'streaming': google_streaming, + 'temperature': google_temperature, + 'top_p': google_top_p, + 'min_p': google_min_p + }, + 'elevenlabs_api': { + 'api_key': elevenlabs_api_key + }, + 'custom_openai_api': { + 'api_key': custom_openai_api_key, + 'api_url': custom_openai_api_url, + 'streaming': custom_openai_api_streaming, + 'temperature': custom_openai_api_temperature, + }, + 'llama_api': { + 'api_ip': llama_api_IP, + 'api_key': llama_api_key, + 'streaming': llama_streaming, + 'temperature': llama_temperature, + 'top_p': llama_top_p, + 'min_p': llama_min_p + }, + 'ooba_api': { + 'api_ip': ooba_api_IP, + 'api_key': ooba_api_key, + 'streaming': ooba_streaming, + 'temperature': ooba_temperature, + 'top_p': ooba_top_p, + 'min_p': ooba_min_p + }, + 'kobold_api': { + 'api_ip': kobold_api_ip, + 'api_key': kobold_api_key, + 'streaming': kobold_streaming, + 'temperature': kobold_temperature, + 'top_p': kobold_top_p, + 'min_p': kobold_min_p + }, + 'tabby_api': { + 'api_ip': tabby_api_IP, + 'api_key': tabby_api_key, + 'model': tabby_model + }, + 'vllm_api': { + 'api_url': vllm_api_url, + 'api_key': vllm_api_key, + 'model': vllm_model + }, + 'ollama_api': { + 'api_url': ollama_api_url, + 'api_key': ollama_api_key, + 'model': ollama_model + }, + 'aphrodite_api': { + 'api_url': aphrodite_api_url, + 'api_key': aphrodite_api_key }, - 'local_api_ip': { - 'kobold': kobold_api_ip, - 'kobold_openai': kobold_openai_api_IP, - 'llama': llama_api_IP, - 'ooba': ooba_api_IP, - 'tabby': tabby_api_IP, - 'vllm': vllm_api_url, - 'ollama': ollama_api_url, - 'aphrodite': aphrodite_api_url, - 'custom_openai_api_ip': custom_openai_api_url + 'llm_api_settings': { + 'default_api': default_api, + 'local_api_timeout': local_api_timeout, + 'local_api_retries': local_api_retries, + 'local_api_retry_delay': local_api_retry_delay, }, 'output_path': output_path, 'processing_choice': processing_choice, @@ -492,53 +630,53 @@ def load_and_log_configs(): # GPT Sovi-TTS }, 'search_settings': { - 'default_search_provider': search_provider_default, - 'search_language_query': search_language_query, - 'search_language_results': search_language_results, - 'search_language_analysis': search_language_analysis, - 'search_default_max_queries': search_default_max_queries, - 'search_enable_subquery': search_enable_subquery, - 'search_enable_subquery_count_max': search_enable_subquery_count_max, - 'search_result_rerank': search_result_rerank, - 'search_result_max': search_result_max, - 'search_result_max_per_query': search_result_max_per_query, - 'search_result_blacklist': search_result_blacklist, - 'search_result_display_type': search_result_display_type, - 'search_result_display_metadata': search_result_display_metadata, - 'search_result_save_to_db': search_result_save_to_db, - 'search_result_analysis_tone': search_result_analysis_tone, + 'default_search_provider': search_provider_default, + 'search_language_query': search_language_query, + 'search_language_results': search_language_results, + 'search_language_analysis': search_language_analysis, + 'search_default_max_queries': search_default_max_queries, + 'search_enable_subquery': search_enable_subquery, + 'search_enable_subquery_count_max': search_enable_subquery_count_max, + 'search_result_rerank': search_result_rerank, + 'search_result_max': search_result_max, + 'search_result_max_per_query': search_result_max_per_query, + 'search_result_blacklist': search_result_blacklist, + 'search_result_display_type': search_result_display_type, + 'search_result_display_metadata': search_result_display_metadata, + 'search_result_save_to_db': search_result_save_to_db, + 'search_result_analysis_tone': search_result_analysis_tone, }, 'search_engines': { - 'baidu_search_api_key': baidu_search_api_key, - 'bing_search_api_key': bing_search_api_key, - 'bing_country_code': bing_country_code, - 'bing_search_api_url': bing_search_api_url, - 'brave_search_api_key': brave_search_api_key, - 'brave_search_ai_api_key': brave_search_ai_api_key, - 'brave_country_code': brave_country_code, - 'duckduckgo_search_api_key': duckduckgo_search_api_key, - 'google_search_api_url': google_search_api_url, - 'google_search_api_key': google_search_api_key, - 'google_search_engine_id': google_search_engine_id, - 'google_simp_trad_chinese': google_simp_trad_chinese, - 'limit_google_search_to_country': limit_google_search_to_country, - 'google_search_country': google_search_country, - 'google_search_country_code': google_search_country_code, - 'google_search_filter_setting': google_filter_setting, - 'google_user_geolocation': google_user_geolocation, - 'google_ui_language': google_ui_language, - 'google_limit_search_results_to_language': google_limit_search_results_to_language, - 'google_site_search_include': google_site_search_include, - 'google_site_search_exclude': google_site_search_exclude, - 'google_sort_results_by': google_sort_results_by, - 'google_default_search_results': google_default_search_results, - 'google_safe_search': google_safe_search, - 'google_enable_site_search' : google_enable_site_search, - 'kagi_search_api_key': kagi_search_api_key, - 'searx_search_api_url': search_engine_searx_api, - 'tavily_search_api_key': tavily_search_api_key, - 'yandex_search_api_key': yandex_search_api_key, - 'yandex_search_engine_id': yandex_search_engine_id + 'baidu_search_api_key': baidu_search_api_key, + 'bing_search_api_key': bing_search_api_key, + 'bing_country_code': bing_country_code, + 'bing_search_api_url': bing_search_api_url, + 'brave_search_api_key': brave_search_api_key, + 'brave_search_ai_api_key': brave_search_ai_api_key, + 'brave_country_code': brave_country_code, + 'duckduckgo_search_api_key': duckduckgo_search_api_key, + 'google_search_api_url': google_search_api_url, + 'google_search_api_key': google_search_api_key, + 'google_search_engine_id': google_search_engine_id, + 'google_simp_trad_chinese': google_simp_trad_chinese, + 'limit_google_search_to_country': limit_google_search_to_country, + 'google_search_country': google_search_country, + 'google_search_country_code': google_search_country_code, + 'google_search_filter_setting': google_filter_setting, + 'google_user_geolocation': google_user_geolocation, + 'google_ui_language': google_ui_language, + 'google_limit_search_results_to_language': google_limit_search_results_to_language, + 'google_site_search_include': google_site_search_include, + 'google_site_search_exclude': google_site_search_exclude, + 'google_sort_results_by': google_sort_results_by, + 'google_default_search_results': google_default_search_results, + 'google_safe_search': google_safe_search, + 'google_enable_site_search' : google_enable_site_search, + 'kagi_search_api_key': kagi_search_api_key, + 'searx_search_api_url': search_engine_searx_api, + 'tavily_search_api_key': tavily_search_api_key, + 'yandex_search_api_key': yandex_search_api_key, + 'yandex_search_engine_id': yandex_search_engine_id } } From 4bf6179d096f4b1f59fc7ea5fa0fa2b391768d7d Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 19:26:32 -0800 Subject: [PATCH 58/88] Update LLM_API_Calls.py --- App_Function_Libraries/LLM_API_Calls.py | 39 +++++++++++++++---------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 7c3ef6fd1..7b8bfc0c0 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -295,7 +295,7 @@ def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retri logging.info("Anthropic: Attempting to use API key from config file") # Ensure 'api_keys' and 'anthropic' keys exist try: - anthropic_api_key = loaded_config_data['api_keys']['anthropic'] + anthropic_api_key = loaded_config_data['anthropic_api']['api_key'] logging.debug(f"Anthropic: Loaded API Key from config: {anthropic_api_key[:5]}...{anthropic_api_key[-5:]}") except (KeyError, TypeError) as e: logging.error(f"Anthropic: Error accessing API key from config: {str(e)}") @@ -323,7 +323,7 @@ def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retri # Retrieve the model from config if not provided if not model: try: - anthropic_model = loaded_config_data['models']['anthropic'] + anthropic_model = loaded_config_data['anthropic_api']['model'] logging.debug(f"Anthropic: Loaded model from config: {anthropic_model}") except (KeyError, TypeError) as e: logging.error(f"Anthropic: Error accessing model from config: {str(e)}") @@ -333,7 +333,9 @@ def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retri logging.debug(f"Anthropic: Using provided model: {anthropic_model}") if temp is None: + temperature = loaded_config_data['anthropic_api']['temperature'] temp = 1.0 + logging.debug(f"Anthropic: Using default temperature: {temp}") headers = { @@ -466,7 +468,7 @@ def chat_with_cohere(api_key, input_data, model=None, custom_prompt_arg=None, sy logging.info("Cohere Chat: API key not provided as parameter") logging.info("Cohere Chat: Attempting to use API key from config file") logging.debug(f"Cohere Chat: Cohere API Key from config: {loaded_config_data['api_keys']['cohere']}") - cohere_api_key = loaded_config_data['api_keys']['cohere'] + cohere_api_key = loaded_config_data['cohere_api']['api_key'] if cohere_api_key: logging.debug(f"Cohere Chat: Cohere API Key from config: {cohere_api_key[:3]}...{cohere_api_key[-3:]}") else: @@ -478,7 +480,7 @@ def chat_with_cohere(api_key, input_data, model=None, custom_prompt_arg=None, sy # Ensure model is set if not model: - model = loaded_config_data['models']['cohere'] + model = loaded_config_data['cohere_api']['model'] logging.debug(f"Cohere Chat: Using model: {model}") if temp is None: @@ -631,7 +633,7 @@ def chat_with_groq(api_key, input_data, custom_prompt_arg, temp=None, system_mes logging.info("Groq: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - groq_api_key = loaded_config_data['api_keys'].get('groq') + groq_api_key = loaded_config_data['groq_api'].get('api_Key') if groq_api_key: logging.info("Groq: Using API key from config file") else: @@ -645,6 +647,11 @@ def chat_with_groq(api_key, input_data, custom_prompt_arg, temp=None, system_mes logging.debug(f"Groq: Using API Key: {groq_api_key[:5]}...{groq_api_key[-5:]}") + streaming = loaded_config_data['groq_api']['streaming'] + if streaming == "true" or "True": + streaming = True + else: + streaming = False # Transcript data handling & Validation if isinstance(input_data, str) and os.path.isfile(input_data): logging.debug("Groq: Loading json data for summarization") @@ -673,7 +680,7 @@ def chat_with_groq(api_key, input_data, custom_prompt_arg, temp=None, system_mes raise ValueError("Groq: Invalid input data format") # Set the model to be used - groq_model = loaded_config_data['models']['groq'] + groq_model = loaded_config_data['groq_api']['model'] if temp is None: temp = 0.2 @@ -783,7 +790,7 @@ def chat_with_openrouter(api_key, input_data, custom_prompt_arg, temp=None, syst logging.info("OpenRouter: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - openrouter_api_key = loaded_config_data['api_keys'].get('openrouter') + openrouter_api_key = loaded_config_data['openrouter_api'].get('api_key') if openrouter_api_key: logging.info("OpenRouter: Using API key from config file") else: @@ -792,7 +799,7 @@ def chat_with_openrouter(api_key, input_data, custom_prompt_arg, temp=None, syst # Model Selection validation logging.debug("OpenRouter: Validating model selection") loaded_config_data = load_and_log_configs() - openrouter_model = loaded_config_data['models']['openrouter'] + openrouter_model = loaded_config_data['openrouter_api']['model'] logging.debug(f"OpenRouter: Using model from config file: {openrouter_model}") # Final check to ensure we have a valid API key @@ -951,7 +958,7 @@ def chat_with_huggingface(api_key, input_data, custom_prompt_arg, system_prompt= logging.info("HuggingFace Chat: API key not provided as parameter") logging.info("HuggingFace Chat: Attempting to use API key from config file") - huggingface_api_key = loaded_config_data['api_keys'].get('huggingface') + huggingface_api_key = loaded_config_data['huggingface_api'].get('api_key') logging.debug(f"HuggingFace Chat: API key from config: {huggingface_api_key[:5]}...{huggingface_api_key[-5:]}") if huggingface_api_key is None or huggingface_api_key.strip() == "": @@ -964,7 +971,7 @@ def chat_with_huggingface(api_key, input_data, custom_prompt_arg, system_prompt= } # Setup model - huggingface_model = loaded_config_data['models']['huggingface'] + huggingface_model = loaded_config_data['huggingface_api']['model'] API_URL = f"https://api-inference.huggingface.co/models/{huggingface_model}/v1/chat/completions" if temp is None: @@ -1065,7 +1072,7 @@ def chat_with_deepseek(api_key, input_data, custom_prompt_arg, temp=0.1, system_ logging.info("DeepSeek: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - deepseek_api_key = loaded_config_data['api_keys'].get('deepseek') + deepseek_api_key = loaded_config_data['deepseek_api'].get('api_key') if deepseek_api_key and deepseek_api_key.strip(): deepseek_api_key = deepseek_api_key.strip() logging.info("DeepSeek: Using API key from config file") @@ -1121,7 +1128,7 @@ def chat_with_deepseek(api_key, input_data, custom_prompt_arg, temp=0.1, system_ raise ValueError("DeepSeek: Invalid input data format") # Retrieve the model from config if not provided - deepseek_model = loaded_config_data['models'].get('deepseek', "deepseek-chat") + deepseek_model = loaded_config_data['deepseek_api'].get('deepseek', "deepseek-chat") logging.debug(f"DeepSeek: Using model: {deepseek_model}") # Ensure temperature is a float within acceptable range @@ -1252,7 +1259,7 @@ def chat_with_mistral(api_key, input_data, custom_prompt_arg, temp=None, system_ logging.info("Mistral: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - mistral_api_key = loaded_config_data['api_keys'].get('mistral') + mistral_api_key = loaded_config_data['mistral_api'].get('api_key') if mistral_api_key: logging.info("Mistral: Using API key from config file") else: @@ -1276,7 +1283,7 @@ def chat_with_mistral(api_key, input_data, custom_prompt_arg, temp=None, system_ else: raise ValueError("Mistral: Invalid input data format") - mistral_model = loaded_config_data['models'].get('mistral', "mistral-large-latest") + mistral_model = loaded_config_data['mistral_api'].get('model', "mistral-large-latest") if temp is None: temp = 0.2 @@ -1386,7 +1393,7 @@ def chat_with_google(api_key, input_data, custom_prompt_arg, temp=None, system_m if not google_api_key: logging.info("Google: API key not provided as parameter") logging.info("Google: Attempting to use API key from config file") - google_api_key = loaded_config_data['api_keys']['google'] + google_api_key = loaded_config_data['google_api']['api_key'] if not google_api_key: logging.error("Google: API key not found or is empty") @@ -1439,7 +1446,7 @@ def chat_with_google(api_key, input_data, custom_prompt_arg, temp=None, system_m logging.debug(f"Google: Extracted text (first 500 chars): {text[:500]}...") logging.debug(f"Google: Custom prompt: {custom_prompt_arg}") - google_model = loaded_config_data['models']['google'] or "gemini-1.5-pro" + google_model = loaded_config_data['google_api']['model'] or "gemini-1.5-pro" logging.debug(f"Google: Using model: {google_model}") headers = { From dfb49cfd37cdc85e1af845a71e1a6cf858656339 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 20:27:59 -0800 Subject: [PATCH 59/88] eh --- App_Function_Libraries/Chat/Chat_Functions.py | 19 +++++++++++-------- App_Function_Libraries/Gradio_UI/Chat_ui.py | 11 ++++++++--- App_Function_Libraries/LLM_API_Calls.py | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index 1bfd00f98..2b8f0a212 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -39,7 +39,7 @@ def approximate_token_count(history): total_tokens = len(total_text.split()) return total_tokens -def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None): +def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None, streaming=False): log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() if not api_key: @@ -50,7 +50,7 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag logging.info(f"Debug - Chat API Call - API Key: {api_key}") logging.info(f"Debug - Chat chat_api_call - API Endpoint: {api_endpoint}") if api_endpoint.lower() == 'openai': - response = chat_with_openai(api_key, input_data, prompt, temp, system_message) + response = chat_with_openai(api_key, input_data, prompt, temp, system_message, streaming) elif api_endpoint.lower() == 'anthropic': # Retrieve the model from config @@ -133,7 +133,7 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt, temperature, - system_message=None): + system_message=None, streaming=False): log_counter("chat_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() try: @@ -172,12 +172,15 @@ def chat(message, history, media_content, selected_parts, api_endpoint, api_key, logging.debug(f"Debug - Chat Function - Prompt: {prompt}") # Use the existing API request code based on the selected endpoint - response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message) + response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming) - chat_duration = time.time() - start_time - log_histogram("chat_duration", chat_duration, labels={"api_endpoint": api_endpoint}) - log_counter("chat_success", labels={"api_endpoint": api_endpoint}) - return response + if streaming: + return response + else: + chat_duration = time.time() - start_time + log_histogram("chat_duration", chat_duration, labels={"api_endpoint": api_endpoint}) + log_counter("chat_success", labels={"api_endpoint": api_endpoint}) + return response except Exception as e: log_counter("chat_error", labels={"api_endpoint": api_endpoint, "error": str(e)}) logging.error(f"Error in chat function: {str(e)}") diff --git a/App_Function_Libraries/Gradio_UI/Chat_ui.py b/App_Function_Libraries/Gradio_UI/Chat_ui.py index 6df00168d..66da49ef4 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_ui.py +++ b/App_Function_Libraries/Gradio_UI/Chat_ui.py @@ -342,6 +342,7 @@ def create_chat_interface(): with gr.Column(scale=2): chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container") msg = gr.Textbox(label="Enter your message") + streaming = gr.Checkbox(label="Streaming") submit = gr.Button("Submit") regenerate_button = gr.Button("Regenerate Last Message") token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) @@ -456,7 +457,7 @@ def on_next_page_click(current_page, total_pages): submit.click( chat_wrapper, inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, conversation_id, - save_conversation, temperature, system_prompt_input], + save_conversation, temperature, system_prompt_input, streaming], outputs=[msg, chatbot, conversation_id] ).then( # Clear the message box after submission lambda x: gr.update(value=""), @@ -577,7 +578,7 @@ def create_chat_interface_stacked(): use_summary = gr.Checkbox(label="Use Summary") use_prompt = gr.Checkbox(label="Use Prompt") save_conversation = gr.Checkbox(label="Save Conversation", value=False, visible=True) - temp = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7) + temp = gr.Slider(label="Temperature", minimum=0.00, maximum=2.0, step=0.05, value=0.7) with gr.Row(): conversation_search = gr.Textbox(label="Search Conversations") with gr.Row(): @@ -608,6 +609,10 @@ def create_chat_interface_stacked(): value=False, visible=True ) + streaming = gr.Checkbox(label="Streaming", + value=False, + visible=True + ) with gr.Row(): preset_prompt = gr.Dropdown( @@ -837,7 +842,7 @@ def on_next_page_click(current_page, total_pages): submit.click( chat_wrapper, inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, - conversation_id, save_conversation, temp, system_prompt], + conversation_id, save_conversation, temp, system_prompt, streaming], outputs=[msg, chatbot, conversation_id] ).then( lambda x: gr.update(value=""), diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 7b8bfc0c0..410d54353 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -116,7 +116,7 @@ def get_openai_embeddings(input_data: str, model: str) -> List[float]: raise ValueError(f"OpenAI: Unexpected error occurred: {str(e)}") -def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_message=None, streaming=False): +def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_message=None, streaming=None): loaded_config_data = load_and_log_configs() openai_api_key = api_key try: From 760645e900c2bc02eb77b32cc9a2e1478e6539e1 Mon Sep 17 00:00:00 2001 From: Robert Date: Tue, 31 Dec 2024 23:28:56 -0800 Subject: [PATCH 60/88] Update RAG_Notes.md --- Docs/RAG_Notes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index b163f3ec6..f1a728c09 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -46,6 +46,7 @@ https://medium.com/@eliot64/bridging-legal-ai-and-trust-how-we-won-the-llm-x-law https://arxiv.org/abs/2412.15563 https://arxiv.org/pdf/2410.10293v1 https://arxiv.org/pdf/2409.02098v1 +https://github.com/OpenSPG/KAG GraphRAG @@ -81,7 +82,7 @@ GraphRAG https://www.youtube.com/watch?v=g6xBklAIrsA https://neuml.hashnode.dev/advanced-rag-with-graph-path-traversal https://volodymyrpavlyshyn.medium.com/unified-knowledge-graph-model-rdf-rdf-vs-lpg-the-end-of-war-a7c14d6ac76f - + https://github.com/zjunlp/OneKE From c9286acddc5e88ae39bb9df73cc022c37c29936a Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 00:38:26 -0800 Subject: [PATCH 61/88] streaming is now in and works. Need to update the surrounding chat functionality though.... --- App_Function_Libraries/Chat/Chat_Functions.py | 5 +- App_Function_Libraries/Gradio_UI/Chat_ui.py | 48 +++++++++---------- App_Function_Libraries/LLM_API_Calls.py | 11 +++-- App_Function_Libraries/Utils/Utils.py | 16 +++---- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index 2b8f0a212..194313e2b 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -55,12 +55,15 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag elif api_endpoint.lower() == 'anthropic': # Retrieve the model from config loaded_config_data = load_and_log_configs() - model = loaded_config_data['models']['anthropic'] if loaded_config_data else None + if not model: + model = loaded_config_data['anthropic_api']['model'] response = chat_with_anthropic( api_key=api_key, input_data=input_data, model=model, custom_prompt_arg=prompt, + max_retries=3, + retry_delay=5, system_prompt=system_message ) diff --git a/App_Function_Libraries/Gradio_UI/Chat_ui.py b/App_Function_Libraries/Gradio_UI/Chat_ui.py index 66da49ef4..521a15ccc 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_ui.py +++ b/App_Function_Libraries/Gradio_UI/Chat_ui.py @@ -85,7 +85,7 @@ def clear_chat_single(): # FIXME - add additional features.... def chat_wrapper(message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, conversation_id, - save_conversation, temperature, system_prompt, max_tokens=None, top_p=None, frequency_penalty=None, + save_conversation, temperature, system_prompt, streaming=False, max_tokens=None, top_p=None, frequency_penalty=None, presence_penalty=None, stop_sequence=None): try: if save_conversation: @@ -107,22 +107,21 @@ def chat_wrapper(message, history, media_content, selected_parts, api_endpoint, full_message = message # Generate bot response - bot_message = chat(full_message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, - temperature, system_prompt) - - logging.debug(f"Bot message being returned: {bot_message}") + bot_message = "" + for chunk in chat(full_message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, + temperature, system_prompt, streaming): + bot_message += chunk # Accumulate the streamed response + logging.debug(f"Bot message being returned: {bot_message}") + # Yield the incremental response and updated history + yield bot_message, history + [(message, bot_message)], conversation_id if save_conversation: # Add assistant message to the database save_message(conversation_id, role="assistant", content=bot_message) - # Update history - new_history = history + [(message, bot_message)] - - return bot_message, new_history, conversation_id except Exception as e: logging.error(f"Error in chat wrapper: {str(e)}") - return "An error occurred.", history, conversation_id + yield "An error occurred.", history, conversation_id def search_conversations(query): @@ -174,7 +173,7 @@ def load_conversation(conversation_id): def regenerate_last_message(history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, - system_prompt): + system_prompt, streaming=False): if not history: return history, "No messages to regenerate." @@ -189,20 +188,19 @@ def regenerate_last_message(history, media_content, selected_parts, api_endpoint if not last_user_message: return new_history, "No user message to regenerate the bot response." - full_message = last_user_message - - bot_message = chat( - full_message, - new_history, - media_content, - selected_parts, - api_endpoint, - api_key, - custom_prompt, - temperature, - system_prompt - ) + # Generate the new bot response + bot_message = "" + if streaming: + # For streaming, consume the generator and accumulate the response + for chunk in chat(last_user_message, new_history, media_content, selected_parts, api_endpoint, api_key, + custom_prompt, temperature, system_prompt, streaming): + bot_message += chunk + else: + # For non-streaming, get the complete response directly + bot_message = next(chat(last_user_message, new_history, media_content, selected_parts, api_endpoint, api_key, + custom_prompt, temperature, system_prompt, streaming)) + # Update the history with the regenerated message new_history.append((last_user_message, bot_message)) return new_history, "Last message regenerated successfully." @@ -522,7 +520,7 @@ def on_next_page_click(current_page, total_pages): regenerate_button.click( regenerate_last_message, inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature, - system_prompt_input], + system_prompt_input, streaming], outputs=[chatbot, save_status] ).then( lambda history: approximate_token_count(history), diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 410d54353..330b3c0d1 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -321,7 +321,7 @@ def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retri logging.debug(f"AnthropicAI: Type of data: {type(input_data)}") # Retrieve the model from config if not provided - if not model: + if model is None: try: anthropic_model = loaded_config_data['anthropic_api']['model'] logging.debug(f"Anthropic: Loaded model from config: {anthropic_model}") @@ -333,9 +333,10 @@ def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retri logging.debug(f"Anthropic: Using provided model: {anthropic_model}") if temp is None: - temperature = loaded_config_data['anthropic_api']['temperature'] - temp = 1.0 - + temp = loaded_config_data['anthropic_api']['temperature'] + temp = float(temp) + else: + temp = 0.7 logging.debug(f"Anthropic: Using default temperature: {temp}") headers = { @@ -467,7 +468,7 @@ def chat_with_cohere(api_key, input_data, model=None, custom_prompt_arg=None, sy else: logging.info("Cohere Chat: API key not provided as parameter") logging.info("Cohere Chat: Attempting to use API key from config file") - logging.debug(f"Cohere Chat: Cohere API Key from config: {loaded_config_data['api_keys']['cohere']}") + logging.debug(f"Cohere Chat: Cohere API Key from config: {loaded_config_data['cohere_api']['api_key']}") cohere_api_key = loaded_config_data['cohere_api']['api_key'] if cohere_api_key: logging.debug(f"Cohere Chat: Cohere API Key from config: {cohere_api_key[:3]}...{cohere_api_key[-3:]}") diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 0e23bf66e..d25ddd7f0 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -241,10 +241,10 @@ def load_and_log_configs(): anthropic_temperature = config.get('API', 'anthropic_temperature', fallback='0.7') anthropic_top_p = config.get('API', 'anthropic_top_p', fallback='0.95') anthropic_min_p = config.get('API', 'anthropic_min_p', fallback='0.05') - cohore_streaming = config.get('API', 'cohere_streaming', fallback='False') - cohore_temperature = config.get('API', 'cohere_temperature', fallback='0.7') - cohore_top_p = config.get('API', 'cohere_top_p', fallback='0.95') - cohore_min_p = config.get('API', 'cohere_min_p', fallback='0.05') + cohere_streaming = config.get('API', 'cohere_streaming', fallback='False') + cohere_temperature = config.get('API', 'cohere_temperature', fallback='0.7') + cohere_top_p = config.get('API', 'cohere_top_p', fallback='0.95') + cohere_min_p = config.get('API', 'cohere_min_p', fallback='0.05') groq_streaming = config.get('API', 'groq_streaming', fallback='False') groq_temperature = config.get('API', 'groq_temperature', fallback='0.7') groq_top_p = config.get('API', 'groq_top_p', fallback='0.95') @@ -465,10 +465,10 @@ def load_and_log_configs(): 'cohere_api': { 'api_key': cohere_api_key, 'model': cohere_model, - 'streaming': cohore_streaming, - 'temperature': cohore_temperature, - 'top_p': cohore_top_p, - 'min_p': cohore_min_p + 'streaming': cohere_streaming, + 'temperature': cohere_temperature, + 'top_p': cohere_top_p, + 'min_p': cohere_min_p }, 'groq_api': { 'api_key': groq_api_key, From 00f55910f3568dd3e4c28becd29ade5caa344272 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 00:50:22 -0800 Subject: [PATCH 62/88] fix local LLM config loads --- App_Function_Libraries/LLM_API_Calls_Local.py | 42 +++++++++---------- App_Function_Libraries/Utils/Utils.py | 5 ++- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/App_Function_Libraries/LLM_API_Calls_Local.py b/App_Function_Libraries/LLM_API_Calls_Local.py index 5bb721551..058190c4b 100644 --- a/App_Function_Libraries/LLM_API_Calls_Local.py +++ b/App_Function_Libraries/LLM_API_Calls_Local.py @@ -135,7 +135,7 @@ def chat_with_llama(input_data, custom_prompt, temp, api_url="http://127.0.0.1:8 if api_key is None: logging.info("llama.cpp: API key not provided as parameter") logging.info("llama.cpp: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['llama'] + api_key = loaded_config_data['llama_api']['api_key'] if api_key is None or api_key.strip() == "": logging.info("llama.cpp: API key not found or is empty") @@ -145,7 +145,7 @@ def chat_with_llama(input_data, custom_prompt, temp, api_url="http://127.0.0.1:8 if api_url is None: logging.info("llama.cpp: API URL not provided as parameter") logging.info("llama.cpp: Attempting to use API URL from config file") - api_url = loaded_config_data['local_api_ip']['llama'] + api_url = loaded_config_data['llama_api']['api_ip'] if api_url is None or api_url.strip() == "": logging.info("llama.cpp: API URL not found or is empty") @@ -260,7 +260,7 @@ def chat_with_kobold(input_data, api_key, custom_prompt_input, kobold_api_ip="ht logging.info("Kobold: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - kobold_api_key = loaded_config_data['api_keys'].get('kobold') + kobold_api_key = loaded_config_data['kobold_api'].get('api_key') if kobold_api_key: logging.info("Kobold: Using API key from config file") else: @@ -314,14 +314,14 @@ def chat_with_kobold(input_data, api_key, custom_prompt_input, kobold_api_ip="ht logging.debug("kobold: Submitting request to API endpoint") print("kobold: Submitting request to API endpoint") - kobold_api_ip = loaded_config_data['local_api_ip']['kobold'] + kobold_api_ip = loaded_config_data['kobold_api']['api_ip'] if streaming: logging.debug("Kobold Summarization: Streaming mode enabled") try: # Send the request with streaming enabled # Get the Streaming API IP from the config - kobold_openai_api_IP = loaded_config_data['local_api_ip']['kobold_openai'] + kobold_openai_api_IP = loaded_config_data['kobold_api']['api_streaming_ip'] response = requests.post( kobold_openai_api_IP, headers=headers, json=data, stream=True ) @@ -421,7 +421,7 @@ def chat_with_oobabooga(input_data, api_key, custom_prompt, api_url="http://127. if api_key is None: logging.info("ooba: API key not provided as parameter") logging.info("ooba: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['ooba'] + api_key = loaded_config_data['ooba_api']['api_key'] if api_key is None or api_key.strip() == "": logging.info("ooba: API key not found or is empty") @@ -540,15 +540,15 @@ def chat_with_tabbyapi( logging.info("TabbyAPI: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - tabby_api_key = loaded_config_data['api_keys'].get('tabby') + tabby_api_key = loaded_config_data['tabby_api'].get('api_key') if tabby_api_key: logging.info("TabbyAPI: Using API key from config file") else: logging.warning("TabbyAPI: No API key found in config file") # Set API IP and model from config.txt - tabby_api_ip = loaded_config_data['local_api_ip']['tabby'] - tabby_model = loaded_config_data['models']['tabby'] + tabby_api_ip = loaded_config_data['tabby_api']['api_ip'] + tabby_model = loaded_config_data['tabby_api']['model'] if temp is None: temp = 0.7 @@ -655,12 +655,12 @@ def chat_with_tabbyapi( # FIXME aphrodite engine - code was literally tab complete in one go from copilot... :/ def chat_with_aphrodite(input_data, custom_prompt_input, api_key=None, api_IP="http://127.0.0.1:8080/completion"): loaded_config_data = load_and_log_configs() - model = loaded_config_data['models']['aphrodite'] + model = loaded_config_data['aphrodite_api']['model'] # API key validation if api_key is None: logging.info("aphrodite: API key not provided as parameter") logging.info("aphrodite: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['aphrodite'] + api_key = loaded_config_data['aphrodite_api']['api_key'] if api_key is None or api_key.strip() == "": logging.info("aphrodite: API key not found or is empty") @@ -707,7 +707,7 @@ def chat_with_ollama( logging.info("Ollama: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - ollama_api_key = loaded_config_data['api_keys'].get('ollama') + ollama_api_key = loaded_config_data['ollama_api'].get('api_key') if ollama_api_key: logging.info("Ollama: Using API key from config file") else: @@ -715,14 +715,14 @@ def chat_with_ollama( # Set model from parameter or config if model is None: - model = loaded_config_data['models'].get('ollama') + model = loaded_config_data['ollama_api'].get('model') if model is None: logging.error("Ollama: Model not found in config file") return "Ollama: Model not found in config file" # Set api_url from parameter or config if api_url is None: - api_url = loaded_config_data['local_api_ip'].get('ollama') + api_url = loaded_config_data['ollama_api'].get('api_ip') if api_url is None: logging.error("Ollama: API URL not found in config file") return "Ollama: API URL not found in config file" @@ -910,13 +910,13 @@ def chat_with_vllm( logging.info("vLLM: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - vllm_api_key = loaded_config_data['api_keys'].get('vllm') + vllm_api_key = loaded_config_data['vllm_api'].get('api_key') if vllm_api_key: logging.info("vLLM: Using API key from config file") else: logging.warning("vLLM: No API key found in config file") - if 'vllm' in loaded_config_data['local_api_ip']: - vllm_api_url = loaded_config_data['local_api_ip']['vllm'] + if 'api_ip' in loaded_config_data['vllm_api']: + vllm_api_url = loaded_config_data['vllm_api']['api_ip'] logging.info(f"vLLM: Using API URL from config file: {vllm_api_url}") else: logging.error("vLLM: API URL not found in config file") @@ -927,7 +927,7 @@ def chat_with_vllm( if system_prompt is None: system_prompt = "You are a helpful AI assistant." - model = model or loaded_config_data['models']['vllm'] + model = model or loaded_config_data['vllm_api']['model'] if system_prompt is None: system_prompt = "You are a helpful AI assistant." @@ -1015,7 +1015,7 @@ def chat_with_custom_openai(api_key, input_data, custom_prompt_arg, temp=None, s if not custom_openai_api_key: logging.info("Custom OpenAI API: API key not provided as parameter") logging.info("Custom OpenAI API: Attempting to use API key from config file") - custom_openai_api_key = loaded_config_data['api_keys']['custom_openai_api_key'] + custom_openai_api_key = loaded_config_data['custom_openai_api']['api_key'] if not custom_openai_api_key: logging.error("Custom OpenAI API: API key not found or is empty") @@ -1068,7 +1068,7 @@ def chat_with_custom_openai(api_key, input_data, custom_prompt_arg, temp=None, s logging.debug(f"Custom OpenAI API: Extracted text (first 500 chars): {text[:500]}...") logging.debug(f"v: Custom prompt: {custom_prompt_arg}") - openai_model = loaded_config_data['models']['openai'] or "gpt-4o" + openai_model = loaded_config_data['custom_openai_api']['model'] logging.debug(f"Custom OpenAI API: Using model: {openai_model}") headers = { @@ -1096,7 +1096,7 @@ def chat_with_custom_openai(api_key, input_data, custom_prompt_arg, temp=None, s "stream": streaming } - custom_openai_url = loaded_config_data['Local_api_ip']['custom_openai_api_ip'] + custom_openai_url = loaded_config_data['custom_openai_api']['api_ip'] if streaming: response = requests.post( diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index d25ddd7f0..5dfcff3c1 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -321,6 +321,7 @@ def load_and_log_configs(): aphrodite_api_url = config.get('Local-API', 'aphrodite_api_IP', fallback='http://127.0.0.1:8080/v1/chat/completions') aphrodite_api_key = config.get('Local-API', 'aphrodite_api_key', fallback='') + aphrodite_model = config.get('Local-API', 'aphrodite_model', fallback='') custom_openai_api_key = config.get('API', 'custom_openai_api_key', fallback=None) custom_openai_api_url = config.get('API', 'custom_openai_url', fallback=None) @@ -550,6 +551,7 @@ def load_and_log_configs(): }, 'kobold_api': { 'api_ip': kobold_api_ip, + 'api_streaming_ip': kobold_openai_api_IP, 'api_key': kobold_api_key, 'streaming': kobold_streaming, 'temperature': kobold_temperature, @@ -573,7 +575,8 @@ def load_and_log_configs(): }, 'aphrodite_api': { 'api_url': aphrodite_api_url, - 'api_key': aphrodite_api_key + 'api_key': aphrodite_api_key, + 'model': aphrodite_model, }, 'llm_api_settings': { 'default_api': default_api, From fbbb97fe48fcfbf36f196b81b64376e53134834a Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 01:21:40 -0800 Subject: [PATCH 63/88] At least one works --- App_Function_Libraries/TTS/TTS.py | 2 +- App_Function_Libraries/TTS/TTS_Providers.py | 61 +++++++++++--------- App_Function_Libraries/Utils/Utils.py | 62 ++++++++++----------- 3 files changed, 66 insertions(+), 59 deletions(-) diff --git a/App_Function_Libraries/TTS/TTS.py b/App_Function_Libraries/TTS/TTS.py index 4c050c779..5bba51de2 100644 --- a/App_Function_Libraries/TTS/TTS.py +++ b/App_Function_Libraries/TTS/TTS.py @@ -129,7 +129,7 @@ def merge_audio(audio_segments: List[AudioData]) -> AudioData: return b"" -# Main TTS function +# Main Podcast generation function def generate_tts_podcast( text: str, provider: str, diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 1e9ceff25..0225a3ae8 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -24,7 +24,7 @@ # OpenAI TTS Provider Functions # https://github.com/leokwsw/OpenAI-TTS-Gradio/blob/main/app.py -def generate_audio_openai(api_key, input_text, voice, model, response_format="mp3", output_file="speech.mp3"): +def generate_audio_openai(api_key, input_text, voice, model, response_format="mp3", output_file="speech.mp3", streaming=False): """ Generate audio using OpenAI's Text-to-Speech API. @@ -44,21 +44,14 @@ def generate_audio_openai(api_key, input_text, voice, model, response_format="mp RuntimeError: If the API request fails. """ # Validate inputs - loaded_config_data = load_and_log_configs() - openai_api_key = api_key # API key validation try: - if not openai_api_key: + if api_key == None: logging.info("OpenAI: API key not provided as parameter") logging.info("OpenAI: Attempting to use API key from config file") - openai_api_key = loaded_config_data['api_keys']['openai'] - - if not openai_api_key: - logging.error("OpenAI: API key not found or is empty") - return "OpenAI: API Key Not Provided/Found in Config file or is empty" - - logging.debug(f"OpenAI: Using API Key: {openai_api_key[:5]}...{openai_api_key[-5:]}") + api_key = loaded_config_data['openai_api']['api_key'] + logging.debug(f"OpenAI: Using API Key: {api_key[:5]}...{api_key[-5:]}") except Exception as e: logging.error(f"OpenAI: Error loading API Key: {str(e)}") return f"OpenAI: Error loading API Key: {str(e)}" @@ -115,28 +108,42 @@ def generate_audio_openai(api_key, input_text, voice, model, response_format="mp "voice": voice, } - try: - # Make the request to the API - response = requests.post(endpoint, headers=headers, json=payload, stream=True) - response.raise_for_status() # Raise an error for HTTP status codes >= 400 + if streaming == True: + try: + # Make the request to the API + response = requests.post(endpoint, headers=headers, json=payload, stream=True) + response.raise_for_status() # Raise an error for HTTP status codes >= 400 - # Save the audio response to a file - with open(output_file, "wb") as f: - f.write(response.content) + # Save the audio response to a file + with open(output_file, "wb") as f: + f.write(response.content) - print(f"Audio successfully generated and saved to {output_file}.") - return output_file + print(f"Audio successfully generated and saved to {output_file}.") + return output_file - except requests.exceptions.RequestException as e: - raise RuntimeError(f"Failed to generate audio: {str(e)}") from e + except requests.exceptions.RequestException as e: + raise RuntimeError(f"Failed to generate audio: {str(e)}") from e + else: + try: + # Make the request to the API + response = requests.post(endpoint, headers=headers, json=payload) + response.raise_for_status() # Raise an error for HTTP status codes >= 400 + # Save the audio response to a file + with open(output_file, "wb") as f: + f.write(response.content) -def test_generate_audio_openai(api_key, input_text, voice, model, response_format="mp3", output_file="speech.mp3"): + print(f"Audio successfully generated and saved to {output_file}.") + return output_file + + except requests.exceptions.RequestException as e: + raise RuntimeError(f"Failed to generate audio: {str(e)}") from e + + +def test_generate_audio_openai(): try: - if not api_key: - logging.info("OpenAI: API key not provided as parameter") - logging.info("OpenAI: Attempting to use API key from config file") - openai_api_key = loaded_config_data['api_keys']['openai'] + logging.info("OpenAI: Attempting to use API key from config file") + api_key = loaded_config_data['openai_api']['api_key'] if not api_key: logging.error("OpenAI: API key not found or is empty") diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 5dfcff3c1..ad7e87e80 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -471,6 +471,22 @@ def load_and_log_configs(): 'top_p': cohere_top_p, 'min_p': cohere_min_p }, + 'deepseek_api': { + 'api_key': deepseek_api_key, + 'model': deepseek_model, + 'streaming': deepseek_streaming, + 'temperature': deepseek_temperature, + 'top_p': deepseek_top_p, + 'min_p': deepseek_min_p + }, + 'google_api': { + 'api_key': google_api_key, + 'model': google_model, + 'streaming': google_streaming, + 'temperature': google_temperature, + 'top_p': google_top_p, + 'min_p': google_min_p + }, 'groq_api': { 'api_key': groq_api_key, 'model': groq_model, @@ -479,35 +495,11 @@ def load_and_log_configs(): 'top_p': groq_top_p, 'min_p': groq_min_p }, - 'openai_api': { - 'api_key': openai_api_key, - 'model': openai_model, - 'streaming': openai_streaming, - 'temperature': openai_temperature, - 'top_p': openai_top_p, - 'min_p': openai_min_p - }, 'huggingface_api': { 'api_key': huggingface_api_key, 'model': huggingface_model, 'streaming': huggingface_streaming, }, - 'openrouter_api': { - 'api_key': openrouter_api_key, - 'model': openrouter_model, - 'streaming': openrouter_streaming, - 'temperature': openrouter_temperature, - 'top_p': openrouter_top_p, - 'min_p': openrouter_min_p - }, - 'deepseek_api': { - 'api_key': deepseek_api_key, - 'model': deepseek_model, - 'streaming': deepseek_streaming, - 'temperature': deepseek_temperature, - 'top_p': deepseek_top_p, - 'min_p': deepseek_min_p - }, 'mistral_api': { 'api_key': mistral_api_key, 'model': mistral_model, @@ -516,13 +508,21 @@ def load_and_log_configs(): 'top_p': mistral_top_p, 'min_p': mistral_min_p }, - 'google_api': { - 'api_key': google_api_key, - 'model': google_model, - 'streaming': google_streaming, - 'temperature': google_temperature, - 'top_p': google_top_p, - 'min_p': google_min_p + 'openrouter_api': { + 'api_key': openrouter_api_key, + 'model': openrouter_model, + 'streaming': openrouter_streaming, + 'temperature': openrouter_temperature, + 'top_p': openrouter_top_p, + 'min_p': openrouter_min_p + }, + 'openai_api': { + 'api_key': openai_api_key, + 'model': openai_model, + 'streaming': openai_streaming, + 'temperature': openai_temperature, + 'top_p': openai_top_p, + 'min_p': openai_min_p }, 'elevenlabs_api': { 'api_key': elevenlabs_api_key From 8042e06352bbb97f8aa370c92a1fe058bbfa978b Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 01:26:28 -0800 Subject: [PATCH 64/88] Update Utils.py --- App_Function_Libraries/Utils/Utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index ad7e87e80..dda80161b 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -381,6 +381,7 @@ def load_and_log_configs(): default_openai_tts_model = config.get('TTS-Settings', 'default_openai_tts_model', fallback='tts-1-hd') default_openai_tts_voice = config.get('TTS-Settings', 'default_openai_tts_voice', fallback='shimmer') default_openai_tts_speed = config.get('TTS-Settings', 'default_openai_tts_speed', fallback='1') + default_openai_tts_output_format = config.get('TTS-Settings', 'default_openai_tts_output_format', fallback='mp3') # Google TTS # FIXME - FIX THESE DEFAULTS default_google_tts_model = config.get('TTS-Settings', 'default_google_tts_model', fallback='en') @@ -617,6 +618,7 @@ def load_and_log_configs(): 'default_openai_tts_voice': default_openai_tts_voice, 'default_openai_tts_speed': default_openai_tts_speed, 'default_openai_tts_model': default_openai_tts_model, + 'default_openai_tts_output_format': default_openai_tts_output_format, # Google 'default_google_tts_model': default_google_tts_model, 'default_google_tts_voice': default_google_tts_voice, From 3f37d00e6a579d9644296fbcca900bdb6f5f5be6 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 12:31:35 -0800 Subject: [PATCH 65/88] Update Backup_Config.txt --- Config_Files/Backup_Config.txt | 183 ++++++++++++++++++++++++++++-- Docs/Design/Diagram_Generation.md | 9 ++ Docs/Design/Education.md | 3 +- Docs/Design/Researcher.md | 19 +++- Docs/Design/TTS_STT.md | 8 ++ Docs/RAG_Notes.md | 1 + 6 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 Docs/Design/Diagram_Generation.md diff --git a/Config_Files/Backup_Config.txt b/Config_Files/Backup_Config.txt index 1ef21b91b..e56be9be8 100644 --- a/Config_Files/Backup_Config.txt +++ b/Config_Files/Backup_Config.txt @@ -1,71 +1,237 @@ [API] anthropic_api_key = anthropic_model = claude-3-5-sonnet-20240620 +anthropic_streaming = True +anthropic_temperature = 0.7 +# cohere_api_key = cohere_model = command-r-plus +cohere_streaming = True +cohere_temperature = 0.7 +# groq_api_key = groq_model = llama3-70b-8192 +groq_streaming = True +groq_temperature = 0.7 +# openai_api_key = openai_model = gpt-4o +openai_streaming = False +openai_temperature = 0.7 +# huggingface_api_key = huggingface_model = meta-llama/Llama-3.1-70B-Instruct +huggingface_streaming = True +huggingface_temperature = 0.7 +# openrouter_api_key = openrouter_model = mistralai/mistral-7b-instruct:free +# deepseek_api_key = deepseek_model = deepseek-chat +deepseek_streaming = True +deepseek_temperature = 0.7 +# mistral_api_key = mistral_model = mistral-large-latest +mistral_streaming = True +mistral_temperature = 0.7 +# google_api_key = +# Available Model Options: google_model = gemini-1.5-pro +google_streaming = True +google_temperature = 0.7 +# +elevenlabs_api_key = +# custom_openai_api_key = custom_openai_api_ip = +custom_openai_api_streaming = True +custom_openai_api_temperature = 0.7 +custom_openai_api_top_p = 0.9 +custom_openai_api_min_p = 0.05 +# default_api = openai + [Local-API] kobold_api_IP = http://127.0.0.1:5001/api/v1/generate kobold_openai_api_IP = http://127.0.0.1:5001/v1/chat/completions kobold_api_key = +kobold_streaming = True +kobold_temperature = 0.7 +kobold_top_p = 0.9 +kobold_min_p = 0.05 +# llama_api_IP = http://127.0.0.1:8080/completion llama_api_key = +llama_streaming = True +llama_temperature = 0.7 +llama_top_p = 0.9 +llama_min_p = 0.05 +# ooba_api_key = ooba_api_IP = http://127.0.0.1:5000/v1/chat/completions +ooba_streaming = True +ooba_temperature = 0.7 +ooba_top_p = 0.9 +ooba_min_p = 0.05 +# tabby_api_IP = http://127.0.0.1:5000/v1/chat/completions tabby_api_key = +tabby_streaming = True +tabby_temperature = 0.7 +# vllm_api_IP = http://127.0.0.1:8000/v1/chat/completions vllm_model = +vllm_api_key = +vllm_streaming = True +vllm_temperature = 0.7 +# ollama_api_IP = http://127.0.0.1:11434/v1/chat/completions ollama_api_key = ollama_model = llama3 +ollama_streaming = True +# aphrodite_api_IP = http://127.0.0.1:8080/completion aphrodite_api_key = +aphrodite_streaming = True +aphrodite_temperature = 0.7 +aphrodite_top_p = 0.9 +aphrodite_min_p = 0.05 +aphrodite_model = +# max_tokens = 4096 +local_api_timeout = 90 +local_api_retries = 3 +local_api_retry_delay = 5 +streaming = True +temperature = 0.7 +top_p = 0.9 +min_p = 0.05 +# https://artefact2.github.io/llm-sampling/ [Processing] processing_choice = cuda [Settings] +# Rename to LLM_API_Settings chunk_duration = 30 words_per_second = 3 save_character_chats = False save_rag_chats = False -local_api_timeout = 90 -local_api_retries = 3 -local_api_retry_delay = 5 -streaming = True -temperature = 0.7 -top_p = 0.9 -min_p = 0.05 -# https://artefact2.github.io/llm-sampling/ [Auto-Save] save_character_chats = False save_rag_chats = False +[TTS-Settings] +# General TTS Settings +default_tts_provider = openai +default_tts_voice = shimmer +default_tts_speed = 1 +# +# OpenAI TTS Settings +# available voices are 'alloy`, `echo`, `fable`, `onyx`, `nova`, and `shimmer' +default_openai_tts_voice = shimmer +default_openai_tts_speed = 1 +# available models are 'tts-1' or 'tts-1-hd' +default_openai_tts_model = tts-1-hd +default_openai_tts_output_format = mp3 +# +# ElevenLabs TTS Settings +default_eleven_tts_voice = pNInz6obpgDQGcFmaJgB +default_eleven_tts_model = FIXME +default_eleven_tts_language_code = FIXME +default_eleven_tts_voice_stability = FIXME +default_eleven_tts_voice_similiarity_boost = FIXME +default_eleven_tts_voice_style = FIXME +default_eleven_tts_voice_use_speaker_boost = FIXME +default_eleven_tts_voice_pronunciation_dictionary_locators_dict_id = FIXME +default_eleven_tts_voice_pronunciation_dictionary_locators_version_id = FIXME +default_eleven_tts_speed = 1 +default_eleven_tts_output_format = FIXME +# Google TTS Settings +default_google_tts_model = FIXME +default_google_tts_voice = FIXME +default_google_tts_speed = 1 +# +# MS Edge TTS Settings +edge_tts_voice = FIXME +# + +[Search-Engines] +# Search Defaults +search_provider_default = google +search_language_query = en +search_language_results = en +search_language_analysis = en +search_default_max_queries = 10 +search_enable_subquery = True +search_enable_subquery_count_max = 5 +search_result_rerank = True +search_result_max = 15 +search_result_max_per_query = 10 +search_result_blacklist = [] +search_result_display_type = list +search_result_display_metadata = False +search_result_save_to_db = True +# How you want the results to be written, think 'style' or voice +search_result_analysis_tone = +#### Search Engines ##### +# Baidu +search_engine_api_key_baidu = > +# +# Bing +search_engine_api_url_bing = https://api.bing.microsoft.com/v7.0/search +search_engine_api_key_bing = +search_engine_country_code_bing = en +# +# Brave +search_engine_api_key_brave_regular = +search_engine_api_key_brave_ai = +search_engine_country_code_brave = US +# +# DuckDuckGo +# +# Google +search_engine_api_url_google = https://www.googleapis.com/customsearch/v1? +search_engine_api_key_google = +search_engine_id_google = +# 0 = Enable / 1 = Disabled +enable_traditional_chinese = 0 +# Restricts search results to documents originating in a particular country. +limit_google_search_to_country = False +google_search_country_code = US +google_filter_setting = 1 +google_user_geolocation = US +google_ui_language = en +google_limit_search_results_to_language = +google_default_search_results = +google_safe_search = "active" +google_enable_site_search = +google_site_search_include = +google_site_search_exclude = +# https://developers.google.com/custom-search/docs/structured_search#sort-by-attribute +google_sort_results_by = +# +# Kagi +search_engine_api_key_kagi = > +# SearX +search_engine_searx_api = https://search.rhscz.eu/ +# Serper +# Tavily +search_engine_api_key_tavily = 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 +# Yandex +search_engine_api_key_yandex = > +search_engine_id_yandex = > [Prompts] prompt_sample = "What is the meaning of life?" video_summarize_prompt = "Above is the transcript of a video. Please read through the transcript carefully. Identify the main topics that are discussed over the course of the transcript. Then, summarize the key points about each main topic in bullet points. The bullet points should cover the key information conveyed about each topic in the video, but should be much shorter than the full transcript. Please output your bullet point summary inside tags. Do not repeat yourself while writing the summary." + [Database] type = sqlite sqlite_path = Databases/media_summary.db @@ -110,7 +276,6 @@ max_bytes = backup_count = 5 #int(os.getenv("tldw_LOG_BACKUP_COUNT", 5)) - #[Comments] #OpenAI Models: # gpt-4o diff --git a/Docs/Design/Diagram_Generation.md b/Docs/Design/Diagram_Generation.md new file mode 100644 index 000000000..41e27b563 --- /dev/null +++ b/Docs/Design/Diagram_Generation.md @@ -0,0 +1,9 @@ +# Diagram Generation + + +### Link Dump: +https://excalidraw.com/ +https://www.napkin.ai/ + + + diff --git a/Docs/Design/Education.md b/Docs/Design/Education.md index 778425aec..66ae8339a 100644 --- a/Docs/Design/Education.md +++ b/Docs/Design/Education.md @@ -4,7 +4,7 @@ ### Link Dump: https://arxiv.org/abs/2412.02035 https://github.com/andreamust/NEON-GPT -https://excalidraw.com/ + https://arxiv.org/abs/2411.07407 https://arxiv.org/abs/2412.16429 https://huggingface.co/papers/2412.15443 @@ -15,6 +15,7 @@ https://bbycroft.net/llm https://github.com/met4citizen/TalkingHead + one2manny — Today at 12:43 AM diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index 330bae150..c450d9ddd 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -103,11 +103,20 @@ Standford STORM https://arxiv.org/abs/2402.14207# https://storm.genie.stanford.edu/ +Google Learn About + https://learning.google.com/experiments/learn-about + +Google Pinpoint + https://journaliststudio.google.com/pinpoint/about/ + +Gemini Deepresearcher + https://blog.google/products/gemini/google-gemini-deep-research/ + https://github.com/assafelovic/gpt-researcher https://arxiv.org/abs/2411.15114 -https://journaliststudio.google.com/pinpoint/about/ + https://github.com/assafelovic/gpt-researcher/tree/master/gpt_researcher -https://blog.google/products/gemini/google-gemini-deep-research/ + https://github.com/neuml/annotateai https://docs.gptr.dev/docs/gpt-researcher/multi_agents/langgraph https://pub.towardsai.net/learn-anything-with-ai-and-the-feynman-technique-00a33f6a02bc @@ -133,13 +142,15 @@ https://www.researchrabbit.ai/ https://github.com/faraz18001/Sales-Llama https://github.com/memgraph/memgraph https://github.com/rashadphz/farfalle/tree/main/src/backend -https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py -https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama + https://github.com/rashadphz/farfalle/blob/main/src/backend/agent_search.py https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py https://github.com/stanford-oval/storm/blob/main/examples/storm_examples/run_storm_wiki_claude.py https://learning.google.com/experiments/learn-about +AI Web Researcher Ollama + https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama/blob/main/Self_Improving_Search.py + https://github.com/TheBlewish/Automated-AI-Web-Researcher-Ollama diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index ca68684b4..6d66d924a 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -228,10 +228,18 @@ https://docs.cartesia.ai/get-started/make-an-api-request https://github.com/JarodMica/open-neruosama https://github.com/flatmax/speech-to-text https://arxiv.org/abs/2412.18566 +https://github.com/Rolandjg/skool4free + + +SoundStorm + https://deepmind.google/discover/blog/pushing-the-frontiers-of-audio-generation/ + https://github.com/lucidrains/soundstorm-pytorch + Google https://github.com/google-gemini/cookbook/tree/main/gemini-2 https://discuss.ai.google.dev/t/how-does-one-get-access-to-the-api-for-tts-features-of-gemini-2-0/53925/15 +https://illuminate.google.com/home?pli=1 ``` import asyncio import base64 diff --git a/Docs/RAG_Notes.md b/Docs/RAG_Notes.md index f1a728c09..fadf7a979 100644 --- a/Docs/RAG_Notes.md +++ b/Docs/RAG_Notes.md @@ -83,6 +83,7 @@ GraphRAG https://neuml.hashnode.dev/advanced-rag-with-graph-path-traversal https://volodymyrpavlyshyn.medium.com/unified-knowledge-graph-model-rdf-rdf-vs-lpg-the-end-of-war-a7c14d6ac76f https://github.com/zjunlp/OneKE +https://blog.gopenai.com/llm-ontology-prompting-for-knowledge-graph-extraction-efdcdd0db3a1?gi=1d8915f0da5e From 070bfbe1f3f46faf8b8febb789c8fa682a1d86c2 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 14:42:46 -0800 Subject: [PATCH 66/88] chat regen works --- App_Function_Libraries/Gradio_UI/Chat_ui.py | 55 ++++++++++++--------- App_Function_Libraries/LLM_API_Calls.py | 5 +- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Chat_ui.py b/App_Function_Libraries/Gradio_UI/Chat_ui.py index 521a15ccc..fe4c757c8 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_ui.py +++ b/App_Function_Libraries/Gradio_UI/Chat_ui.py @@ -42,8 +42,8 @@ def show_delete_message(selected): def debug_output(media_content, selected_parts): - print(f"Debug - Media Content: {media_content}") - print(f"Debug - Selected Parts: {selected_parts}") + logging.debug(f"Debug - Media Content: {media_content}") + logging.debug(f"Debug - Selected Parts: {selected_parts}") return "" @@ -55,7 +55,7 @@ def update_selected_parts(use_content, use_summary, use_prompt): selected_parts.append("summary") if use_prompt: selected_parts.append("prompt") - print(f"Debug - Update Selected Parts: {selected_parts}") + logging.debug(f"Debug - Update Selected Parts: {selected_parts}") return selected_parts @@ -177,33 +177,42 @@ def regenerate_last_message(history, media_content, selected_parts, api_endpoint if not history: return history, "No messages to regenerate." - last_entry = history[-1] - last_user_message, last_bot_message = last_entry + logging.debug("Starting regenerate_last_message") - if last_bot_message is None: - return history, "The last message is not from the bot." - - new_history = history[:-1] + # Find the last user message and its corresponding bot response + last_user_message = None + last_bot_message = None + for i in range(len(history) - 1, -1, -1): + if history[i][0]: # This is a user message + last_user_message = history[i][0] + if i + 1 < len(history): + last_bot_message = history[i + 1][1] + break if not last_user_message: - return new_history, "No user message to regenerate the bot response." + return history, "No user message found to regenerate the bot response." + + # Remove the last bot message from history + new_history = history[:-1] if last_bot_message else history # Generate the new bot response bot_message = "" - if streaming: - # For streaming, consume the generator and accumulate the response - for chunk in chat(last_user_message, new_history, media_content, selected_parts, api_endpoint, api_key, - custom_prompt, temperature, system_prompt, streaming): + for chunk in chat(last_user_message, new_history, media_content, selected_parts, api_endpoint, api_key, + custom_prompt, temperature, system_prompt, streaming): + if isinstance(chunk, str): bot_message += chunk - else: - # For non-streaming, get the complete response directly - bot_message = next(chat(last_user_message, new_history, media_content, selected_parts, api_endpoint, api_key, - custom_prompt, temperature, system_prompt, streaming)) + elif isinstance(chunk, dict) and "choices" in chunk: + content = chunk["choices"][0].get("delta", {}).get("content", "") + bot_message += content - # Update the history with the regenerated message - new_history.append((last_user_message, bot_message)) + # Update the chatbot interface with the partial response + new_history_with_regenerated = new_history + [(last_user_message, bot_message)] + yield new_history_with_regenerated, "Regenerating..." - return new_history, "Last message regenerated successfully." + # Update the history with the final regenerated message + new_history_with_regenerated = new_history + [(last_user_message, bot_message)] + logging.debug("Finished regenerating message") + yield new_history_with_regenerated, "Last message regenerated successfully." def update_dropdown_multiple(query, search_type, keywords=""): @@ -340,7 +349,7 @@ def create_chat_interface(): with gr.Column(scale=2): chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container") msg = gr.Textbox(label="Enter your message") - streaming = gr.Checkbox(label="Streaming") + streaming = gr.Checkbox(label="Streaming", value=False, visible=True) submit = gr.Button("Submit") regenerate_button = gr.Button("Regenerate Last Message") token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) @@ -521,7 +530,7 @@ def on_next_page_click(current_page, total_pages): regenerate_last_message, inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature, system_prompt_input, streaming], - outputs=[chatbot, save_status] + outputs=[chatbot, gr.Textbox(label="Regenerate Status")] ).then( lambda history: approximate_token_count(history), inputs=[chatbot], diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 330b3c0d1..6dfa423d7 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -132,8 +132,9 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m logging.debug(f"OpenAI: Using API Key: {openai_api_key[:5]}...{openai_api_key[-5:]}") - streaming = loaded_config_data['openai_api']['streaming'] - if streaming == "true" or "True": + if not streaming: + streaming = loaded_config_data['openai_api']['streaming'] + if streaming == "true" or True: streaming = True else: streaming = False From 635a5cdd2b7dad71da845d11ffed982305e7d6e9 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 17:48:25 -0800 Subject: [PATCH 67/88] TTS support for Alltalk, ElevenLabs, and OpenAI. --- App_Function_Libraries/Gradio_UI/Chat_ui.py | 2 + App_Function_Libraries/TTS/TTS_Providers.py | 376 +++++++++++++++----- App_Function_Libraries/Utils/Utils.py | 13 + 3 files changed, 296 insertions(+), 95 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Chat_ui.py b/App_Function_Libraries/Gradio_UI/Chat_ui.py index fe4c757c8..9c99b3ce0 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_ui.py +++ b/App_Function_Libraries/Gradio_UI/Chat_ui.py @@ -350,6 +350,8 @@ def create_chat_interface(): chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container") msg = gr.Textbox(label="Enter your message") streaming = gr.Checkbox(label="Streaming", value=False, visible=True) + # Fixme - this should trigger teh chatbot to speak the response message when its checked + speak_message = gr.Checkbox(label="Speak Message", value=False, visible=True) submit = gr.Button("Submit") regenerate_button = gr.Button("Regenerate Last Message") token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 0225a3ae8..91319e10e 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -2,15 +2,19 @@ # Description: This file contains the functions to allow for usage of different TTS providers. # # Imports +import json import logging import os import tempfile +import time import uuid - # # External Imports import edge_tts import requests +from openai import api_key +from pydub import AudioSegment +from pydub.playback import play # # Local Imports from App_Function_Libraries.Utils.Utils import load_and_log_configs, loaded_config_data @@ -19,6 +23,69 @@ # # Functions: +def play_mp3(file_path): + """Play an MP3 file using the pydub library.""" + try: + from pydub.utils import which + print(f"Debug: ffmpeg path: {which('ffmpeg')}") + print(f"Debug: ffplay path: {which('ffplay')}") + + absolute_path = os.path.abspath(file_path) + audio = AudioSegment.from_mp3(absolute_path) + print("Debug: File loaded successfully") + play(audio) + except Exception as e: + print(f"Debug: Exception type: {type(e)}") + print(f"Debug: Exception args: {e.args}") + print(f"Error playing the audio file: {e}") + + +def generate_audio(api_key, text, provider, voice=None, model=None, voice2=None, output_file=None, response_format=None, streaming=False): + """Generate audio using the specified TTS provider.""" + if provider == "openai": + if api_key == None: + api_key = loaded_config_data['openai_api']['api_key'] + return generate_audio_openai(api_key, text, voice, model, response_format, output_file, streaming) + elif provider == "elevenlabs": + if api_key == None: + api_key = loaded_config_data['elevenlabs_api']['api_key'] + return generate_audio_elevenlabs(text, voice, model, api_key) + # FIXME - add gemini + # elif provider == "gemini": + # return generate_audio_gemini(text, voice, model) + elif provider == "alltalk": + return generate_audio_alltalk(text, voice, model) + # FIXME - add gpt-sovit + # elif provider == "gpt-sovit": + # return generate_audio_gpt_sovit(text, voice, model) + else: + raise ValueError(f"Invalid TTS provider: {provider}") + + +def test_generate_audio(): + """Test the generate_audio function with a real API request.""" + api_key = None + text = "The quick brown fox jumped over the lazy dog." + provider = ["openai", "elevenlabs", "alltalk"] + voice = "alloy" + model = None + response_format = "mp3" + output_file = "speech.mp3" + + # Call the function + for provider in provider: + result = generate_audio(api_key, text, provider, voice, model, output_file, response_format) + + # Assertions + assert os.path.exists(result), f"The file {result} should exist." + assert result.endswith(".mp3"), f"The file {result} should be an MP3 file." + + print(f"Attempting to play file: {result} from provider: {provider}") + if os.path.exists(result): + play_mp3(result) # Single play call + print("Test successful") + + ####################################################### # # OpenAI TTS Provider Functions @@ -163,6 +230,7 @@ def test_generate_audio_openai(): try: output_file = generate_audio_openai(api_key, input_text, voice, model) print(f"Generated audio file: {output_file}") + play_mp3(output_file) except Exception as e: print(f"Error: {e}") @@ -182,51 +250,6 @@ def test_generate_audio_openai(): ####################################################### -####################################################### -# -# MS Edge TTS Provider Functions - NOPE -# -# # https://github.com/rany2/edge-tts -# -# def generate_audio_edge(text, voice, model=None, voice2=None): -# # FIXME - SSML tags -# common_ssml_tags = ['lang', 'p', 'phoneme', 's', 'sub'] -# def generate_audio(self, text: str, voice: str, model: str, voice2: str = None) -> bytes: -# """Generate audio using Edge TTS.""" -# import nest_asyncio -# import asyncio -# -# # Apply nest_asyncio to allow nested event loops -# nest_asyncio.apply() -# -# async def _generate(): -# communicate = edge_tts.Communicate(text, voice) -# # Create a temporary file with proper context management -# with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as tmp_file: -# temp_path = tmp_file.name -# -# try: -# # Save audio to temporary file -# await communicate.save(temp_path) -# # Read the audio data -# with open(temp_path, 'rb') as f: -# return f.read() -# finally: -# # Clean up temporary file -# if os.path.exists(temp_path): -# os.remove(temp_path) -# -# # Use nest_asyncio to handle nested event loops -# loop = asyncio.get_event_loop() -# return loop.run_until_complete(_generate()) -# result = generate_audio(text, voice, model, voice2) -# return result - -# -# End of MS Edge TTS Provider Functions -####################################################### - - ####################################################### # # ElvenLabs TTS Provider Functions @@ -243,7 +266,7 @@ def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): if not elevenlabs_api_key: logging.info("ElevenLabs: API key not provided as parameter") logging.info("ElevenLabs: Attempting to use API key from config file") - elevenlabs_api_key = loaded_config_data['api_keys']['elevenlabs'] + elevenlabs_api_key = loaded_config_data['elevenlabs_api']['api_key'] if not elevenlabs_api_key: logging.error("ElevenLabs: API key not found or is empty") @@ -278,8 +301,6 @@ def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): return f"ElevenLabs: Error loading Speaker ID(Voice): {str(e)}" # Handle Model ID/Selection - # Set Voice Model - model="eleven_turbo_v2_5", # use the turbo model for low latency try: if not model: logging.info("ElevenLabs: Model not provided as parameter") @@ -293,48 +314,32 @@ def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): return f"ElevenLabs: Error Selecting Model: {str(e)}" # FIXME - add SSML tags - - # File output (non-streaming) - output_format="mp3_22050_32", - # Set the parameters for the TTS conversion try: - default_eleven_tts_voice_stability = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_stability', 0.0) - except Exception as e: - logging.error(f"ElevenLabs: Error loading Stability: {str(e)}") - return f"ElevenLabs: Error loading Stability: {str(e)}" + # Stability + stability_str = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_stability', '0.0') + default_eleven_tts_voice_stability = float(stability_str) if stability_str else 0.0 - try: # Similarity Boost - default_eleven_tts_voice_similiarity_boost = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_similiarity_boost', 1.0) - except Exception as e: - logging.error(f"ElevenLabs: Error loading Similarity Boost: {str(e)}") - return f"ElevenLabs: Error loading Similarity Boost: {str(e)}" + similarity_boost_str = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_similiarity_boost', '1.0') + default_eleven_tts_voice_similiarity_boost = float(similarity_boost_str) if similarity_boost_str else 1.0 - try: # Style - default_eleven_tts_voice_style = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_style', 0.0) - except Exception as e: - logging.error(f"ElevenLabs: Error loading Style: {str(e)}") - return f"ElevenLabs: Error loading Style: {str(e)}" + style_str = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_style', '0.0') + default_eleven_tts_voice_style = float(style_str) if style_str else 0.0 - try: # Use Speaker Boost - default_eleven_tts_voice_use_speaker_boost = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_use_speaker_boost', True) - except Exception as e: - logging.error(f"ElevenLabs: Error loading Use Speaker Boost: {str(e)}") - return f"ElevenLabs: Error loading Use Speaker Boost: {str(e)}" + use_speaker_boost_str = loaded_config_data['tts_settings'].get('default_eleven_tts_voice_use_speaker_boost', 'True') + default_eleven_tts_voice_use_speaker_boost = use_speaker_boost_str.lower() == 'true' if use_speaker_boost_str else True - try: # Output Format - default_eleven_tts_output_format = loaded_config_data['tts_settings'].get('default_eleven_tts_output_format', "mp3_44100_192") + default_eleven_tts_output_format = loaded_config_data['tts_settings'].get('default_eleven_tts_output_format', 'mp3_44100_192') except Exception as e: - logging.error(f"ElevenLabs: Error loading Output Format: {str(e)}") - return f"ElevenLabs: Error loading Output Format: {str(e)}" + logging.error(f"ElevenLabs: Error loading voice settings: {str(e)}") + return f"ElevenLabs: Error loading voice settings: {str(e)}" # Make the API request - # Construct the URL for the Text-to-Speech API request - tts_url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream?output_format=mp3_44100_192" + tts_url = f"https://api.elevenlabs.io/v1/text-to-speech/{voice}/stream?output_format={default_eleven_tts_output_format}" # Set up headers for the API request, including the API key for authentication headers = { @@ -354,37 +359,72 @@ def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): "use_speaker_boost": default_eleven_tts_voice_use_speaker_boost } } + try: # Make the POST request to the TTS API with headers and data, enabling streaming response with requests.post(tts_url, headers=headers, json=data, stream=True) as response: # Check if the request was successful if response.ok: - # Create a temporary file - FIXME - with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file: - # Read the response in chunks and write to the file - for chunk in response.iter_content(chunk_size=CHUNK_SIZE): - tmp_file.write(chunk) - temp_file_path = tmp_file.name - # Inform the user of success + # Create temp file but don't use context manager + tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + tmp_file.write(chunk) + tmp_file.flush() + tmp_file.close() # Explicitly close the file handle + temp_file_path = tmp_file.name print(f"Audio stream saved successfully to {temp_file_path}.") - if not response.ok: + return temp_file_path + else: logging.error(f"API request failed: {response.status_code} - {response.text}") return f"API request failed: {response.status_code} - {response.text}" except requests.exceptions.RequestException as e: raise RuntimeError(f"Failed to generate audio: {str(e)}") from e - save_file_path = f"{uuid.uuid4()}.mp3" - # Writing the audio to a file - with open(save_file_path, "wb") as f: - for chunk in response: - if chunk: - f.write(chunk) +def test_generate_audio_elevenlabs_real_request(): + """Test the function with a real API request.""" + api_key = None + input_text = "This is a test text for generating audio." + voice = None + model = "eleven_turbo_v2" + + # Call the function + result = generate_audio_elevenlabs(input_text=input_text, voice=voice, model=model, api_key=api_key) + + # Assertions + assert os.path.exists(result), f"The file {result} should exist." + assert result.endswith(".mp3"), f"The file {result} should be an MP3 file." + + print(f"Attempting to play file: {result}") + if os.path.exists(result): + play_mp3(result) # Single play call + + +def test_generate_audio_elevenlabs_invalid_api_key(): + """Test the function with an invalid API key.""" + # Use an invalid API key + api_key = "invalid_api_key" + input_text = "This is a test text for generating audio." + voice = "your_voice_id" # Replace with a valid voice ID from ElevenLabs - print(f"{save_file_path}: A new audio file was saved successfully!") + # Call the function + result = generate_audio_elevenlabs(input_text=input_text, voice=voice, api_key=api_key) - # Return the path of the saved audio file - return save_file_path + # Assertions + assert "API request failed" in result, "The function should return an error message for an invalid API key." + +def test_generate_audio_elevenlabs_missing_input_text(): + """Test the function with missing input text.""" + # Use a valid API key but no input text + api_key = "your_actual_api_key" + input_text = "" + voice = "your_voice_id" # Replace with a valid voice ID from ElevenLabs + + # Call the function + result = generate_audio_elevenlabs(input_text=input_text, voice=voice, api_key=api_key) + + # Assertions + assert "Error loading input text" in result, "The function should return an error message for missing input text." # End of ElvenLabs TTS Provider Functions ####################################################### @@ -402,6 +442,152 @@ def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): ####################################################### +############################################################ LOCAL ##################################################### + + +####################################################### +# +# AllTalk TTS Provider Functions +# https://github.com/erew123/alltalk_tts +# https://github.com/erew123/alltalk_tts/wiki/API-%E2%80%90-OpenAI-V1-Speech-Compatible-Endpoint + +def generate_audio_alltalk(input_text, voice=None, model=None, response_format=None, speed=None): + """Generate audio using AllTalk API. + + Args: + input_text (str): Text to convert to speech (max 4096 chars) + voice (str, optional): Voice ID ('alloy', 'echo', 'fable', 'nova', 'onyx', 'shimmer') + model (str, optional): Model ID (placeholder) + response_format (str, optional): Audio format (defaults to 'wav') + speed (float, optional): Speech speed (0.25 to 4.0, defaults to 1.0) + + Returns: + str: Path to the generated audio file + """ + + # Input validation + try: + if not input_text: + raise ValueError("Text input is required.") + logging.debug(f"AllTalk: Raw input data type: {type(input_text)}") + logging.debug(f"AllTalk: Raw input data (first 500 chars): {str(input_text)[:500]}...") + except Exception as e: + logging.error(f"AllTalk: Error loading input text: {str(e)}") + return f"AllTalk: Error loading input text: {str(e)}" + try: + if input_text > 4096: + raise ValueError("Text input must be less than 4096 characters.") + except Exception as e: + logging.error(f"AllTalk: Error loading input text(more than 4096 characters): {str(e)}") + return f"AllTalk: Error loading input text(more than 4096 characters): {str(e)}" + + # Handle Voice + try: + if not voice: + logging.info("AllTalk: Voice not provided as parameter") + logging.info("AllTalk: Attempting to use voice from config file") + voice = loaded_config_data['alltalk_api']['voice'] + + if not voice: + raise ValueError("Voice is required. Default voice not found in config file and no voice selection was passed.") + except Exception as e: + logging.error(f"AllTalk: Error loading voice: {str(e)}") + return f"AllTalk: Error loading voice: {str(e)}" + + # Handle Response Format + try: + if not response_format: + logging.info("AllTalk: Format not provided as parameter") + logging.info("AllTalk: Attempting to use format from config file") + response_format = loaded_config_data['alltalk_api']['default_alltalk_tts_output_format'] + + if not response_format: + logging.debug("AllTalk: No response format provided. Defaulting to 'wav'") + response_format = "wav" + except Exception as e: + logging.error(f"AllTalk: Error setting format: {str(e)}") + return f"AllTalk: Error setting format: {str(e)}" + + # Handle Speed + try: + if not speed: + logging.info("AllTalk: Speed not provided as parameter") + logging.info("AllTalk: Attempting to use speed from config file") + speed = loaded_config_data['alltalk_api']['default_alltalk_tts_speed'] + + if not speed: + logging.debug("AllTalk: No speed provided. Defaulting to '1.0'") + speed = 1.0 + + speed = float(speed) + if not 0.25 <= speed <= 4.0: + raise ValueError("Speed must be between 0.25 and 4.0") + except Exception as e: + logging.error(f"AllTalk: Error setting speed: {str(e)}") + return f"AllTalk: Error setting speed: {str(e)}" + + # API URL + try: + alltalk_api_url = loaded_config_data['alltalk_api']['api_ip'] + if not alltalk_api_url: + raise ValueError("API URL not found in config") + except Exception as e: + logging.error(f"AllTalk: Error loading API URL: {str(e)}") + return f"AllTalk: Error loading API URL: {str(e)}" + + # Prepare request + payload = { + "model": model, + "input": input_text, + "voice": voice, + "response_format": response_format, + "speed": speed + } + + headers = { + "Content-Type": "application/json" + } + + try: + # Make the API request without streaming + response = requests.post(alltalk_api_url, json=payload, headers=headers) + + if response.ok: + # Create a temporary file + with tempfile.NamedTemporaryFile(delete=False, + suffix=f".{response_format}") as tmp_file: + # Write the entire response content at once + tmp_file.write(response.content) + tmp_file.flush() + temp_file_path = tmp_file.name + + print(f"Audio stream saved successfully to {temp_file_path}.") + return temp_file_path + else: + error_msg = f"API request failed: {response.status_code} - {response.text}" + logging.error(error_msg) + return error_msg + + except requests.exceptions.RequestException as e: + error_msg = f"Failed to generate audio: {str(e)}" + logging.error(error_msg) + return error_msg + + +def test_generate_audio_alltalk(): + model = "placeholder" + input_text = "The quick brown fox jumped over the yellow lazy dog." + voice = "alloy" + response_format = "wav" + speed = 1.0 + + generate_audio_alltalk(model, input_text, voice, response_format, speed) + +# +# End of AllTalk TTS Provider Functions +####################################################### + + ####################################################### # # gpt-soviTTS TTS Provider Functions diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index dda80161b..2648bf3a6 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -397,6 +397,12 @@ def load_and_log_configs(): default_eleven_tts_voice_use_speaker_boost = config.get('TTS-Settings', 'default_eleven_tts_voice_use_speaker_boost', fallback='FIXME') default_eleven_tts_output_format = config.get('TTS-Settings', 'default_eleven_tts_output_format', fallback='mp3_44100_192') + # AllTalk TTS + alltalk_api_ip = config.get('TTS-Settings', 'alltalk_api_ip', fallback='http://127.0.0.1:7851/v1/audio/speech') + default_alltalk_tts_model = config.get('TTS-Settings', 'default_alltalk_tts_model', fallback='alltalk_model') + default_alltalk_tts_voice = config.get('TTS-Settings', 'default_alltalk_tts_voice', fallback='alloy') + default_alltalk_tts_speed = config.get('TTS-Settings', 'default_alltalk_tts_speed', fallback=1.0) + default_alltalk_tts_output_format = config.get('TTS-Settings', 'default_alltalk_tts_output_format', fallback='mp3') # Search Engines search_provider_default = config.get('Search-Engines', 'search_provider_default', fallback='google') @@ -528,6 +534,13 @@ def load_and_log_configs(): 'elevenlabs_api': { 'api_key': elevenlabs_api_key }, + 'alltalk_api': { + 'api_ip': alltalk_api_ip, + 'default_alltalk_tts_model': default_alltalk_tts_model, + 'default_alltalk_tts_voice': default_alltalk_tts_voice, + 'default_alltalk_tts_speed': default_alltalk_tts_speed, + 'default_alltalk_tts_output_format': default_alltalk_tts_output_format, + }, 'custom_openai_api': { 'api_key': custom_openai_api_key, 'api_url': custom_openai_api_url, From 67f8ae39098003ddcd954d26a6ba1730b78fc54a Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 18:53:53 -0800 Subject: [PATCH 68/88] Sweet. --- App_Function_Libraries/Gradio_UI/Chat_ui.py | 181 ++++++++++++++++++-- App_Function_Libraries/TTS/TTS_Providers.py | 102 +++++++++-- App_Function_Libraries/Utils/Utils.py | 4 +- 3 files changed, 255 insertions(+), 32 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Chat_ui.py b/App_Function_Libraries/Gradio_UI/Chat_ui.py index 9c99b3ce0..f728ab344 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_ui.py +++ b/App_Function_Libraries/Gradio_UI/Chat_ui.py @@ -20,7 +20,11 @@ from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_db_connection from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_user_prompt from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram -from App_Function_Libraries.Utils.Utils import default_api_endpoint, format_api_name, global_api_endpoints +from App_Function_Libraries.TTS.TTS_Providers import generate_audio, play_mp3 +from App_Function_Libraries.Utils.Utils import default_api_endpoint, format_api_name, global_api_endpoints, \ + loaded_config_data + + # # ######################################################################################################################## @@ -268,6 +272,24 @@ def create_chat_interface(): .chatbot-container .message-wrap .message { font-size: 14px !important; } + #tts-status { + font-weight: bold; + padding: 5px; + border-radius: 4px; + margin-top: 5px; + } + #tts-status[value*="Error"], #tts-status[value*="Failed"] { + color: red; + background-color: #ffe6e6; + } + #tts-status[value*="Generating"], #tts-status[value*="Playing"] { + color: #0066cc; + background-color: #e6f2ff; + } + #tts-status[value*="Finished"] { + color: green; + background-color: #e6ffe6; + } """ with gr.TabItem("Remote LLM Chat (Horizontal)", visible=True): gr.Markdown("# Chat with a designated LLM Endpoint, using your selected item as starting context") @@ -350,18 +372,22 @@ def create_chat_interface(): chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container") msg = gr.Textbox(label="Enter your message") streaming = gr.Checkbox(label="Streaming", value=False, visible=True) - # Fixme - this should trigger teh chatbot to speak the response message when its checked - speak_message = gr.Checkbox(label="Speak Message", value=False, visible=True) submit = gr.Button("Submit") + with gr.Row(): + speak_button = gr.Button("Speak Response") + tts_status = gr.Textbox(label="TTS Status", interactive=False) regenerate_button = gr.Button("Regenerate Last Message") - token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) - clear_chat_button = gr.Button("Clear Chat") + with gr.Row(): + token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) + clear_chat_button = gr.Button("Clear Chat") chat_media_name = gr.Textbox(label="Custom Chat Name(optional)") - save_chat_history_to_db = gr.Button("Save Chat History to DataBase") - save_status = gr.Textbox(label="Save Status", interactive=False) - save_chat_history_as_file = gr.Button("Save Chat History as File") - download_file = gr.File(label="Download Chat History") + with gr.Row(): + save_chat_history_to_db = gr.Button("Save Chat History to DataBase") + save_status = gr.Textbox(label="Save Status", interactive=False) + with gr.Row(): + save_chat_history_as_file = gr.Button("Save Chat History as File") + download_file = gr.File(label="Download Chat History") # Restore original functionality search_button.click( @@ -427,6 +453,64 @@ def on_preset_prompt_checkbox_change(is_checked): outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state] ) + # TTS Generation and Playback + def speak_last_response(chatbot): + """Handle speaking the last chat response.""" + logging.debug("Starting speak_last_response") + try: + # If there's no chat history, return + if not chatbot or len(chatbot) == 0: + logging.debug("No messages in chatbot history") + return gr.update(value="No messages to speak", visible=True) + + # Log the chatbot content for debugging + logging.debug(f"Chatbot history: {chatbot}") + + # Get the last message from the assistant + last_message = chatbot[-1][1] + logging.debug(f"Last message to speak: {last_message}") + + # Update status to generating + yield gr.update(value="Generating audio...", visible=True) + + # Generate audio using your preferred TTS provider + try: + audio_file = generate_audio( + api_key=None, # Use default API key + text=last_message, + provider="openai", # or get from config + output_file="last_response.mp3" + ) + logging.debug(f"Generated audio file: {audio_file}") + except Exception as e: + logging.error(f"Failed to generate audio: {e}") + yield gr.update(value=f"Failed to generate audio: {str(e)}", visible=True) + return + + # Update status to playing + yield gr.update(value="Playing audio...", visible=True) + + # Play the audio + if audio_file and os.path.exists(audio_file): + try: + play_mp3(audio_file) + yield gr.update(value="Finished playing audio", visible=True) + except Exception as e: + logging.error(f"Failed to play audio: {e}") + yield gr.update(value=f"Failed to play audio: {str(e)}", visible=True) + else: + logging.error("Audio file not found") + yield gr.update(value="Failed: Audio file not found", visible=True) + + except Exception as e: + logging.error(f"Error in speak_last_response: {str(e)}") + yield gr.update(value=f"Error: {str(e)}", visible=True) + speak_button.click( + fn=speak_last_response, + inputs=[chatbot], + outputs=[tts_status], + api_name="speak_response" + ) def on_prev_page_click(current_page, total_pages): new_page = max(current_page - 1, 1) prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20) @@ -654,15 +738,21 @@ def create_chat_interface_stacked(): with gr.Row(): with gr.Column(): submit = gr.Button("Submit") + with gr.Row(): + speak_button = gr.Button("Speak Response") + tts_status = gr.Textbox(label="TTS Status", interactive=False) regenerate_button = gr.Button("Regenerate Last Message") - token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) - clear_chat_button = gr.Button("Clear Chat") - chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True) - save_chat_history_to_db = gr.Button("Save Chat History to DataBase") - save_status = gr.Textbox(label="Save Status", interactive=False) - save_chat_history_as_file = gr.Button("Save Chat History as File") - with gr.Column(): - download_file = gr.File(label="Download Chat History") + with gr.Row(): + token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) + clear_chat_button = gr.Button("Clear Chat") + + chat_media_name = gr.Textbox(label="Custom Chat Name(optional)") + with gr.Row(): + save_chat_history_to_db = gr.Button("Save Chat History to DataBase") + save_status = gr.Textbox(label="Save Status", interactive=False) + with gr.Row(): + save_chat_history_as_file = gr.Button("Save Chat History as File") + download_file = gr.File(label="Download Chat History") # Restore original functionality search_button.click( @@ -840,7 +930,64 @@ def on_next_page_click(current_page, total_pages): inputs=[current_page_state, total_pages_state], outputs=[preset_prompt, page_display, current_page_state] ) + # TTS Generation and Playback + def speak_last_response(chatbot): + """Handle speaking the last chat response.""" + logging.debug("Starting speak_last_response") + try: + # If there's no chat history, return + if not chatbot or len(chatbot) == 0: + logging.debug("No messages in chatbot history") + return gr.update(value="No messages to speak", visible=True) + # Log the chatbot content for debugging + logging.debug(f"Chatbot history: {chatbot}") + + # Get the last message from the assistant + last_message = chatbot[-1][1] + logging.debug(f"Last message to speak: {last_message}") + + # Update status to generating + yield gr.update(value="Generating audio...", visible=True) + + # Generate audio using your preferred TTS provider + try: + audio_file = generate_audio( + api_key=None, # Use default API key + text=last_message, + provider="openai", # or get from config + output_file="last_response.mp3" + ) + logging.debug(f"Generated audio file: {audio_file}") + except Exception as e: + logging.error(f"Failed to generate audio: {e}") + yield gr.update(value=f"Failed to generate audio: {str(e)}", visible=True) + return + + # Update status to playing + yield gr.update(value="Playing audio...", visible=True) + + # Play the audio + if audio_file and os.path.exists(audio_file): + try: + play_mp3(audio_file) + yield gr.update(value="Finished playing audio", visible=True) + except Exception as e: + logging.error(f"Failed to play audio: {e}") + yield gr.update(value=f"Failed to play audio: {str(e)}", visible=True) + else: + logging.error("Audio file not found") + yield gr.update(value="Failed: Audio file not found", visible=True) + + except Exception as e: + logging.error(f"Error in speak_last_response: {str(e)}") + yield gr.update(value=f"Error: {str(e)}", visible=True) + speak_button.click( + fn=speak_last_response, + inputs=[chatbot], + outputs=[tts_status], + api_name="speak_response" + ) # Update prompts when a preset is selected preset_prompt.change( update_prompts, diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 91319e10e..c498ccb35 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -42,24 +42,98 @@ def play_mp3(file_path): def generate_audio(api_key, text, provider, voice=None, model=None, voice2=None, output_file=None, response_format=None, streaming=False): """Generate audio using the specified TTS provider.""" + logging.info(f"Starting generate_audio function") + + # Get default provider if none specified + if not provider: + provider = loaded_config_data['tts_settings'].get('default_tts_provider', 'openai') + logging.info(f"No provider specified, using default: {provider}") + + # Set default output file if none specified + if not output_file: + # FIXME - use a temp file + output_file = "speech.mp3" + logging.info(f"No output file specified, using default: {output_file}") + + logging.info(f"Generating audio using {provider} TTS provider") + if provider == "openai": - if api_key == None: + logging.info("Using OpenAI TTS provider") + if api_key is None: + logging.info("No API key provided, attempting to use config file") api_key = loaded_config_data['openai_api']['api_key'] - return generate_audio_openai(api_key, text, voice, model, response_format, output_file, streaming) + return generate_audio_openai( + api_key=api_key, + input_text=text, + voice=voice, + model=model, + response_format=response_format, + output_file=output_file, + streaming=streaming + ) + elif provider == "elevenlabs": - if api_key == None: + logging.info("Using ElevenLabs TTS provider") + if api_key is None: + logging.info("No API key provided, attempting to use config file") api_key = loaded_config_data['elevenlabs_api']['api_key'] - return generate_audio_elevenlabs(text, voice, model, api_key) - # FIXME - add gemini - # elif provider == "gemini": - # return generate_audio_gemini(text, voice, model) + return generate_audio_elevenlabs( + input_text=text, + voice=voice, + model=model, + api_key=api_key + ) + elif provider == "alltalk": - return generate_audio_alltalk(text, voice, model) - # FIXME - add gpt-sovit - # elif provider == "gpt-sovit": - # return generate_audio_gpt_sovit(text, voice, model) + logging.info("Using AllTalk TTS provider") + return generate_audio_alltalk( + input_text=text, + voice=voice, + model=model + ) + else: - raise ValueError(f"Invalid TTS provider: {provider}") + error_msg = f"Invalid TTS provider: {provider}" + logging.error(error_msg) + raise ValueError(error_msg) + + +def speak_last_response(chatbot): + """Handle speaking the last chat response.""" + logging.debug("Starting speak_last_response") + try: + # If there's no chat history, return + if not chatbot or len(chatbot) == 0: + logging.debug("No messages in chatbot history") + return "No messages to speak" + + # Log the chatbot content for debugging + logging.debug(f"Chatbot history: {chatbot}") + + # Get the last message from the assistant + last_message = chatbot[-1][1] + logging.debug(f"Last message to speak: {last_message}") + + # Generate audio using your preferred TTS provider + audio_file = generate_audio( + text=last_message, + provider="openai", # or get from config + output_file="last_response.mp3" # specify output file + ) + + logging.debug(f"Generated audio file: {audio_file}") + + # Play the audio + if audio_file and os.path.exists(audio_file): + play_mp3(audio_file) + return "Speaking response..." + else: + logging.error("Failed to generate audio file") + return "Failed to generate audio" + + except Exception as e: + logging.error(f"Error in speak_last_response: {str(e)}") + return f"Error speaking response: {str(e)}" def test_generate_audio(): @@ -110,6 +184,7 @@ def generate_audio_openai(api_key, input_text, voice, model, response_format="mp ValueError: If required inputs are missing or invalid. RuntimeError: If the API request fails. """ + logging.info("Generating audio using OpenAI API.") # Validate inputs # API key validation @@ -151,7 +226,7 @@ def generate_audio_openai(api_key, input_text, voice, model, response_format="mp if not model: logging.info("OpenAI: Model not provided as parameter") logging.info("OpenAI: Attempting to use Model from config file") - model = loaded_config_data['tts_settings']['default_openai+tts_model'] + model = loaded_config_data['tts_settings']['default_openai_tts_model'] if not model: raise ValueError("Model is required. Default model not found in config and no model selection was passed.") @@ -259,6 +334,7 @@ def test_generate_audio_openai(): #https://elevenlabs.io/docs/api-reference/text-to-speech def generate_audio_elevenlabs(input_text, voice, model=None, api_key=None): """Generate audio using ElevenLabs API.""" + logging.info("Generating audio using ElevenLabs API.") CHUNK_SIZE = 1024 # API key validation elevenlabs_api_key = api_key diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 2648bf3a6..65e764a08 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -375,7 +375,7 @@ def load_and_log_configs(): # TTS Settings # FIXME - tts_provider = config.get('TTS-Settings', 'default_tts_provider', fallback='openai') + default_tts_provider = config.get('TTS-Settings', 'default_tts_provider', fallback='openai') tts_voice = config.get('TTS-Settings', 'default_tts_voice', fallback='shimmer') # Open AI TTS default_openai_tts_model = config.get('TTS-Settings', 'default_openai_tts_model', fallback='tts-1-hd') @@ -625,7 +625,7 @@ def load_and_log_configs(): 'default_api': default_api, 'local_api_timeout': local_api_timeout, 'tts_settings': { - 'tts_provider': tts_provider, + 'default_tts_provider': default_tts_provider, 'tts_voice': tts_voice, # OpenAI 'default_openai_tts_voice': default_openai_tts_voice, From 80047720a1a8ee5f631690eb74ec2686e460c795 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 18:56:31 -0800 Subject: [PATCH 69/88] Update Character_Chat_tab.py --- App_Function_Libraries/Gradio_UI/Character_Chat_tab.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index 56ee24113..027f0e97d 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -317,11 +317,13 @@ def create_character_card_interaction_tab(): chat_history = gr.Chatbot(label="Conversation", height=800) user_input = gr.Textbox(label="Your message") send_message_button = gr.Button("Send Message") - answer_for_me_button = gr.Button("Answer for Me") - continue_talking_button = gr.Button("Continue Talking") + with gr.Row(): + answer_for_me_button = gr.Button("Answer for Me") + continue_talking_button = gr.Button("Continue Talking") regenerate_button = gr.Button("Regenerate Last Message") - token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) - clear_chat_button = gr.Button("Clear Chat") + with gr.Row(): + token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) + clear_chat_button = gr.Button("Clear Chat") save_snapshot_button = gr.Button("Save Chat Snapshot") update_chat_dropdown = gr.Dropdown(label="Select Chat to Update", choices=[], visible=False) load_selected_chat_button = gr.Button("Load Selected Chat", visible=False) From 6b6268c4ea5ab214f0995d154caf4d840b03f233 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 19:11:28 -0800 Subject: [PATCH 70/88] and we have TTS in all the singular chats! Fuck Yea. --- .../Gradio_UI/Character_Chat_tab.py | 62 ++++++++++++++++++ .../Gradio_UI/RAG_QA_Chat_tab.py | 63 +++++++++++++++++++ .../Summarization_General_Lib.py | 36 +++++------ 3 files changed, 143 insertions(+), 18 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index 027f0e97d..83c6e57e0 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -34,6 +34,7 @@ delete_character_card, update_character_card, search_character_chats, save_chat_history_to_character_db, ) +from App_Function_Libraries.TTS.TTS_Providers import generate_audio from App_Function_Libraries.Utils.Utils import sanitize_user_input, format_api_name, global_api_endpoints, \ default_api_endpoint, load_comprehensive_config @@ -317,6 +318,9 @@ def create_character_card_interaction_tab(): chat_history = gr.Chatbot(label="Conversation", height=800) user_input = gr.Textbox(label="Your message") send_message_button = gr.Button("Send Message") + with gr.Row(): + speak_button = gr.Button("Speak Response") + tts_status = gr.Textbox(label="TTS Status", interactive=False) with gr.Row(): answer_for_me_button = gr.Button("Answer for Me") continue_talking_button = gr.Button("Continue Talking") @@ -962,7 +966,65 @@ def answer_for_me( # Define States for conversation_id and media_content, which are required for saving chat history conversation_id = gr.State(str(uuid.uuid4())) media_content = gr.State({}) + # TTS Generation and Playback + def speak_last_response(chat_history): + """Handle speaking the last chat response.""" + logging.debug("Starting speak_last_response") + try: + # If there's no chat history, return + if not chat_history or len(chat_history) == 0: + logging.debug("No messages in chat history") + return gr.update(value="No messages to speak", visible=True) + + # Log the chat history content for debugging + logging.debug(f"Chat history: {chat_history}") + + # Get the last message from the assistant + last_message = chat_history[-1][1] # Second element of the last tuple + logging.debug(f"Last message to speak: {last_message}") + + # Update status to generating + yield gr.update(value="Generating audio...", visible=True) + + # Generate audio using your preferred TTS provider + try: + audio_file = generate_audio( + text=last_message, + provider="openai", # or get from config + output_file="last_response.mp3" + ) + logging.debug(f"Generated audio file: {audio_file}") + except Exception as e: + logging.error(f"Failed to generate audio: {e}") + yield gr.update(value=f"Failed to generate audio: {str(e)}", visible=True) + return + + # Update status to playing + yield gr.update(value="Playing audio...", visible=True) + + # Play the audio + if audio_file and os.path.exists(audio_file): + try: + play_mp3(audio_file) + yield gr.update(value="Finished playing audio", visible=True) + except Exception as e: + logging.error(f"Failed to play audio: {e}") + yield gr.update(value=f"Failed to play audio: {str(e)}", visible=True) + else: + logging.error("Audio file not found") + yield gr.update(value="Failed: Audio file not found", visible=True) + except Exception as e: + logging.error(f"Error in speak_last_response: {str(e)}") + yield gr.update(value=f"Error: {str(e)}", visible=True) + + # Update the click handler + speak_button.click( + fn=speak_last_response, + inputs=[chat_history], # Use chat_history instead of chatbot + outputs=[tts_status], + api_name="speak_response" + ) # Button Callbacks # Add the new button callbacks here diff --git a/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py b/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py index 0c88aaeb3..4ad2ac0e1 100644 --- a/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py @@ -28,6 +28,7 @@ from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_text_and_format_from_pdf from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer, enhanced_rag_pipeline from App_Function_Libraries.RAG.RAG_QA_Chat import search_database, rag_qa_chat +from App_Function_Libraries.TTS.TTS_Providers import play_mp3, generate_audio from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name, \ load_comprehensive_config @@ -199,6 +200,9 @@ def update_conversation_list(): with gr.Column(scale=2): chatbot = gr.Chatbot(height=700) msg = gr.Textbox(label="Enter your message") + with gr.Row(): + speak_button = gr.Button("Speak Response") + tts_status = gr.Textbox(label="TTS Status", interactive=False) submit = gr.Button("Submit") clear_chat = gr.Button("Clear Chat History") @@ -789,6 +793,65 @@ def clear_chat_history(): outputs=[chatbot, msg, rating_display, rating_input] ) + # TTS Generation and Playback + def speak_last_response(chatbot): + """Handle speaking the last chat response.""" + logging.debug("Starting speak_last_response") + try: + # If there's no chat history, return + if not chatbot or len(chatbot) == 0: + logging.debug("No messages in chatbot history") + return gr.update(value="No messages to speak", visible=True) + + # Log the chatbot content for debugging + logging.debug(f"Chatbot history: {chatbot}") + + # Get the last message from the assistant + last_message = chatbot[-1][1] + logging.debug(f"Last message to speak: {last_message}") + + # Update status to generating + yield gr.update(value="Generating audio...", visible=True) + + # Generate audio using your preferred TTS provider + try: + audio_file = generate_audio( + api_key=None, # Use default API key + text=last_message, + provider="openai", # or get from config + output_file="last_response.mp3" + ) + logging.debug(f"Generated audio file: {audio_file}") + except Exception as e: + logging.error(f"Failed to generate audio: {e}") + yield gr.update(value=f"Failed to generate audio: {str(e)}", visible=True) + return + + # Update status to playing + yield gr.update(value="Playing audio...", visible=True) + + # Play the audio + if audio_file and os.path.exists(audio_file): + try: + play_mp3(audio_file) + yield gr.update(value="Finished playing audio", visible=True) + except Exception as e: + logging.error(f"Failed to play audio: {e}") + yield gr.update(value=f"Failed to play audio: {str(e)}", visible=True) + else: + logging.error("Audio file not found") + yield gr.update(value="Failed: Audio file not found", visible=True) + + except Exception as e: + logging.error(f"Error in speak_last_response: {str(e)}") + yield gr.update(value=f"Error: {str(e)}", visible=True) + speak_button.click( + fn=speak_last_response, + inputs=[chatbot], + outputs=[tts_status], + api_name="speak_response" + ) + return ( context_source, existing_file, diff --git a/App_Function_Libraries/Summarization/Summarization_General_Lib.py b/App_Function_Libraries/Summarization/Summarization_General_Lib.py index f6dd46c9d..b42f98dbd 100644 --- a/App_Function_Libraries/Summarization/Summarization_General_Lib.py +++ b/App_Function_Libraries/Summarization/Summarization_General_Lib.py @@ -128,7 +128,7 @@ def summarize_with_openai(api_key, input_data, custom_prompt_arg, temp=None, sys if not api_key or api_key.strip() == "": logging.info("OpenAI: #1 API key not provided as parameter") logging.info("OpenAI: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['openai'] + api_key = loaded_config_data['openai_api']['api_Key'] if not api_key or api_key.strip() == "": logging.error("OpenAI: #2 API key not found or is empty") @@ -182,7 +182,7 @@ def summarize_with_openai(api_key, input_data, custom_prompt_arg, temp=None, sys logging.debug(f"OpenAI: Extracted text (first 500 chars): {text[:500]}...") logging.debug(f"OpenAI: Custom prompt: {custom_prompt_arg}") - openai_model = loaded_config_data['models']['openai'] or "gpt-4o" + openai_model = loaded_config_data['openai_api']['model'] or "gpt-4o" logging.debug(f"OpenAI: Using model: {openai_model}") headers = { @@ -285,7 +285,7 @@ def summarize_with_anthropic(api_key, input_data, custom_prompt_arg, temp=None, logging.info("Anthropic: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - anthropic_api_key = loaded_config_data['api_keys'].get('anthropic') + anthropic_api_key = loaded_config_data['anthropic_api'].get('api_key') if anthropic_api_key: logging.info("Anthropic: Using API key from config file") else: @@ -344,7 +344,7 @@ def summarize_with_anthropic(api_key, input_data, custom_prompt_arg, temp=None, "content": f"{text} \n\n\n\n{anthropic_prompt}" } - model = loaded_config_data['models']['anthropic'] + model = loaded_config_data['anthropic_api']['model'] data = { "model": model, @@ -462,7 +462,7 @@ def summarize_with_cohere(api_key, input_data, custom_prompt_arg, temp=None, sys logging.info("Cohere: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - cohere_api_key = loaded_config_data['api_keys'].get('cohere') + cohere_api_key = loaded_config_data['cohere_api'].get('api_key') if cohere_api_key: logging.info("Cohere: Using API key from config file") else: @@ -507,7 +507,7 @@ def summarize_with_cohere(api_key, input_data, custom_prompt_arg, temp=None, sys else: raise ValueError("Cohere: Invalid input data format") - cohere_model = loaded_config_data['models']['cohere'] + cohere_model = loaded_config_data['cohere']['model'] if temp is None: temp = 0.3 @@ -612,7 +612,7 @@ def summarize_with_groq(api_key, input_data, custom_prompt_arg, temp=None, syste logging.info("Groq: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - groq_api_key = loaded_config_data['api_keys'].get('groq') + groq_api_key = loaded_config_data['groq_api'].get('api_key') if groq_api_key: logging.info("Groq: Using API key from config file") else: @@ -652,7 +652,7 @@ def summarize_with_groq(api_key, input_data, custom_prompt_arg, temp=None, syste raise ValueError("Groq: Invalid input data format") # Set the model to be used - groq_model = loaded_config_data['models']['groq'] + groq_model = loaded_config_data['groq_api']['model'] if temp is None: temp = 0.2 @@ -762,7 +762,7 @@ def summarize_with_openrouter(api_key, input_data, custom_prompt_arg, temp=None, logging.info("OpenRouter: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - openrouter_api_key = loaded_config_data['api_keys'].get('openrouter') + openrouter_api_key = loaded_config_data['openrouter_api'].get('api_key') if openrouter_api_key: logging.info("OpenRouter: Using API key from config file") else: @@ -771,7 +771,7 @@ def summarize_with_openrouter(api_key, input_data, custom_prompt_arg, temp=None, # Model Selection validation logging.debug("OpenRouter: Validating model selection") loaded_config_data = load_and_log_configs() - openrouter_model = loaded_config_data['models']['openrouter'] + openrouter_model = loaded_config_data['openrouter_api']['model'] logging.debug(f"OpenRouter: Using model from config file: {openrouter_model}") # Final check to ensure we have a valid API key @@ -937,7 +937,7 @@ def summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp=None logging.info("HuggingFace: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - huggingface_api_key = loaded_config_data['api_keys'].get('huggingface') + huggingface_api_key = loaded_config_data['huggingface_api'].get('api_key') logging.debug(f"HuggingFace: API key from config: {huggingface_api_key[:5]}...{huggingface_api_key[-5:]}") if huggingface_api_key: logging.info("HuggingFace: Using API key from config file") @@ -982,7 +982,7 @@ def summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp=None headers = { "Authorization": f"Bearer {huggingface_api_key}" } - huggingface_model = loaded_config_data['models']['huggingface'] + huggingface_model = loaded_config_data['huggingface_api']['model'] API_URL = f"https://api-inference.huggingface.co/models/{huggingface_model}" if temp is None: temp = 0.1 @@ -1068,7 +1068,7 @@ def summarize_with_deepseek(api_key, input_data, custom_prompt_arg, temp=None, s logging.info("DeepSeek: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - deepseek_api_key = loaded_config_data['api_keys'].get('deepseek') + deepseek_api_key = loaded_config_data['deepseek_api'].get('api_key') if deepseek_api_key: logging.info("DeepSeek: Using API key from config file") else: @@ -1107,7 +1107,7 @@ def summarize_with_deepseek(api_key, input_data, custom_prompt_arg, temp=None, s else: raise ValueError("DeepSeek: Invalid input data format") - deepseek_model = loaded_config_data['models']['deepseek'] or "deepseek-chat" + deepseek_model = loaded_config_data['deepseek']['model'] or "deepseek-chat" if temp is None: temp = 0.1 @@ -1205,7 +1205,7 @@ def summarize_with_mistral(api_key, input_data, custom_prompt_arg, temp=None, sy logging.info("Mistral: Using API key provided as parameter") else: # If no parameter is provided, use the key from the config - mistral_api_key = loaded_config_data['api_keys'].get('mistral') + mistral_api_key = loaded_config_data['mistral_api'].get('api_key') if mistral_api_key: logging.info("Mistral: Using API key from config file") else: @@ -1244,7 +1244,7 @@ def summarize_with_mistral(api_key, input_data, custom_prompt_arg, temp=None, sy else: raise ValueError("Mistral: Invalid input data format") - mistral_model = loaded_config_data['models']['mistral'] or "mistral-large-latest" + mistral_model = loaded_config_data['mistral_api']['model'] or "mistral-large-latest" if temp is None: temp = 0.2 @@ -1349,7 +1349,7 @@ def summarize_with_google(api_key, input_data, custom_prompt_arg, temp=None, sys if not api_key or api_key.strip() == "": logging.info("Google: #1 API key not provided as parameter") logging.info("Google: Attempting to use API key from config file") - api_key = loaded_config_data['api_keys']['google'] + api_key = loaded_config_data['google_api']['api_key'] if not api_key or api_key.strip() == "": logging.error("Google: #2 API key not found or is empty") @@ -1403,7 +1403,7 @@ def summarize_with_google(api_key, input_data, custom_prompt_arg, temp=None, sys logging.debug(f"Google: Extracted text (first 500 chars): {text[:500]}...") logging.debug(f"Google: Custom prompt: {custom_prompt_arg}") - google_model = loaded_config_data['models']['google'] or "gemini-1.5-pro" + google_model = loaded_config_data['google_api']['model'] or "gemini-1.5-pro" logging.debug(f"Google: Using model: {google_model}") headers = { From 42f5ad354a88fe2b199d18c9e6db28f809f49700 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 20:42:54 -0800 Subject: [PATCH 71/88] Update TTS_Providers.py --- App_Function_Libraries/TTS/TTS_Providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index c498ccb35..494aafe76 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -10,7 +10,7 @@ import uuid # # External Imports -import edge_tts +#import edge_tts import requests from openai import api_key from pydub import AudioSegment From be9f4a3663e123ea2f566fb33e9dc711aaed879d Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 21:30:04 -0800 Subject: [PATCH 72/88] sanity checkpoint --- App_Function_Libraries/Chat/Chat_Functions.py | 2 +- App_Function_Libraries/Gradio_Related.py | 3 + .../Gradio_UI/WebSearch_tab.py | 265 ++++++++++++++++++ .../Gradio_UI/Website_scraping_tab.py | 6 +- App_Function_Libraries/LLM_API_Calls.py | 10 +- .../Web_Scraping/WebSearch_APIs.py | 8 +- 6 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 App_Function_Libraries/Gradio_UI/WebSearch_tab.py diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index 194313e2b..569a37e5c 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -39,7 +39,7 @@ def approximate_token_count(history): total_tokens = len(total_text.split()) return total_tokens -def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None, streaming=False): +def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming): log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() if not api_key: diff --git a/App_Function_Libraries/Gradio_Related.py b/App_Function_Libraries/Gradio_Related.py index 5f5d7b420..8b82375c1 100644 --- a/App_Function_Libraries/Gradio_Related.py +++ b/App_Function_Libraries/Gradio_Related.py @@ -65,6 +65,7 @@ create_utilities_yt_video_tab from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab +from App_Function_Libraries.Gradio_UI.WebSearch_tab import create_websearch_tab from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab from App_Function_Libraries.Gradio_UI.Workflows_tab import chat_workflows_tab from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_view_all_mediadb_with_versions_tab, \ @@ -468,6 +469,8 @@ def launch_ui(share_public=None, server_mode=False): create_chat_interface_four() chat_workflows_tab() + with gr.TabItem("Web Search & Review", id="websearch group", visible=True): + create_websearch_tab() with gr.TabItem("Character Chat", id="character chat group", visible=True): create_character_card_interaction_tab() create_character_chat_mgmt_tab() diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py new file mode 100644 index 000000000..553d82668 --- /dev/null +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -0,0 +1,265 @@ +# WebSearch_tab.py +# Gradio UI for performing web searches with aggregated results +# +# Imports +import asyncio + +# +# External Imports +import gradio as gr + +from App_Function_Libraries.Web_Scraping.WebSearch_APIs import generate_and_search, analyze_and_aggregate + + +# +# Local Imports + +# +######################################################################################################################## +# +# Functions: + +def create_websearch_tab(): + with gr.TabItem("Web Search & Review", visible=True): + with gr.Blocks() as perplexity_interface: + gr.Markdown("# Perplexity-Style Search Interface") + + # State variables for managing the review process + state = gr.State({ + "phase1_results": None, + "search_params": None, + "selected_results": [] + }) + + with gr.Row(): + with gr.Column(): + # Input components + question_input = gr.Textbox( + label="Enter your question", + placeholder="What would you like to know?", + lines=2 + ) + + with gr.Row(): + search_engine = gr.Dropdown( + choices=["google", "bing", "duckduckgo", "brave"], + value="google", + label="Search Engine" + ) + result_count = gr.Slider( + minimum=1, + maximum=20, + value=10, + step=1, + label="Number of Results" + ) + + with gr.Row(): + content_country = gr.Dropdown( + choices=["US", "UK", "CA", "AU"], + value="US", + label="Content Country" + ) + search_lang = gr.Dropdown( + choices=["en", "es", "fr", "de"], + value="en", + label="Search Language" + ) + + # Status displays + with gr.Row(): + status_display = gr.Markdown( + value="Ready", + label="Status", + elem_classes="status-display" + ) + progress_display = gr.HTML( + value='', + visible=True + ) + + search_btn = gr.Button("Search", variant="primary") + + # Results review section + with gr.Column(visible=False) as review_column: + gr.Markdown("### Review Search Results") + results_review = gr.Dataframe( + headers=["Select", "Title", "URL", "Content Preview"], + datatype=["bool", "str", "str", "str"], + label="Search Results", + interactive=True + ) + confirm_selection_btn = gr.Button("Generate Answer from Selected Results") + + # Final output section + with gr.Column(visible=False) as output_column: + answer_output = gr.Markdown(label="Answer") + sources_output = gr.JSON(label="Sources") + + # Add CSS for status styling + gr.HTML(""" + + """) + + # Event handlers + def update_status(message, status_type="normal"): + status_classes = { + "normal": "status-normal", + "processing": "status-processing", + "error": "status-error", + "success": "status-success" + } + progress_visibility = status_type == "processing" + return ( + gr.Markdown(value=message, elem_classes=status_classes[status_type]), + gr.HTML(visible=progress_visibility) + ) + + def initial_search(question, engine, count, country, lang, state): + try: + # Update status to processing + status, progress = update_status("Initializing search...", "processing") + yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + + search_params = { + "engine": engine, + "content_country": country, + "search_lang": lang, + "output_lang": lang, + "result_count": count, + "subquery_generation": True, + "subquery_generation_llm": "openai", + "relevance_analysis_llm": "openai", + "final_answer_llm": "openai" + } + + status, progress = update_status("Generating sub-queries...", "processing") + yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + + phase1_results = generate_and_search(question, search_params) + + status, progress = update_status("Processing search results...", "processing") + yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + + review_data = [] + for idx, result in enumerate(phase1_results["web_search_results_dict"]["results"]): + review_data.append([ + False, + result.get("title", "No title"), + result.get("url", "No URL"), + result.get("content", "No content")[:200] + "..." + ]) + + state = { + "phase1_results": phase1_results, + "search_params": search_params, + "selected_results": [] + } + + status, progress = update_status("Search completed successfully!", "success") + yield status, progress, state, review_data, gr.Column(visible=True), gr.Column(visible=False) + + except Exception as e: + error_message = f"Error during search: {str(e)}" + status, progress = update_status(error_message, "error") + yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + + def generate_final_answer(selected_results, state): + try: + status, progress = update_status("Generating final answer...", "processing") + yield status, progress, "Processing...", {}, gr.Column(visible=False) + + if not state["phase1_results"] or not state["search_params"]: + raise ValueError("No search results available") + + filtered_results = {} + all_results = state["phase1_results"]["web_search_results_dict"]["results"] + for idx, (is_selected, _, _, _) in enumerate(selected_results): + if is_selected and idx < len(all_results): + filtered_results[str(idx)] = all_results[idx] + + if not filtered_results: + raise ValueError("No results selected") + + status, progress = update_status("Analyzing selected results...", "processing") + yield status, progress, "Analyzing...", {}, gr.Column(visible=False) + + phase2_results = asyncio.run(analyze_and_aggregate( + {"results": filtered_results}, + state["phase1_results"]["sub_query_dict"], + state["search_params"] + )) + + status, progress = update_status("Answer generated successfully!", "success") + yield status, progress, phase2_results["final_answer"]["Report"], phase2_results["final_answer"][ + "evidence"], gr.Column(visible=True) + + except Exception as e: + error_message = f"Error generating answer: {str(e)}" + status, progress = update_status(error_message, "error") + yield status, progress, "Error occurred while generating answer", {}, gr.Column(visible=False) + + # Connect event handlers + search_btn.click( + fn=initial_search, + inputs=[ + question_input, + search_engine, + result_count, + content_country, + search_lang, + state + ], + outputs=[ + status_display, + progress_display, + state, + results_review, + review_column, + output_column + ] + ) + + confirm_selection_btn.click( + fn=generate_final_answer, + inputs=[ + results_review, + state + ], + outputs=[ + status_display, + progress_display, + answer_output, + sources_output, + output_column + ] + ) + + return perplexity_interface + +# +# End of File +######################################################################################################################## diff --git a/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py b/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py index c32dfb286..470b70a1b 100644 --- a/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py +++ b/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py @@ -10,14 +10,11 @@ from concurrent.futures import ThreadPoolExecutor from typing import Optional, List, Dict, Any from urllib.parse import urlparse, urljoin - # # External Imports import gradio as gr from playwright.async_api import TimeoutError, async_playwright from playwright.sync_api import sync_playwright - -from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name # # Local Imports from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_from_sitemap, scrape_by_url_level, \ @@ -25,8 +22,7 @@ from App_Function_Libraries.DB.DB_Manager import list_prompts from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize - - +from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name # ######################################################################################################################## # diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 6dfa423d7..73600a85e 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -116,7 +116,7 @@ def get_openai_embeddings(input_data: str, model: str) -> List[float]: raise ValueError(f"OpenAI: Unexpected error occurred: {str(e)}") -def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_message=None, streaming=None): +def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_message, streaming): loaded_config_data = load_and_log_configs() openai_api_key = api_key try: @@ -132,12 +132,10 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_m logging.debug(f"OpenAI: Using API Key: {openai_api_key[:5]}...{openai_api_key[-5:]}") - if not streaming: + if isinstance(streaming, str): + streaming = streaming.lower() == "true" + elif streaming is None: streaming = loaded_config_data['openai_api']['streaming'] - if streaming == "true" or True: - streaming = True - else: - streaming = False if streaming: logging.debug("OpenAI: Streaming mode enabled") else: diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 538c087f4..53af4f453 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -104,7 +104,6 @@ def generate_and_search(question: str, search_params: Dict) -> Dict: # 1. Generate sub-queries if requested logging.info(f"Generating sub-queries for the query: {question}") - sub_queries = [] sub_query_dict = { "main_goal": question, "sub_questions": [], @@ -294,7 +293,8 @@ def analyze_question(question: str, api_endpoint) -> Dict: for attempt in range(3): try: logging.info(f"Generating sub-questions (attempt {attempt + 1})") - response = chat_api_call(api_endpoint, None, input_data, sub_question_generation_prompt, temp=0.7) + + response = chat_api_call(api_endpoint, None, input_data, sub_question_generation_prompt, temp=0.7, system_message=None, streaming=False) if response: try: # Try to parse as JSON first @@ -384,7 +384,9 @@ async def search_result_relevance( api_key=None, input_data=input_data, prompt=eval_prompt, - temp=0.7 + temp=0.7, + system_message=None, + streaming=False ) # FIXME From 1b1ef5927c1279def04874b50de33657be21313c Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 21:36:03 -0800 Subject: [PATCH 73/88] Update WebSearch_APIs.py --- App_Function_Libraries/Web_Scraping/WebSearch_APIs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 53af4f453..7edc1f37c 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -57,7 +57,7 @@ def initialize_web_search_results_dict(search_params: Dict) -> Dict: "output_lang": search_params.get('output_lang', 'en'), "result_count": 0, "date_range": search_params.get('date_range'), - "safesearch": search_params.get('safesearch', 'moderate'), + "safesearch": search_params.get('safesearch', 'active'), "site_blacklist": search_params.get('site_blacklist', []), "exactTerms": search_params.get('exactTerms'), "excludeTerms": search_params.get('excludeTerms'), @@ -137,7 +137,7 @@ def generate_and_search(question: str, search_params: Dict) -> Dict: output_lang=search_params.get('output_lang', 'en'), result_count=search_params.get('result_count', 10), date_range=search_params.get('date_range'), - safesearch=search_params.get('safesearch', 'moderate'), + safesearch=search_params.get('safesearch', 'active'), site_blacklist=search_params.get('site_blacklist', []), exactTerms=search_params.get('exactTerms'), excludeTerms=search_params.get('excludeTerms'), @@ -690,7 +690,7 @@ def test_perform_websearch_google(): test_1 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10) print(f"Test 1: {test_1}") # FIXME - Fails. Need to fix arg formatting - test_2 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="moderate", site_blacklist=["spam-site.com"]) + test_2 = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10, date_range="y", safesearch="active", site_blacklist=["spam-site.com"]) print(f"Test 2: {test_2}") test_3 = results = perform_websearch("google", "What is the capital of France?", "US", "en", "en", 10) print(f"Test 3: {test_3}") From b6936b2aa5b5ce5e26bf9219d656d8752c66c3c8 Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 21:39:50 -0800 Subject: [PATCH 74/88] Update WebSearch_tab.py --- .../Gradio_UI/WebSearch_tab.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py index 553d82668..6410f8cb1 100644 --- a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -68,6 +68,7 @@ def create_websearch_tab(): # Status displays with gr.Row(): + search_btn = gr.Button("Search", variant="primary") status_display = gr.Markdown( value="Ready", label="Status", @@ -78,19 +79,21 @@ def create_websearch_tab(): visible=True ) - search_btn = gr.Button("Search", variant="primary") - # Results review section - with gr.Column(visible=False) as review_column: - gr.Markdown("### Review Search Results") - results_review = gr.Dataframe( - headers=["Select", "Title", "URL", "Content Preview"], - datatype=["bool", "str", "str", "str"], - label="Search Results", - interactive=True - ) - confirm_selection_btn = gr.Button("Generate Answer from Selected Results") + with gr.Row(): + # Results review section + with gr.Column(visible=False) as review_column: + gr.Markdown("### Review Search Results") + results_review = gr.Dataframe( + headers=["Select", "Title", "URL", "Content Preview"], + datatype=["bool", "str", "str", "str"], + label="Search Results", + interactive=True + ) + confirm_selection_btn = gr.Button("Generate Answer from Selected Results") + + with gr.Row(): # Final output section with gr.Column(visible=False) as output_column: answer_output = gr.Markdown(label="Answer") From ce8e80d424aa662355ab57f22c97433c77d60e0f Mon Sep 17 00:00:00 2001 From: Robert Date: Wed, 1 Jan 2025 22:03:19 -0800 Subject: [PATCH 75/88] Update WebSearch_tab.py --- .../Gradio_UI/WebSearch_tab.py | 213 +++++++++++++----- 1 file changed, 154 insertions(+), 59 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py index 6410f8cb1..6d95f940d 100644 --- a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -22,6 +22,109 @@ def create_websearch_tab(): with gr.TabItem("Web Search & Review", visible=True): with gr.Blocks() as perplexity_interface: + gr.HTML(""" + + """) gr.Markdown("# Perplexity-Style Search Interface") # State variables for managing the review process @@ -79,55 +182,31 @@ def create_websearch_tab(): visible=True ) + with gr.Row(): + # Subqueries display section + with gr.Column(visible=False) as subqueries_column: + gr.Markdown("### Generated Subqueries") + subqueries_display = gr.JSON(label="Subqueries") + with gr.Row(): + # Results review section + with gr.Column(visible=False) as review_column: + gr.Markdown("### Review Search Results") + results_table = gr.Dataframe( + headers=["Include", "Title", "URL", "Content Preview"], + datatype=["bool", "str", "str", "str"], + label="Search Results", + interactive=True + ) + confirm_selection_btn = gr.Button("Generate Answer from Selected Results") - with gr.Row(): - # Results review section - with gr.Column(visible=False) as review_column: - gr.Markdown("### Review Search Results") - results_review = gr.Dataframe( - headers=["Select", "Title", "URL", "Content Preview"], - datatype=["bool", "str", "str", "str"], - label="Search Results", - interactive=True - ) - confirm_selection_btn = gr.Button("Generate Answer from Selected Results") - - with gr.Row(): - # Final output section - with gr.Column(visible=False) as output_column: - answer_output = gr.Markdown(label="Answer") - sources_output = gr.JSON(label="Sources") - - # Add CSS for status styling - gr.HTML(""" - - """) + with gr.Row(): + # Final output section + with gr.Column(visible=False) as output_column: + answer_output = gr.Markdown(label="Answer") + sources_output = gr.JSON(label="Sources") - # Event handlers + # [CSS styles remain the same] def update_status(message, status_type="normal"): status_classes = { "normal": "status-normal", @@ -140,12 +219,12 @@ def update_status(message, status_type="normal"): gr.Markdown(value=message, elem_classes=status_classes[status_type]), gr.HTML(visible=progress_visibility) ) - def initial_search(question, engine, count, country, lang, state): try: # Update status to processing status, progress = update_status("Initializing search...", "processing") - yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( + visible=False) search_params = { "engine": engine, @@ -160,17 +239,20 @@ def initial_search(question, engine, count, country, lang, state): } status, progress = update_status("Generating sub-queries...", "processing") - yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( + visible=False) phase1_results = generate_and_search(question, search_params) status, progress = update_status("Processing search results...", "processing") - yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( + visible=False) + # Prepare results table data review_data = [] - for idx, result in enumerate(phase1_results["web_search_results_dict"]["results"]): + for result in phase1_results["web_search_results_dict"]["results"]: review_data.append([ - False, + True, # Default to included result.get("title", "No title"), result.get("url", "No URL"), result.get("content", "No content")[:200] + "..." @@ -183,14 +265,24 @@ def initial_search(question, engine, count, country, lang, state): } status, progress = update_status("Search completed successfully!", "success") - yield status, progress, state, review_data, gr.Column(visible=True), gr.Column(visible=False) + yield ( + status, + progress, + state, + review_data, + phase1_results["sub_query_dict"], + gr.Column(visible=True), + gr.Column(visible=True), + gr.Column(visible=False) + ) except Exception as e: error_message = f"Error during search: {str(e)}" status, progress = update_status(error_message, "error") - yield status, progress, state, [], gr.Column(visible=False), gr.Column(visible=False) + yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( + visible=False) - def generate_final_answer(selected_results, state): + def generate_final_answer(results_data, state): try: status, progress = update_status("Generating final answer...", "processing") yield status, progress, "Processing...", {}, gr.Column(visible=False) @@ -200,8 +292,9 @@ def generate_final_answer(selected_results, state): filtered_results = {} all_results = state["phase1_results"]["web_search_results_dict"]["results"] - for idx, (is_selected, _, _, _) in enumerate(selected_results): - if is_selected and idx < len(all_results): + + for idx, row in enumerate(results_data): + if row[0] and idx < len(all_results): # If included (checkbox checked) filtered_results[str(idx)] = all_results[idx] if not filtered_results: @@ -240,7 +333,9 @@ def generate_final_answer(selected_results, state): status_display, progress_display, state, - results_review, + results_table, + subqueries_display, + subqueries_column, review_column, output_column ] @@ -249,7 +344,7 @@ def generate_final_answer(selected_results, state): confirm_selection_btn.click( fn=generate_final_answer, inputs=[ - results_review, + results_table, state ], outputs=[ From 0f1b3f6c7dce707b3f182527655bbf5232a84a1c Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 17:37:41 -0800 Subject: [PATCH 76/88] f --- App_Function_Libraries/TTS/TTS_Providers.py | 61 +++++++++++++ App_Function_Libraries/Utils/Utils.py | 11 ++- .../Web_Scraping/WebSearch_APIs.py | 9 +- Docs/Design/Diagram_Generation.md | 2 + Docs/Design/Researcher.md | 2 + Docs/Design/Structured_Outputs.md | 4 + Docs/Design/Summarization.md | 1 + Docs/Design/TTS_STT.md | 14 ++- Docs/Design/UX.md | 3 + Docs/Issues/Citations_and_Confabulations.md | 85 ++++++++++++------- requirements.txt | 1 + 11 files changed, 156 insertions(+), 37 deletions(-) diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 494aafe76..322050370 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -92,6 +92,20 @@ def generate_audio(api_key, text, provider, voice=None, model=None, voice2=None, model=model ) + elif provider == "google": + pass + + elif provider == "gpt-soviTTS": + pass + + elif provider == "piper": + logging.info("Using Piper TTS provider") + return generate_audio_piper( + input_text=text, + model=model, + output_file=output_file, + ) + else: error_msg = f"Invalid TTS provider: {provider}" logging.error(error_msg) @@ -664,6 +678,53 @@ def test_generate_audio_alltalk(): ####################################################### +####################################################### +# +# Piper TTS Provider Functions +# https://github.com/rhasspy/piper +# https://github.com/erew123/alltalk_tts/wiki/API-%E2%80%90-OpenAI-V1-Speech-Compatible-Endpoint + +def generate_audio_piper(input_text, voice=None, model=None, response_format=None, speed=None): + """Generate audio using Piper TTS. + + Args: + + Returns: + str: Path to the generated audio file + """ + + # Input validation + pass + +# +# End of Piper TTS Provider Functions +####################################################### + + +####################################################### +# +# Vevo TTS Provider Functions +# +# https://github.com/open-mmlab/Amphion +# https://huggingface.co/amphion/Vevo + +def generate_audio_piper(input_text, voice=None, model=None, response_format=None, speed=None): + """Generate audio using Piper TTS. + + Args: + + Returns: + str: Path to the generated audio file + """ + + # Input validation + pass + +# +# End of Piper TTS Provider Functions +####################################################### + + ####################################################### # # gpt-soviTTS TTS Provider Functions diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 65e764a08..5c222c485 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -460,6 +460,10 @@ def load_and_log_configs(): yandex_search_api_key = config.get('Search-Engines', 'search_engine_api_key_yandex', fallback='') yandex_search_engine_id = config.get('Search-Engines', 'search_engine_id_yandex', fallback='') + # Prompts + sub_question_generation_prompt = config.get('Prompts', 'sub_question_generation_prompt', fallback='') + search_result_relevance_eval_prompt = config.get('Prompts', 'search_result_relevance_eval_prompt', fallback='') + analyze_search_results_prompt = config.get('Prompts', 'analyze_search_results_prompt', fallback='') return { 'anthropic_api': { @@ -695,7 +699,12 @@ def load_and_log_configs(): 'tavily_search_api_key': tavily_search_api_key, 'yandex_search_api_key': yandex_search_api_key, 'yandex_search_engine_id': yandex_search_engine_id - } + }, + 'prompts': { + 'sub_question_generation_prompt': sub_question_generation_prompt, + 'search_result_relevance_eval_prompt': search_result_relevance_eval_prompt, + 'analyze_search_results_prompt': analyze_search_results_prompt, + }, } except Exception as e: diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 7edc1f37c..65decd543 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -478,6 +478,7 @@ def aggregate_results( - evidence (List[Dict]): List of relevant content items included in the summary. - confidence (float): A rough confidence score (placeholder). """ + logging.info("Aggregating and summarizing relevant results") if not relevant_results: return { "Report": "No relevant results found. Unable to provide an answer.", @@ -493,7 +494,6 @@ def aggregate_results( ) # Example analysis prompt - # FIXME - Add to config.txt/utils.py analyze_search_results_prompt = f""" Generate a comprehensive, well-structured, and informative answer for a given question, using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). @@ -535,13 +535,17 @@ def aggregate_results( input_data = "Follow the above instructions." try: + logging.info("Generating the report") returned_response = chat_api_call( api_endpoint=api_endpoint, api_key=None, input_data=input_data, prompt=analyze_search_results_prompt, - temp=0.7 + temp=0.7, + system_message=None, + streaming=False ) + logging.debug(f"Returned response from LLM: {returned_response}") if returned_response: # You could do further parsing or confidence estimation here return { @@ -552,6 +556,7 @@ def aggregate_results( except Exception as e: logging.error(f"Error aggregating results: {e}") + logging.error("Could not create the report due to an error.") return { "summary": "Could not create the report due to an error.", "evidence": list(relevant_results.values()), diff --git a/Docs/Design/Diagram_Generation.md b/Docs/Design/Diagram_Generation.md index 41e27b563..e71f72ad3 100644 --- a/Docs/Design/Diagram_Generation.md +++ b/Docs/Design/Diagram_Generation.md @@ -4,6 +4,8 @@ ### Link Dump: https://excalidraw.com/ https://www.napkin.ai/ +https://github.com/southbridgeai/diagen + diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index c450d9ddd..f4f467ea9 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -115,6 +115,8 @@ Gemini Deepresearcher https://github.com/assafelovic/gpt-researcher https://arxiv.org/abs/2411.15114 +https://github.com/binary-husky/gpt_academic/blob/master/docs/README.English.md +https://arxiv.org/abs/2409.13741 https://github.com/assafelovic/gpt-researcher/tree/master/gpt_researcher https://github.com/neuml/annotateai diff --git a/Docs/Design/Structured_Outputs.md b/Docs/Design/Structured_Outputs.md index 01a158bac..72d803061 100644 --- a/Docs/Design/Structured_Outputs.md +++ b/Docs/Design/Structured_Outputs.md @@ -1,5 +1,9 @@ # Structured Outputs + +https://towardsdatascience.com/diving-deeper-with-structured-outputs-b4a5d280c208 + + ## Introduction This page serves as documentation regarding the structured outputs within tldw and provides context/justification for the decisions made within the module. diff --git a/Docs/Design/Summarization.md b/Docs/Design/Summarization.md index e28d4a543..ed32ef8b6 100644 --- a/Docs/Design/Summarization.md +++ b/Docs/Design/Summarization.md @@ -18,4 +18,5 @@ https://neptune.ai/blog/llm-evaluation-text-summarization https://phoenix.arize.com/llm-summarization-getting-to-production/ https://blog.metrostar.com/iteratively-summarize-long-documents-llm https://arxiv.org/html/2412.15487v1 +https://arxiv.org/pdf/2204.01849 diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 6d66d924a..02622b4e4 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -34,9 +34,6 @@ https://github.com/albirrkarim/react-speech-highlight-demo https://funaudiollm.github.io/cosyvoice2/ https://funaudiollm.github.io/cosyvoice2/ https://github.com/InternLM/InternLM-XComposer/tree/main/InternLM-XComposer-2.5-OmniLive -https://github.com/huggingface/transformers.js-examples/tree/main/moonshine-web -https://huggingface.co/onnx-community/moonshine-base-ONNX -https://github.com/usefulsensors/moonshine https://github.com/Azure-Samples/aisearch-openai-rag-audio https://www.reddit.com/r/LocalLLaMA/comments/1f0awd6/best_local_open_source_texttospeech_and/ https://github.com/FanaHOVA/smol-podcaster @@ -47,11 +44,17 @@ https://huggingface.co/spaces/mozilla-ai/document-to-podcast/blob/main/app.py https://huggingface.co/spaces/Nymbo/Voice-Clone-Multilingual/tree/main https://github.com/aedocw/epub2tts https://github.com/microsoft/SpeechT5 +https://www.lightnote.co/?utm_source=www.hivefive.community&utm_medium=newsletter&utm_campaign=hive-five-202-a-bias-to-action https://github.com/smellslikeml/dolla_llama - +https://github.com/dnhkng/GlaDOS STT https://github.com/KoljaB/RealtimeSTT + https://github.com/southbridgeai/offmute + MoonShine + https://github.com/usefulsensors/moonshine + https://github.com/huggingface/transformers.js-examples/tree/main/moonshine-web + https://huggingface.co/onnx-community/moonshine-base-ONNX TTS https://github.com/KoljaB/RealtimeTTS @@ -59,6 +62,7 @@ TTS 101 https://www.inferless.com/learn/comparing-different-text-to-speech---tts--models-for-different-use-cases https://clideo.com/resources/what-is-tts + https://pub.towardsai.net/the-ultimate-guide-to-audio-processing-principles-techniques-and-applications-7724efea00e8 RVC 101 https://gudgud96.github.io/2024/09/26/annotated-rvc/ @@ -130,6 +134,8 @@ MaskGCT Parler https://github.com/huggingface/parler-tts +Sherpa ONNX + https://github.com/k2-fsa/sherpa-onnx YourTTS https://github.com/Edresson/YourTTS diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 8c11ad893..9fbd8e2af 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -84,6 +84,9 @@ https://markwhen.com/ https://kando.menu/ https://deepseek-artifacts.vercel.app/ +Not waifus, but clippy: + https://github.com/fghrsh/live2d_demo + Prompt Engineering page: - 3 Sections diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index 27c001c47..577d04e90 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -6,20 +6,21 @@ 3. [References](#references) - -https://arxiv.org/abs/2412.04235 -https://arxiv.org/abs/2412.11536 -https://github.com/sunnynexus/RetroLLM -https://huggingface.co/PleIAs/Pleias-Nano -https://deepmind.google/discover/blog/facts-grounding-a-new-benchmark-for-evaluating-the-factuality-of-large-language-models/ -https://arxiv.org/abs/2412.14860 -https://arxiv.org/abs/2412.14686 -https://arxiv.org/abs/2412.18069 - - - RAG https://www.lycee.ai/blog/rag-ragallucinations-and-how-to-fight-them + https://huggingface.co/PleIAs/Pleias-Nano + https://arxiv.org/abs/2412.11536 + https://cloud.google.com/generative-ai-app-builder/docs/check-grounding + https://cloud.google.com/generative-ai-app-builder/docs/grounded-gen + https://arxiv.org/html/2412.15189v1#S6 + https://aclanthology.org/2024.fever-1.10/ + https://arxiv.org/pdf/2412.15189 + https://huggingface.co/papers/2408.12060 + https://primer.ai/research/rag-v-divide-and-conquer-with-factual-claims/ + https://arxiv.org/abs/2411.06037 + https://www.sciencedirect.com/science/article/abs/pii/S0306457320309675 + https://github.com/Huffon/factsumm + https://arxiv.org/abs/2410.07176 Finetuning: - https://eugeneyan.com/writing/finetuning/ @@ -30,8 +31,12 @@ Finetuning: - **101** - Unsorted - https://mattyyeung.github.io/deterministic-quoting#7-conclusion-is-this-really-ready-for-healthcare + https://github.com/sunnynexus/RetroLLM + - https://github.com/MadryLab/context-cite - +Abstractive Proposition Segmentation + https://arxiv.org/abs/2406.19803 + https://huggingface.co/google/gemma-2b-aps-it Anthropic: ``` @@ -71,6 +76,12 @@ Benchmarks https://huggingface.co/spaces/vectara/Hallucination-evaluation-leaderboard https://huggingface.co/spaces/hallucinations-leaderboard/leaderboard https://osu-nlp-group.github.io/AttributionBench/ + Fake News + https://arxiv.org/abs/2412.14686 + FACTS + https://www.kaggle.com/facts-leaderboard + https://storage.googleapis.com/deepmind-media/FACTS/FACTS_grounding_paper.pdf + https://deepmind.google/discover/blog/facts-grounding-a-new-benchmark-for-evaluating-the-factuality-of-large-language-models/ Detecting Hallucinations using Semantic Entropy: @@ -93,38 +104,52 @@ Explainability Research https://github.com/EdinburghNLP/awesome-hallucination-detection - https://arxiv.org/abs/2407.13481 - https://arxiv.org/abs/2408.06195 - https://arxiv.org/abs/2407.19813 https://www.lycee.ai/blog/rag-ragallucinations-and-how-to-fight-them - https://arxiv.org/abs/2407.16557 - https://arxiv.org/abs/2407.16604 - https://arxiv.org/pdf/2309.11495 - https://deepmind.google/research/publications/85420/ https://thetechoasis.beehiiv.com/p/eliminating-hallucinations-robots-imitate-us - https://arxiv.org/abs/2407.19825 - https://arxiv.org/abs/2411.14257 - https://arxiv.org/pdf/2406.02543 - https://arxiv.org/pdf/2410.19385 - https://arxiv.org/abs/2406.10279 - https://arxiv.org/abs/2402.17811 - https://arxiv.org/pdf/2409.18475 https://llm-editing.github.io/ - https://arxiv.org/abs/2411.14257 - https://arxiv.org/pdf/2407.03651 https://cleanlab.ai/blog/trustworthy-language-model/ - https://arxiv.org/abs/2408.07852 + General + https://arxiv.org/pdf/2410.19385 + https://arxiv.org/pdf/2409.18475 + https://arxiv.org/pdf/2406.02543 + https://arxiv.org/abs/2407.19825 + https://arxiv.org/abs/2407.16604 + https://arxiv.org/abs/2407.16557 + https://arxiv.org/abs/2412.04235 + Attention/Long Context + https://arxiv.org/abs/2407.13481 + https://arxiv.org/pdf/2407.03651 + CoV + https://arxiv.org/pdf/2309.11495 + KnowledgeGraph + https://arxiv.org/abs/2408.07852 + Mutual Reasoning + https://arxiv.org/abs/2408.06195 + Self-Reasoning + https://arxiv.org/abs/2407.19813 + https://arxiv.org/abs/2412.14860 Detecting Hallucinations https://arxiv.org/abs/2410.22071 https://arxiv.org/abs/2410.02707 + https://arxiv.org/abs/2411.14257 Reflective thinking https://arxiv.org/html/2404.09129v1 https://github.com/yanhong-lbh/LLM-SelfReflection-Eval Semantic Entropy https://www.nature.com/articles/s41586-024-07421-0 https://arxiv.org/abs/2406.15927 + Software Packages + https://arxiv.org/abs/2406.10279 + TruthX + https://arxiv.org/abs/2402.17811 + Working memory + https://arxiv.org/abs/2412.18069 HALVA https://research.google/blog/halva-hallucination-attenuated-language-and-vision-assistant/ +Long Form Factuality - Google + https://github.com/google-deepmind/long-form-factuality + https://deepmind.google/research/publications/85420/ + LLM As Judge: diff --git a/requirements.txt b/requirements.txt index a85f2f6fe..6e0597f93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,7 @@ openai #outlines pandas Pillow +#piper-tts playwright # I'm not using postgres yet... #psycopg2 From d7f5f24a4d32afe22f4cf81a30aecbe3abf5762f Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 19:59:51 -0800 Subject: [PATCH 77/88] f --- App_Function_Libraries/TTS/TTS_Providers.py | 4 +- .../Web_Scraping/WebSearch_APIs.py | 155 +++++++++++++++++- Docs/Design/Inference_Engines.md | 19 ++- Docs/Design/TTS_STT.md | 33 +++- Docs/Issues/Citations_and_Confabulations.md | 1 + Docs/Issues/Evaluation_Plans.md | 1 + 6 files changed, 196 insertions(+), 17 deletions(-) diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 322050370..49b1bea96 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -708,7 +708,7 @@ def generate_audio_piper(input_text, voice=None, model=None, response_format=Non # https://github.com/open-mmlab/Amphion # https://huggingface.co/amphion/Vevo -def generate_audio_piper(input_text, voice=None, model=None, response_format=None, speed=None): +def generate_audio_vevo(input_text, voice=None, model=None, response_format=None, speed=None): """Generate audio using Piper TTS. Args: @@ -721,7 +721,7 @@ def generate_audio_piper(input_text, voice=None, model=None, response_format=Non pass # -# End of Piper TTS Provider Functions +# End of Vevo TTS Provider Functions ####################################################### diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index 65decd543..ba241f882 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -20,6 +20,8 @@ from requests import RequestException from requests.adapters import HTTPAdapter from urllib3 import Retry + +from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize # # Local Imports from App_Function_Libraries.Utils.Utils import loaded_config_data @@ -351,12 +353,28 @@ async def search_result_relevance( Dict[str, Dict]: A dictionary of relevant results, keyed by a unique ID or index. """ relevant_results = {} + + # Summarization prompt template + summarization_prompt = """ + Summarize the following text in a concise way that captures the key information relevant to this question: "{question}" + + Text to summarize: + {content} + + Instructions: + 1. Focus on information relevant to the question + 2. Keep the summary under 2000 characters + 3. Maintain factual accuracy + 4. Include key details and statistics if present + """ + for idx, result in enumerate(search_results): content = result.get("content", "") if not content: logging.error("No Content found in search results array!") continue + # First, evaluate relevance eval_prompt = f""" Given the following search results for the user's question: "{original_question}" and the generated sub-questions: {sub_questions}, evaluate the relevance of the search result to the user's question. Explain your reasoning for selection. @@ -377,8 +395,11 @@ async def search_result_relevance( input_data = "Evaluate the relevance of the search result." try: - # Perform API call to evaluate relevance - sleep_time = random.uniform(0.2, 0.6) # Add a random delay to avoid rate limiting + # Add delay to avoid rate limiting + sleep_time = random.uniform(0.2, 0.6) + await asyncio.sleep(sleep_time) + + # Evaluate relevance relevancy_result = chat_api_call( api_endpoint=api_endpoint, api_key=None, @@ -411,22 +432,47 @@ async def search_result_relevance( reasoning = reasoning_match.group(1).strip() if is_relevant: + logging.debug("Relevant result found.") # Use the 'id' from the result if available, otherwise use idx result_id = result.get("id", str(idx)) # Scrape the content of the relevant result scraped_content = await scrape_article(result['url']) + + # Create Summarization prompt + logging.debug(f"Creating Summarization Prompt for result idx={idx}") + summary_prompt = summarization_prompt.format( + question=original_question, + content=scraped_content['content'] + ) + + # Add delay before summarization + await asyncio.sleep(sleep_time) + + # Generate summary using the summarize function + logging.info(f"Summarizing relevant result: ID={result_id}") + summary = summarize( + input_data=scraped_content['content'], + custom_prompt_arg=summary_prompt, + api_name=api_endpoint, + api_key=None, + temp=0.7, + system_message=None, + streaming=False + ) + relevant_results[result_id] = { - "content": scraped_content['content'], + "content": summary, # Store the summary instead of full content + "original_content": scraped_content['content'], # Keep original content if needed "reasoning": reasoning } - logging.info(f"Relevant result found: ID={result_id} Reasoning={reasoning}") + logging.info(f"Relevant result found and summarized: ID={result_id}; Reasoning={reasoning}") else: logging.info(f"Irrelevant result: {reasoning}") else: logging.warning("Failed to parse the API response for relevance analysis.") except Exception as e: - logging.error(f"Error during relevance evaluation for result idx={idx}: {e}") + logging.error(f"Error during relevance evaluation/summarization for result idx={idx}: {e}") return relevant_results @@ -486,15 +532,22 @@ def aggregate_results( "confidence": 0.0 } + # FIXME - Add summarization loop + logging.info("Summarizing relevant results") + # ADD Code here to summarize the relevant results + + # FIXME - Validate and test thoroughly, also structured generation - # Concatenate relevant contents for summarization + # Concatenate relevant contents for final analysis concatenated_texts = "\n\n".join( f"ID: {rid}\nContent: {res['content']}\nReasoning: {res['reasoning']}" for rid, res in relevant_results.items() ) - # Example analysis prompt - analyze_search_results_prompt = f""" + current_date = time.strftime("%Y-%m-%d") + + # Aggregation Prompt #1 + analyze_search_results_prompt_1 = f""" Generate a comprehensive, well-structured, and informative answer for a given question, using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). Use an unbiased, journalistic tone, adapting the level of formality to match the user’s question. @@ -532,6 +585,90 @@ def aggregate_results( Answer (in the language of the user's question): """ + # Aggregation Prompt #2 + analyze_search_results_prompt_2 = f"""INITIAL_QUERY: Here are some sources {concatenated_texts}. Read these carefully, as you will be asked a Query about them. + # General Instructions + + Write an accurate, detailed, and comprehensive response to the user's query located at INITIAL_QUERY. Additional context is provided as "USER_INPUT" after specific questions. Your answer should be informed by the provided "Search results". Your answer must be precise, of high-quality, and written by an expert using an unbiased and journalistic tone. Your answer must be written in the same language as the query, even if language preference is different. + + You MUST cite the most relevant search results that answer the query. Do not mention any irrelevant results. You MUST ADHERE to the following instructions for citing search results: + - to cite a search result, enclose its index located above the summary with brackets at the end of the corresponding sentence, for example "Ice is less dense than water[1][2]." or "Paris is the capital of France[1][4][5]." + - NO SPACE between the last word and the citation, and ALWAYS use brackets. Only use this format to cite search results. NEVER include a References section at the end of your answer. + - If you don't know the answer or the premise is incorrect, explain why. + If the search results are empty or unhelpful, answer the query as well as you can with existing knowledge. + + You MUST NEVER use moralization or hedging language. AVOID using the following phrases: + - "It is important to ..." + - "It is inappropriate ..." + - "It is subjective ..." + + You MUST ADHERE to the following formatting instructions: + - Use markdown to format paragraphs, lists, tables, and quotes whenever possible. + - Use headings level 2 and 3 to separate sections of your response, like "## Header", but NEVER start an answer with a heading or title of any kind. + - Use single new lines for lists and double new lines for paragraphs. + - Use markdown to render images given in the search results. + - NEVER write URLs or links. + + # Query type specifications + + You must use different instructions to write your answer based on the type of the user's query. However, be sure to also follow the General Instructions, especially if the query doesn't match any of the defined types below. Here are the supported types. + + ## Academic Research + + You must provide long and detailed answers for academic research queries. Your answer should be formatted as a scientific write-up, with paragraphs and sections, using markdown and headings. + + ## Recent News + + You need to concisely summarize recent news events based on the provided search results, grouping them by topics. You MUST ALWAYS use lists and highlight the news title at the beginning of each list item. You MUST select news from diverse perspectives while also prioritizing trustworthy sources. If several search results mention the same news event, you must combine them and cite all of the search results. Prioritize more recent events, ensuring to compare timestamps. You MUST NEVER start your answer with a heading of any kind. + + ## Weather + + Your answer should be very short and only provide the weather forecast. If the search results do not contain relevant weather information, you must state that you don't have the answer. + + ## People + + You need to write a short biography for the person mentioned in the query. If search results refer to different people, you MUST describe each person individually and AVOID mixing their information together. NEVER start your answer with the person's name as a header. + + ## Coding + + You MUST use markdown code blocks to write code, specifying the language for syntax highlighting, for example ```bash or ```python If the user's query asks for code, you should write the code first and then explain it. + + ## Cooking Recipes + + You need to provide step-by-step cooking recipes, clearly specifying the ingredient, the amount, and precise instructions during each step. + + ## Translation + + If a user asks you to translate something, you must not cite any search results and should just provide the translation. + + ## Creative Writing + + If the query requires creative writing, you DO NOT need to use or cite search results, and you may ignore General Instructions pertaining only to search. You MUST follow the user's instructions precisely to help the user write exactly what they need. + + ## Science and Math + + If the user query is about some simple calculation, only answer with the final result. Follow these rules for writing formulas: + - Always use \( and\) for inline formulas and\[ and\] for blocks, for example\(x^4 = x - 3 \) + - To cite a formula add citations to the end, for example\[ \sin(x) \] [1][2] or \(x^2-2\) [4]. + - Never use $ or $$ to render LaTeX, even if it is present in the user query. + - Never use unicode to render math expressions, ALWAYS use LaTeX. + - Never use the \label instruction for LaTeX. + + ## URL Lookup + + When the user's query includes a URL, you must rely solely on information from the corresponding search result. DO NOT cite other search results, ALWAYS cite the first result, e.g. you need to end with [1]. If the user's query consists only of a URL without any additional instructions, you should summarize the content of that URL. + + ## Shopping + + If the user query is about shopping for a product, you MUST follow these rules: + - Organize the products into distinct sectors. For example, you could group shoes by style (boots, sneakers, etc.) + - Cite at most 9 search results using the format provided in General Instructions to avoid overwhelming the user with too many options. + + The current date is: {current_date} + + The user's query is: {question} + """ + input_data = "Follow the above instructions." try: @@ -540,7 +677,7 @@ def aggregate_results( api_endpoint=api_endpoint, api_key=None, input_data=input_data, - prompt=analyze_search_results_prompt, + prompt=analyze_search_results_prompt_2, temp=0.7, system_message=None, streaming=False diff --git a/Docs/Design/Inference_Engines.md b/Docs/Design/Inference_Engines.md index 893f339b3..dce6e81b5 100644 --- a/Docs/Design/Inference_Engines.md +++ b/Docs/Design/Inference_Engines.md @@ -1,10 +1,27 @@ # Inference Engines +## Introduction + + +### HuggingFace Transformers +https://huggingface.co/docs/transformers/main/en/conversations + +### Llama.cpp + + +### Llamafile + + +### TabbyAPI + + +### vLLM + ### Link Dump https://github.com/intel/neural-compressor/tree/v3.2 - +https://southbridge-research.notion.site/Entropixplained-11e5fec70db18022b083d7d7b0e93505 https://magazine.sebastianraschka.com/p/practical-tips-for-finetuning-llms?utm_medium=email https://magazine.sebastianraschka.com/p/lora-and-dora-from-scratch?utm_medium=email https://arxiv.org/abs/2405.09673 \ No newline at end of file diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 02622b4e4..6b072eb13 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -69,16 +69,24 @@ TTS Datasets(?) https://voice-models.com/ +Auralis + https://github.com/astramind-ai/Auralis + https://www.astramind.ai/post/auralis + +Amphion + https://github.com/open-mmlab/Amphion + https://huggingface.co/amphion/Vevo + https://github.com/open-mmlab/Amphion/blob/main/models/vc/vevo/README.md + https://openreview.net/pdf?id=anQDiQZhDP + https://versavoice.github.io/ + Bark https://github.com/suno-ai/bark - ChatTTS https://huggingface.co/2Noise/ChatTTS https://chattts.com/#Demo - - Coqui TTS https://github.com/idiap/coqui-ai-TTS https://huggingface.co/spaces/coqui/xtts/blob/main/app.py @@ -110,6 +118,7 @@ MoonShine Gemini https://ai.google.dev/gemini-api/docs#rest https://ai.google.dev/gemini-api/docs/models/gemini-v2 + https://github.com/google-gemini/cookbook/blob/main/quickstarts/Audio.ipynb ElevenLabs https://github.com/elevenlabs/elevenlabs-examples/blob/main/examples/text-to-speech/python/text_to_speech_file.py @@ -131,9 +140,23 @@ MaskGCT https://github.com/open-mmlab/Amphion/blob/main/models/tts/maskgct/maskgct_inference.py https://huggingface.co/amphion/MaskGCT +Mimic + https://github.com/MycroftAI/mimic3 + + Parler https://github.com/huggingface/parler-tts +Piper (linux only) + https://github.com/rhasspy/piper + https://github.com/rhasspy/piper/issues/644 + https://github.com/rhasspy/piper/discussions/326#discussioncomment-7935208 + https://noerguerra.com/how-to-read-text-aloud-with-piper-and-python/ + https://ssamjh.nz/create-custom-piper-tts-voice/ + https://www.trycatchdebug.net/news/1377664/realtime-tts-with-pipertts-and-openai + https://huggingface.co/rhasspy/piper-voices/tree/main + https://huggingface.co/datasets/rhasspy/piper-checkpoints/tree/main + Sherpa ONNX https://github.com/k2-fsa/sherpa-onnx @@ -428,7 +451,7 @@ GPT-SoviTTS https://github.com/HanxSmile/Simplify-GPT-SoVITS https://github.com/lrxwisdom001/GPT-SoVITS-Novels/tree/main/voice_synthesis openneurosama - https://github.com/JarodMica/open-neruosama/blob/master/main.py - + https://huggingface.co/cpumaxx/SoVITS-anime-mini-tts https://tts.x86.st/ Finetuning is very quick (about 5 minutes). Captioning of audio was automated with faster-whisper (it is required that the audio is captioned). @@ -464,6 +487,6 @@ Comfyui integration - F ### Dataset Creation/Curation - +https://voiceguide.arimil.com/ diff --git a/Docs/Issues/Citations_and_Confabulations.md b/Docs/Issues/Citations_and_Confabulations.md index 577d04e90..0a0de9615 100644 --- a/Docs/Issues/Citations_and_Confabulations.md +++ b/Docs/Issues/Citations_and_Confabulations.md @@ -37,6 +37,7 @@ Finetuning: Abstractive Proposition Segmentation https://arxiv.org/abs/2406.19803 https://huggingface.co/google/gemma-2b-aps-it + https://ritvik19.medium.com/papers-explained-244-gemma-aps-8fac1838b9ef Anthropic: ``` diff --git a/Docs/Issues/Evaluation_Plans.md b/Docs/Issues/Evaluation_Plans.md index ee5e1d0b1..4a76b6a63 100644 --- a/Docs/Issues/Evaluation_Plans.md +++ b/Docs/Issues/Evaluation_Plans.md @@ -50,6 +50,7 @@ https://www.atla-ai.com/post/evaluating-the-evaluator https://hamel.dev/blog/posts/llm-judge/ https://github.com/scrapinghub/article-extraction-benchmark https://github.com/Zhe-Young/SelfCorrectDecompose +https://eugeneyan.com/writing/evals/ Have LLMs play Social deception games Different results depending on batch size during evaluations - https://x.com/bnjmn_marie/status/1846834917608407199 From 1b828235df89725084ae235b0ca97860b19f9c64 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 22:02:52 -0800 Subject: [PATCH 78/88] f --- App_Function_Libraries/Chat/Chat_Functions.py | 1 + .../Gradio_UI/WebSearch_tab.py | 474 ++++++++---------- .../Web_Scraping/Article_Extractor_Lib.py | 12 +- 3 files changed, 209 insertions(+), 278 deletions(-) diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index 569a37e5c..91b6bdbda 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -40,6 +40,7 @@ def approximate_token_count(history): return total_tokens def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming): + logging.info(f"Debug - Chat API Call - API Endpoint: {api_endpoint}") log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() if not api_key: diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py index 6d95f940d..bdda69386 100644 --- a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -3,17 +3,13 @@ # # Imports import asyncio - +import logging # # External Imports import gradio as gr - -from App_Function_Libraries.Web_Scraping.WebSearch_APIs import generate_and_search, analyze_and_aggregate - - # # Local Imports - +from App_Function_Libraries.Web_Scraping.WebSearch_APIs import generate_and_search, analyze_and_aggregate # ######################################################################################################################## # @@ -22,6 +18,7 @@ def create_websearch_tab(): with gr.TabItem("Web Search & Review", visible=True): with gr.Blocks() as perplexity_interface: + # Add CSS gr.HTML(""" """) - gr.Markdown("# Perplexity-Style Search Interface") - # State variables for managing the review process + gr.Markdown("# Web Search Interface") + + # State for managing the review process state = gr.State({ "phase1_results": None, "search_params": None, - "selected_results": [] + "selected_indices": [] }) with gr.Row(): @@ -169,192 +102,181 @@ def create_websearch_tab(): label="Search Language" ) - # Status displays - with gr.Row(): - search_btn = gr.Button("Search", variant="primary") - status_display = gr.Markdown( - value="Ready", - label="Status", - elem_classes="status-display" - ) - progress_display = gr.HTML( - value='', - visible=True - ) - + # Status and progress displays with gr.Row(): - # Subqueries display section - with gr.Column(visible=False) as subqueries_column: - gr.Markdown("### Generated Subqueries") - subqueries_display = gr.JSON(label="Subqueries") + search_btn = gr.Button("Search", variant="primary") + status_display = gr.Markdown("Ready", elem_classes=["status-display", "status-normal"]) + progress_display = gr.HTML(visible=False) + + # Results review section + with gr.Column(visible=False) as review_column: + gr.Markdown("### Search Results") + results_container = gr.HTML() # Container for results + confirm_selection_btn = gr.Button("Generate Answer from Selected Results") + + # Final output section + with gr.Column(visible=False) as output_column: + answer_output = gr.Markdown(label="Generated Answer") + sources_output = gr.JSON(label="Sources") + + def update_status(message, status_type="normal"): + """Update the status display with the given message and type.""" + status_classes = { + "normal": "status-normal", + "processing": "status-processing", + "error": "status-error", + "success": "status-success" + } + return ( + gr.Markdown(value=message, elem_classes=["status-display", status_classes[status_type]]), + gr.HTML(visible=(status_type == "processing")) + ) - with gr.Row(): - # Results review section - with gr.Column(visible=False) as review_column: - gr.Markdown("### Review Search Results") - results_table = gr.Dataframe( - headers=["Include", "Title", "URL", "Content Preview"], - datatype=["bool", "str", "str", "str"], - label="Search Results", - interactive=True - ) - confirm_selection_btn = gr.Button("Generate Answer from Selected Results") + def format_results_html(results): + """Format search results as HTML with checkboxes.""" + html = "" + for idx, result in enumerate(results): + html += f""" +
+
+ +
+
{result.get('title', 'No title')}
+
{result.get('url', 'No URL')}
+
{result.get('content', 'No content')[:200]}...
+
+ """ + return html + + def initial_search(question, engine, count, country, lang, state): + try: + status, progress = update_status("Initializing search...", "processing") + yield status, progress, state, "", gr.Column(visible=False), gr.Column(visible=False) + + search_params = { + "engine": engine, + "content_country": country, + "search_lang": lang, + "output_lang": lang, + "result_count": count + } - with gr.Row(): - # Final output section - with gr.Column(visible=False) as output_column: - answer_output = gr.Markdown(label="Answer") - sources_output = gr.JSON(label="Sources") - - # [CSS styles remain the same] - def update_status(message, status_type="normal"): - status_classes = { - "normal": "status-normal", - "processing": "status-processing", - "error": "status-error", - "success": "status-success" - } - progress_visibility = status_type == "processing" - return ( - gr.Markdown(value=message, elem_classes=status_classes[status_type]), - gr.HTML(visible=progress_visibility) - ) - def initial_search(question, engine, count, country, lang, state): - try: - # Update status to processing - status, progress = update_status("Initializing search...", "processing") - yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( - visible=False) - - search_params = { - "engine": engine, - "content_country": country, - "search_lang": lang, - "output_lang": lang, - "result_count": count, - "subquery_generation": True, - "subquery_generation_llm": "openai", - "relevance_analysis_llm": "openai", - "final_answer_llm": "openai" - } + # Generate and search + phase1_results = generate_and_search(question, search_params) - status, progress = update_status("Generating sub-queries...", "processing") - yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( - visible=False) - - phase1_results = generate_and_search(question, search_params) - - status, progress = update_status("Processing search results...", "processing") - yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( - visible=False) - - # Prepare results table data - review_data = [] - for result in phase1_results["web_search_results_dict"]["results"]: - review_data.append([ - True, # Default to included - result.get("title", "No title"), - result.get("url", "No URL"), - result.get("content", "No content")[:200] + "..." - ]) - - state = { - "phase1_results": phase1_results, - "search_params": search_params, - "selected_results": [] - } + # Get results list + results_list = phase1_results["web_search_results_dict"]["results"] + + # Format results as HTML + results_html = format_results_html(results_list) - status, progress = update_status("Search completed successfully!", "success") - yield ( - status, - progress, + # Update state with the list indices + state = { + "phase1_results": phase1_results, + "search_params": search_params, + "selected_indices": list(range(len(results_list))) + } + + logging.info(f"Search completed. Results count: {len(results_list)}") + logging.info(f"Selected indices: {state['selected_indices']}") + + status, progress = update_status("Search completed successfully!", "success") + yield status, progress, state, results_html, gr.Column(visible=True), gr.Column(visible=False) + + except Exception as e: + error_message = f"Error during search: {str(e)}" + logging.error(f"Search error: {error_message}") + logging.error("Traceback: ", exc_info=True) + status, progress = update_status(error_message, "error") + yield status, progress, state, "", gr.Column(visible=False), gr.Column(visible=False) + + def generate_final_answer(state): + try: + status, progress = update_status("Generating final answer...", "processing") + yield status, progress, "Processing...", {}, gr.Column(visible=False) + + if not state["phase1_results"] or not state["search_params"]: + raise ValueError("No search results available") + + # Get selected results + filtered_results = {} + web_search_results = state["phase1_results"]["web_search_results_dict"]["results"] + + logging.info(f"Processing web search results: {web_search_results}") + logging.info(f"Selected indices: {state['selected_indices']}") + + # Convert list results to dictionary format expected by analyze_and_aggregate + for idx in state["selected_indices"]: + if idx < len(web_search_results): + result = web_search_results[idx] + filtered_results[str(idx)] = { + 'content': result.get('content', ''), + 'url': result.get('url', ''), + 'title': result.get('title', ''), + 'metadata': result.get('metadata', {}) + } + + if not filtered_results: + raise ValueError("No results selected") + + logging.info(f"Filtered results prepared for analysis: {filtered_results}") + + # Create the input structure expected by analyze_and_aggregate + input_data = { + "results": filtered_results + } + + # Generate final answer + phase2_results = asyncio.run(analyze_and_aggregate( + input_data, + state["phase1_results"].get("sub_query_dict", {}), + state["search_params"] + )) + + status, progress = update_status("Answer generated successfully!", "success") + yield status, progress, phase2_results["final_answer"]["Report"], phase2_results["final_answer"][ + "evidence"], gr.Column(visible=True) + + except Exception as e: + error_message = f"Error generating answer: {str(e)}" + logging.error(f"Error in generate_final_answer: {error_message}") + logging.error(f"State: {state}") + logging.error(f"Traceback: ", exc_info=True) # Add full traceback + status, progress = update_status(error_message, "error") + yield status, progress, "Error occurred while generating answer", {}, gr.Column(visible=False) + + # Connect event handlers + search_btn.click( + fn=initial_search, + inputs=[ + question_input, + search_engine, + result_count, + content_country, + search_lang, + state + ], + outputs=[ + status_display, + progress_display, state, - review_data, - phase1_results["sub_query_dict"], - gr.Column(visible=True), - gr.Column(visible=True), - gr.Column(visible=False) - ) + results_container, + review_column, + output_column + ] + ) - except Exception as e: - error_message = f"Error during search: {str(e)}" - status, progress = update_status(error_message, "error") - yield status, progress, state, [], {}, gr.Column(visible=False), gr.Column(visible=False), gr.Column( - visible=False) - - def generate_final_answer(results_data, state): - try: - status, progress = update_status("Generating final answer...", "processing") - yield status, progress, "Processing...", {}, gr.Column(visible=False) - - if not state["phase1_results"] or not state["search_params"]: - raise ValueError("No search results available") - - filtered_results = {} - all_results = state["phase1_results"]["web_search_results_dict"]["results"] - - for idx, row in enumerate(results_data): - if row[0] and idx < len(all_results): # If included (checkbox checked) - filtered_results[str(idx)] = all_results[idx] - - if not filtered_results: - raise ValueError("No results selected") - - status, progress = update_status("Analyzing selected results...", "processing") - yield status, progress, "Analyzing...", {}, gr.Column(visible=False) - - phase2_results = asyncio.run(analyze_and_aggregate( - {"results": filtered_results}, - state["phase1_results"]["sub_query_dict"], - state["search_params"] - )) - - status, progress = update_status("Answer generated successfully!", "success") - yield status, progress, phase2_results["final_answer"]["Report"], phase2_results["final_answer"][ - "evidence"], gr.Column(visible=True) - - except Exception as e: - error_message = f"Error generating answer: {str(e)}" - status, progress = update_status(error_message, "error") - yield status, progress, "Error occurred while generating answer", {}, gr.Column(visible=False) - - # Connect event handlers - search_btn.click( - fn=initial_search, - inputs=[ - question_input, - search_engine, - result_count, - content_country, - search_lang, - state - ], - outputs=[ - status_display, - progress_display, - state, - results_table, - subqueries_display, - subqueries_column, - review_column, - output_column - ] - ) - - confirm_selection_btn.click( - fn=generate_final_answer, - inputs=[ - results_table, - state - ], - outputs=[ - status_display, - progress_display, - answer_output, - sources_output, - output_column - ] - ) + confirm_selection_btn.click( + fn=generate_final_answer, + inputs=[state], + outputs=[ + status_display, + progress_display, + answer_output, + sources_output, + output_column + ] + ) return perplexity_interface diff --git a/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py b/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py index 4a212b3ce..b1dc35a2d 100644 --- a/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py +++ b/App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py @@ -66,7 +66,9 @@ def get_page_title(url: str) -> str: async def scrape_article(url: str, custom_cookies: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: + logging.info(f"Scraping article from URL: {url}") async def fetch_html(url: str) -> str: + logging.info(f"Fetching HTML from {url}") async with async_playwright() as p: browser = await p.chromium.launch(headless=True) try: @@ -79,9 +81,10 @@ async def fetch_html(url: str) -> str: await context.add_cookies(custom_cookies) page = await context.new_page() try: - await page.goto(url, wait_until="domcontentloaded") - await page.wait_for_load_state("networkidle", timeout=30000) + await page.goto(url, wait_until="domcontentloaded", timeout=10000) # 10-second timeout + await page.wait_for_load_state("networkidle", timeout=10000) # 10-second timeout content = await page.content() + logging.info(f"HTML fetched successfully from {url}") log_counter("html_fetched", labels={"url": url}) return content except Exception as e: @@ -91,6 +94,7 @@ async def fetch_html(url: str) -> str: await browser.close() def extract_article_data(html: str, url: str) -> dict: + logging.info(f"Extracting article data from HTML for {url}") # FIXME - Add option for extracting comments/tables/images downloaded = trafilatura.extract(html, include_comments=False, include_tables=False, include_images=False) metadata = trafilatura.extract_metadata(html) @@ -105,6 +109,7 @@ def extract_article_data(html: str, url: str) -> dict: } if downloaded: + logging.info(f"Content extracted successfully from {url}") log_counter("article_extracted", labels={"success": "true", "url": url}) # Add metadata to content result['content'] = ContentMetadataHandler.format_content_with_metadata( @@ -134,6 +139,7 @@ def extract_article_data(html: str, url: str) -> dict: return result def convert_html_to_markdown(html: str) -> str: + logging.info("Converting HTML to Markdown") soup = BeautifulSoup(html, 'html.parser') for para in soup.find_all('p'): # Add a newline at the end of each paragraph for markdown separation @@ -145,9 +151,11 @@ def convert_html_to_markdown(html: str) -> str: article_data = extract_article_data(html, url) if article_data['extraction_successful']: article_data['content'] = convert_html_to_markdown(article_data['content']) + logging.info(f"Article content length: {len(article_data['content'])}") log_histogram("article_content_length", len(article_data['content']), labels={"url": url}) return article_data + # FIXME - Add keyword integration/tagging async def scrape_and_summarize_multiple( urls: str, From 7d019b3616b79eddeec199caecc3b88a7dfb01c9 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 22:26:04 -0800 Subject: [PATCH 79/88] meh, i'll take it. --- .../Gradio_UI/WebSearch_tab.py | 355 ++++++++---------- App_Function_Libraries/Utils/Utils.py | 4 + 2 files changed, 164 insertions(+), 195 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py index bdda69386..b99d4d686 100644 --- a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -4,9 +4,13 @@ # Imports import asyncio import logging +from typing import Dict + # # External Imports import gradio as gr + +from App_Function_Libraries.Utils.Utils import loaded_config_data # # Local Imports from App_Function_Libraries.Web_Scraping.WebSearch_APIs import generate_and_search, analyze_and_aggregate @@ -14,23 +18,13 @@ ######################################################################################################################## # # Functions: - def create_websearch_tab(): - with gr.TabItem("Web Search & Review", visible=True): - with gr.Blocks() as perplexity_interface: - # Add CSS + with gr.TabItem("Web Search & Review"): + with gr.Blocks() as interface: + search_state = gr.State(value=None) + # Basic styling gr.HTML(""" """) - gr.Markdown("# Web Search Interface") - - # State for managing the review process - state = gr.State({ - "phase1_results": None, - "search_params": None, - "selected_indices": [] - }) - + # Input Section with gr.Row(): with gr.Column(): - # Input components - question_input = gr.Textbox( - label="Enter your question", - placeholder="What would you like to know?", + query = gr.Textbox( + label="Search Query", + placeholder="What would you like to search for?", lines=2 ) with gr.Row(): - search_engine = gr.Dropdown( + engine = gr.Dropdown( choices=["google", "bing", "duckduckgo", "brave"], value="google", label="Search Engine" ) - result_count = gr.Slider( - minimum=1, - maximum=20, - value=10, - step=1, + num_results = gr.Slider( + minimum=1, maximum=20, value=10, step=1, label="Number of Results" ) with gr.Row(): - content_country = gr.Dropdown( + country = gr.Dropdown( choices=["US", "UK", "CA", "AU"], value="US", - label="Content Country" + label="Content Region" ) - search_lang = gr.Dropdown( + language = gr.Dropdown( choices=["en", "es", "fr", "de"], value="en", - label="Search Language" + label="Language" ) - # Status and progress displays + # Action Buttons and Status with gr.Row(): search_btn = gr.Button("Search", variant="primary") - status_display = gr.Markdown("Ready", elem_classes=["status-display", "status-normal"]) - progress_display = gr.HTML(visible=False) - - # Results review section - with gr.Column(visible=False) as review_column: - gr.Markdown("### Search Results") - results_container = gr.HTML() # Container for results - confirm_selection_btn = gr.Button("Generate Answer from Selected Results") - - # Final output section - with gr.Column(visible=False) as output_column: - answer_output = gr.Markdown(label="Generated Answer") - sources_output = gr.JSON(label="Sources") - - def update_status(message, status_type="normal"): - """Update the status display with the given message and type.""" - status_classes = { - "normal": "status-normal", - "processing": "status-processing", - "error": "status-error", - "success": "status-success" - } - return ( - gr.Markdown(value=message, elem_classes=["status-display", status_classes[status_type]]), - gr.HTML(visible=(status_type == "processing")) - ) + status = gr.Markdown("Ready") + + # Results Section + results_display = gr.HTML(visible=False) + analyze_btn = gr.Button("Analyze Selected Results", visible=False) + + # Final Output Section + with gr.Column(visible=False) as output_section: + answer = gr.Markdown(label="Analysis") + sources = gr.JSON(label="Sources") - def format_results_html(results): - """Format search results as HTML with checkboxes.""" + def format_results(results: list) -> str: + """Format search results as HTML.""" html = "" for idx, result in enumerate(results): html += f"""
-
- -
+
{result.get('title', 'No title')}
{result.get('url', 'No URL')}
{result.get('content', 'No content')[:200]}...
@@ -148,137 +98,152 @@ def format_results_html(results): """ return html - def initial_search(question, engine, count, country, lang, state): + relevance_analysis_llm = loaded_config_data['search_settings']["relevance_analysis_llm"] + final_answer_llm = loaded_config_data['search_settings']["final_answer_llm"] + + def perform_search(query: str, engine: str, num_results: int, + country: str, language: str) -> Dict: + """Execute the search operation.""" + search_params = { + "engine": engine, + "content_country": country, + "search_lang": language, + "output_lang": language, + "result_count": num_results, + # Add LLM settings + "relevance_analysis_llm": relevance_analysis_llm, + "final_answer_llm": final_answer_llm + } + + return generate_and_search(query, search_params) + + def search_handler(query, engine, num_results, country, language): try: - status, progress = update_status("Initializing search...", "processing") - yield status, progress, state, "", gr.Column(visible=False), gr.Column(visible=False) + # Call perform_search with individual arguments + results = perform_search( + query=query, + engine=engine, + num_results=num_results, + country=country, + language=language + ) - search_params = { - "engine": engine, - "content_country": country, - "search_lang": lang, - "output_lang": lang, - "result_count": count + logging.debug(f"Search results: {results}") + + if not results.get("web_search_results_dict") or not results["web_search_results_dict"].get( + "results"): + raise ValueError("No search results returned") + + results_html = format_results(results["web_search_results_dict"]["results"]) + + # Store complete results including search params + state_to_store = { + "web_search_results_dict": results["web_search_results_dict"], + "sub_query_dict": results.get("sub_query_dict", {}), + "search_params": { + "engine": engine, + "content_country": country, + "search_lang": language, + "output_lang": language, + "relevance_analysis_llm": relevance_analysis_llm, + "final_answer_llm": final_answer_llm + } } - # Generate and search - phase1_results = generate_and_search(question, search_params) - - # Get results list - results_list = phase1_results["web_search_results_dict"]["results"] + logging.info( + f"Storing state with {len(state_to_store['web_search_results_dict']['results'])} results") - # Format results as HTML - results_html = format_results_html(results_list) + return ( + gr.Markdown("Search completed successfully"), + gr.HTML(results_html, visible=True), + gr.Button(visible=True), + gr.Column(visible=False), + state_to_store + ) + except Exception as e: + logging.error(f"Search error: {str(e)}", exc_info=True) + return ( + gr.Markdown(f"Error: {str(e)}"), + gr.HTML(visible=False), + gr.Button(visible=False), + gr.Column(visible=False), + None + ) - # Update state with the list indices - state = { - "phase1_results": phase1_results, - "search_params": search_params, - "selected_indices": list(range(len(results_list))) - } + async def analyze_handler(state): + logging.debug(f"Received state for analysis: {state}") + try: + if not state or not isinstance(state, dict): + raise ValueError(f"Invalid state received: {state}") - logging.info(f"Search completed. Results count: {len(results_list)}") - logging.info(f"Selected indices: {state['selected_indices']}") + if not state.get("web_search_results_dict"): + raise ValueError("No web search results in state") - status, progress = update_status("Search completed successfully!", "success") - yield status, progress, state, results_html, gr.Column(visible=True), gr.Column(visible=False) + if not state["web_search_results_dict"].get("results"): + raise ValueError("No results array in web search results") - except Exception as e: - error_message = f"Error during search: {str(e)}" - logging.error(f"Search error: {error_message}") - logging.error("Traceback: ", exc_info=True) - status, progress = update_status(error_message, "error") - yield status, progress, state, "", gr.Column(visible=False), gr.Column(visible=False) + relevance_analysis_llm = loaded_config_data['search_settings']["relevance_analysis_llm"] + final_answer_llm = loaded_config_data['search_settings']["final_answer_llm"] - def generate_final_answer(state): - try: - status, progress = update_status("Generating final answer...", "processing") - yield status, progress, "Processing...", {}, gr.Column(visible=False) - - if not state["phase1_results"] or not state["search_params"]: - raise ValueError("No search results available") - - # Get selected results - filtered_results = {} - web_search_results = state["phase1_results"]["web_search_results_dict"]["results"] - - logging.info(f"Processing web search results: {web_search_results}") - logging.info(f"Selected indices: {state['selected_indices']}") - - # Convert list results to dictionary format expected by analyze_and_aggregate - for idx in state["selected_indices"]: - if idx < len(web_search_results): - result = web_search_results[idx] - filtered_results[str(idx)] = { - 'content': result.get('content', ''), - 'url': result.get('url', ''), - 'title': result.get('title', ''), - 'metadata': result.get('metadata', {}) - } - - if not filtered_results: - raise ValueError("No results selected") - - logging.info(f"Filtered results prepared for analysis: {filtered_results}") - - # Create the input structure expected by analyze_and_aggregate - input_data = { - "results": filtered_results + # Create search params with required LLM settings + search_params = { + "engine": state["web_search_results_dict"]["search_engine"], + "content_country": state["web_search_results_dict"]["content_country"], + "search_lang": state["web_search_results_dict"]["search_lang"], + "output_lang": state["web_search_results_dict"]["output_lang"], + # Add LLM settings + "relevance_analysis_llm": relevance_analysis_llm, + "final_answer_llm": final_answer_llm } - # Generate final answer - phase2_results = asyncio.run(analyze_and_aggregate( - input_data, - state["phase1_results"].get("sub_query_dict", {}), - state["search_params"] - )) + # Analyze results + analysis = await analyze_and_aggregate( + state["web_search_results_dict"], + state.get("sub_query_dict", {}), + search_params + ) + + logging.debug(f"Analysis results: {analysis}") + + if not analysis.get("final_answer"): + raise ValueError("Analysis did not produce a final answer") - status, progress = update_status("Answer generated successfully!", "success") - yield status, progress, phase2_results["final_answer"]["Report"], phase2_results["final_answer"][ - "evidence"], gr.Column(visible=True) + formatted_answer = f""" + ### Analysis Result + {analysis["final_answer"]["Report"]} + ### Sources + """ + + return ( + gr.Markdown("Analysis completed successfully"), + gr.Markdown(formatted_answer), + analysis["final_answer"]["evidence"], + gr.Column(visible=True) + ) except Exception as e: - error_message = f"Error generating answer: {str(e)}" - logging.error(f"Error in generate_final_answer: {error_message}") - logging.error(f"State: {state}") - logging.error(f"Traceback: ", exc_info=True) # Add full traceback - status, progress = update_status(error_message, "error") - yield status, progress, "Error occurred while generating answer", {}, gr.Column(visible=False) + logging.error(f"Analysis error: {str(e)}", exc_info=True) + return ( + gr.Markdown(f"Error during analysis: {str(e)}"), + gr.Markdown("Analysis failed"), + {}, + gr.Column(visible=False) + ) # Connect event handlers search_btn.click( - fn=initial_search, - inputs=[ - question_input, - search_engine, - result_count, - content_country, - search_lang, - state - ], - outputs=[ - status_display, - progress_display, - state, - results_container, - review_column, - output_column - ] + fn=search_handler, + inputs=[query, engine, num_results, country, language], + outputs=[status, results_display, analyze_btn, output_section, search_state] # Use search_state ) - confirm_selection_btn.click( - fn=generate_final_answer, - inputs=[state], - outputs=[ - status_display, - progress_display, - answer_output, - sources_output, - output_column - ] + analyze_btn.click( + fn=analyze_handler, + inputs=[search_state], # Use search_state as input + outputs=[status, answer, sources, output_section] ) - return perplexity_interface + return interface # # End of File diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index 5c222c485..fcaf707c7 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -420,6 +420,8 @@ def load_and_log_configs(): search_result_display_metadata = config.get('Search-Engines', 'search_result_display_metadata', fallback='False') search_result_save_to_db = config.get('Search-Engines', 'search_result_save_to_db', fallback='True') search_result_analysis_tone = config.get('Search-Engines', 'search_result_analysis_tone', fallback='') + relevance_analysis_llm = config.get('Search-Engines', 'relevance_analysis_llm', fallback='False') + final_answer_llm = config.get('Search-Engines', 'final_answer_llm', fallback='False') # Search Engine Specifics baidu_search_api_key = config.get('Search-Engines', 'search_engine_api_key_baidu', fallback='') # Bing Search Settings @@ -667,6 +669,8 @@ def load_and_log_configs(): 'search_result_display_metadata': search_result_display_metadata, 'search_result_save_to_db': search_result_save_to_db, 'search_result_analysis_tone': search_result_analysis_tone, + 'relevance_analysis_llm': relevance_analysis_llm, + 'final_answer_llm': final_answer_llm, }, 'search_engines': { 'baidu_search_api_key': baidu_search_api_key, From e3d2d447d62cf856a4cac1157be95b6f2b093511 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 22:39:00 -0800 Subject: [PATCH 80/88] w.o.w. --- .../Summarization/Summarization_General_Lib.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/App_Function_Libraries/Summarization/Summarization_General_Lib.py b/App_Function_Libraries/Summarization/Summarization_General_Lib.py index b42f98dbd..b3765dce7 100644 --- a/App_Function_Libraries/Summarization/Summarization_General_Lib.py +++ b/App_Function_Libraries/Summarization/Summarization_General_Lib.py @@ -126,9 +126,9 @@ def summarize_with_openai(api_key, input_data, custom_prompt_arg, temp=None, sys try: # API key validation if not api_key or api_key.strip() == "": - logging.info("OpenAI: #1 API key not provided as parameter") - logging.info("OpenAI: Attempting to use API key from config file") - api_key = loaded_config_data['openai_api']['api_Key'] + logging.info("OpenAI Summarize: API key not provided as parameter") + logging.info("OpenAI Summarize: Attempting to use API key from config file") + loaded_config_data.get('openai_api', {}).get('api_key', "") if not api_key or api_key.strip() == "": logging.error("OpenAI: #2 API key not found or is empty") From 409de884a8173b0ad810c97cc78b96589e460adc Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 22:57:50 -0800 Subject: [PATCH 81/88] dollar store perplexity. --- App_Function_Libraries/Chat/Chat_Functions.py | 4 +- .../Gradio_UI/WebSearch_tab.py | 118 ++++++++++++++---- .../Summarization_General_Lib.py | 4 +- 3 files changed, 103 insertions(+), 23 deletions(-) diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index 91b6bdbda..d099c7e43 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -39,7 +39,9 @@ def approximate_token_count(history): total_tokens = len(total_text.split()) return total_tokens -def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming): + +# FIXME - add model parameter +def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming, model=None): logging.info(f"Debug - Chat API Call - API Endpoint: {api_endpoint}") log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py index b99d4d686..7c1e14ed0 100644 --- a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -24,17 +24,33 @@ def create_websearch_tab(): search_state = gr.State(value=None) # Basic styling gr.HTML(""" - """) @@ -77,7 +93,18 @@ def create_websearch_tab(): # Results Section results_display = gr.HTML(visible=False) - analyze_btn = gr.Button("Analyze Selected Results", visible=False) + # Analysis button and status container + with gr.Row() as analyze_container: + analyze_btn = gr.Button("Analyze Selected Results", visible=False) + analysis_status = gr.HTML( + """ +
+ + +
+ """, + visible=False + ) # Final Output Section with gr.Column(visible=False) as output_section: @@ -157,6 +184,7 @@ def search_handler(query, engine, num_results, country, language): gr.Markdown("Search completed successfully"), gr.HTML(results_html, visible=True), gr.Button(visible=True), + gr.HTML(visible=True), gr.Column(visible=False), state_to_store ) @@ -166,6 +194,7 @@ def search_handler(query, engine, num_results, country, language): gr.Markdown(f"Error: {str(e)}"), gr.HTML(visible=False), gr.Button(visible=False), + gr.HTML(visible=False), gr.Column(visible=False), None ) @@ -173,6 +202,21 @@ def search_handler(query, engine, num_results, country, language): async def analyze_handler(state): logging.debug(f"Received state for analysis: {state}") try: + yield ( + gr.HTML( + """ +
+ + Processing results... +
+ """, + visible=True + ), + gr.Markdown("Analysis in progress..."), + gr.JSON(None), + gr.Column(visible=False) + ) + if not state or not isinstance(state, dict): raise ValueError(f"Invalid state received: {state}") @@ -200,7 +244,7 @@ async def analyze_handler(state): analysis = await analyze_and_aggregate( state["web_search_results_dict"], state.get("sub_query_dict", {}), - search_params + state.get("search_params", {}) ) logging.debug(f"Analysis results: {analysis}") @@ -215,32 +259,64 @@ async def analyze_handler(state): ### Sources """ - return ( - gr.Markdown("Analysis completed successfully"), + yield ( + gr.HTML( + """ +
+ ✓ Analysis complete +
+ """, + visible=True + ), gr.Markdown(formatted_answer), analysis["final_answer"]["evidence"], gr.Column(visible=True) ) except Exception as e: logging.error(f"Analysis error: {str(e)}", exc_info=True) - return ( - gr.Markdown(f"Error during analysis: {str(e)}"), + yield ( + gr.HTML( + f""" +
+ ❌ Error: {str(e)} +
+ """, + visible=True + ), gr.Markdown("Analysis failed"), - {}, + gr.JSON(None), gr.Column(visible=False) ) # Connect event handlers search_btn.click( fn=search_handler, - inputs=[query, engine, num_results, country, language], - outputs=[status, results_display, analyze_btn, output_section, search_state] # Use search_state + inputs=[ + query, + engine, + num_results, + country, + language, + ], + outputs=[ + status, + results_display, + analyze_btn, + analysis_status, + output_section, + search_state # Update state + ] ) analyze_btn.click( fn=analyze_handler, - inputs=[search_state], # Use search_state as input - outputs=[status, answer, sources, output_section] + inputs=[search_state], # Use the state + outputs=[ + analysis_status, + answer, + sources, + output_section + ] ) return interface diff --git a/App_Function_Libraries/Summarization/Summarization_General_Lib.py b/App_Function_Libraries/Summarization/Summarization_General_Lib.py index b3765dce7..feeb1e9c0 100644 --- a/App_Function_Libraries/Summarization/Summarization_General_Lib.py +++ b/App_Function_Libraries/Summarization/Summarization_General_Lib.py @@ -122,13 +122,15 @@ def extract_text_from_segments(segments): def summarize_with_openai(api_key, input_data, custom_prompt_arg, temp=None, system_message=None, streaming=False): - loaded_config_data = load_and_log_configs() + try: # API key validation if not api_key or api_key.strip() == "": logging.info("OpenAI Summarize: API key not provided as parameter") logging.info("OpenAI Summarize: Attempting to use API key from config file") + loaded_config_data = load_and_log_configs() loaded_config_data.get('openai_api', {}).get('api_key', "") + logging.debug(f"OpenAI Summarize: Using API key from config file: {api_key[:5]}...{api_key[-5:]}") if not api_key or api_key.strip() == "": logging.error("OpenAI: #2 API key not found or is empty") From 3337ed18807c20df11ad16fbfaf6def6d353e617 Mon Sep 17 00:00:00 2001 From: Robert Date: Thu, 2 Jan 2025 23:30:48 -0800 Subject: [PATCH 82/88] Update WebSearch.md --- Docs/Design/WebSearch.md | 169 ++++++++++++++------------------------- 1 file changed, 62 insertions(+), 107 deletions(-) diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 22b68a807..183116f3f 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -17,6 +17,7 @@ This page serves as documentation regarding the web search functionality within - Bing, Brave, DDG, Google work for simple searches. Advanced search options are not fully working yet. - Brave: https://api.search.brave.com/app/documentation/web-search/query#WebSearchAPIQueryParameters - Bing: https://docs.microsoft.com/en-us/rest/api/cognitiveservices-bingsearch/bing-web-api-v7-reference + - DuckDuckGo: https://duckduckgo.com/ - Google: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list - Baidu, SearX, Serper, Tavily, Yandex are not implemented yet. - Kagi & SearX are implemented but not working (Kagi because API access and SearX because I'm not sure why) @@ -95,117 +96,71 @@ This page serves as documentation regarding the web search functionality within ---------------- ### Web Search API - TBD -```def perform_websearch(search_engine, - search_query, - content_country, - search_lang, - output_lang, - result_count, - date_range=None, - safesearch=None, - site_blacklist=None, - exactTerms=None, - excludeTerms=None, - filter=None, - geolocation=None, - search_result_language=None, - sort_results_by=None ``` +def perform_websearch(search_engine + search_query, + content_country, + search_lang, + output_lang, + result_count, - Number of results to return(not applicable to all engines, but necessary for consistency) + date_range=None, + safesearch=None, + site_blacklist=None, + exactTerms=None, + excludeTerms=None, + filter=None, + geolocation=None, + search_result_language=None + sort_results_by=None): +``` + -ask.py https://github.com/pengfeng/ask.py - 1. User inputs a search query - 2. Search query is then used to generate several sub-queries - 3. Each sub-query is then used to perform a search via API call - 4. Sub-queries are setup using a specific prompt, they do 'at least 3, but no more than 5' - makes me wonder about how many sub-queries should be the default and the range thereof - 5. Search results returned, passed to groq - - -OpenPerplex - 1. Search Query - 2. Perform search via API - 3. Collect results, grab story titles + snippets - 4. Extract content from each URL - 5. Concat if exceeds X length, re-rank, then combine into a single document - 6. Present final doc to user - - - -llm-websearch - https://github.com/Jay4242/llm-websearch - 1. Performs query -> Fetches results - 2. Iterates over each page description and decides whether to 'investigate' the page's content - 3. If so, it fetches the page content and creates a summary of it - 4. Repeat, until all URLs are processed. - 5. Create summary of all summaries/page contents and present final doc to user - - - -mindsearch - Uses Knowledge Graph for results - Interesting, seems to combat repetition in search results - top_p=0.8, - top_k=1, - temperature=0, - max_new_tokens=8192, - repetition_penalty=1.02, - - - -Perplexity - 1. Search Query - 2. Perform search via API - 3. Collect results, grab story titles + snippets - 4. Extract content from each URL - 5. Concat if exceeds X length, re-rank, then combine into a single document - 6. Present final doc to user - 7. Display top 3 sources at top, with option to show/hide remaining sources - can click on sources to go that URL - 8. Option for Follow up questions + Potential other questions, if these are selected, the content is added to the 'ongoing' document - 9. Ongoing document is saved to the DB, and can be accessed later - - Option for ranking/share/rewrite/copy to clipboard - - Pro allows for seemingly 'planning'/sub-question breakdown of the user's original query, maybe just asking 'what would be questions likely asked?'? - - Performs a planning step and then sub-query breakdown searches into 3(?) sub-queries - - Non-pro simply performs the search, combines the results, and presents them to the user while showing the sources - - Spaces: Seems to be a way of sharing collections of research document chains, with the ability to add comments, etc. - - - -### Implementaiton -- Configuration Options: - - Language - - Language used for writing query - - language used for performing query (query submitted to search engine) - - language used for summary/analysis of search results - - Query count - - Whether to create Sub-Queries - - Sub-Query count - - Search Engine Selection - - Search Engine API Key - - Search Engine Customization Options - - Safe Search - - Language - - Date Range - - Search Result Options - - Total number of queries to perform - - Total number of results/pages to review per query - - Result Sorting - Auto rerank according to ? - - Result Filtering - Filter according to a blacklist of URLs? Maybe also content? - - Search Result Display Options - - Display Format (List, Briefing, Full) - - Display Metadata (URL, Date, etc) - - Search Result Default Saving Options - - Save to DB - - Save to File - - Save to Clipboard - - Save to Notes DB - Create a new note with the search results + metadata & query - - Output Options - - Default Output Format (Markdown, HTML, PDF) - ehhhh - - Default Output Location (Display/ephemeral, DB) - - Output style (Briefing, Full, List) - - Output Metadata (URL, Date, etc) - - Word count limit for output (per search result & total) - - Output dialect (US, UK, a NY Radio DJ, etc) +### Implementation +config.txt options explained +``` +# Search Defaults +search_provider_default = google +search_language_query = en - Language Queries will be performed in +search_language_results = en - Language Results will be returned in +search_language_analysis = en - Language Analysis will be performed in +search_default_max_queries = 10 - Maximum number of queries to perform +search_enable_subquery = True - Enable subqueries +search_enable_subquery_count_max = 5 - Maximum number of subqueries to generate +search_result_rerank = True - Enable result reranking +search_result_max = 15 - Maximum number of results to return +search_result_max_per_query = 10 - Maximum number of results to return per query +search_result_blacklist = [] +search_result_display_type = list - Display type for search results, does nothing right now. +search_result_display_metadata = False - Display metadata for search results, does nothing right now. +search_result_save_to_db = True - Save search results to the database (not implemented yet) +# How you want the results to be written, think 'style' or voice +search_result_analysis_tone = neutral - Tone of the analysis (not implemented yet) +relevance_analysis_llm = openai - LLM to use for relevance analysis +final_answer_llm = openai - LLM to use for final answer generation +#### Search Engines ##### +# Bing +search_engine_country_code_bing = en - Country code for Bing, Where Search 'takes place from' +# +# Brave +search_engine_country_code_brave = US - Country code for Brave, Where Search 'takes place from' +# +# Google +# Restricts search results to documents originating in a particular country. +limit_google_search_to_country = False +google_search_country_code = US - Country code for Google, Where Search 'takes place from' +google_filter_setting = 1 - Filter setting for Google, 0 = No filtering, 1 = Moderate filtering, 2 = Strict filtering +google_user_geolocation = US - Geolocation for user performing the search +google_limit_search_results_to_language = False - Limit search results to a specific language +google_default_search_results = 10 - Default number of search results to return +google_safe_search = "active" - Safe search setting for Google, active, moderate, or off +google_enable_site_search = False - Enable site search +google_site_search_include = - Sites to include in the search +google_site_search_exclude = - Sites to exclude from the search +# https://developers.google.com/custom-search/docs/structured_search#sort-by-attribute +google_sort_results_by = - Sort results by attribute (I honestly couldn't find much about this one) +``` Results dictionary: ``` From b8931d3af6840f572bc7d95e36791a78f85dad05 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 3 Jan 2025 18:57:23 -0800 Subject: [PATCH 83/88] closer --- .../Confabulation_check.py | 9 +- .../Benchmarks_Evaluations/ms_g_eval.py | 2 +- .../Character_Chat/Character_Chat_Lib.py | 14 +- App_Function_Libraries/Chat/Chat_Functions.py | 11 +- .../DB/Character_Chat_DB.py | 25 +- .../Gradio_UI/Character_Chat_tab.py | 1792 +++++++++-------- .../Gradio_UI/WebSearch_tab.py | 73 +- App_Function_Libraries/LLM_API_Calls.py | 44 +- .../Prompt_Engineering/Prompt_Engineering.py | 4 +- App_Function_Libraries/TTS/TTS_Providers.py | 22 +- App_Function_Libraries/Utils/Utils.py | 4 - .../Web_Scraping/WebSearch_APIs.py | 2 +- Docs/Design/Creative_Writing.md | 2 +- Docs/Design/TTS_STT.md | 5 + Docs/Design/Text2SQL.md | 3 + Docs/Design/UX.md | 2 + Docs/Design/VLMs.md | 4 + 17 files changed, 1070 insertions(+), 948 deletions(-) diff --git a/App_Function_Libraries/Benchmarks_Evaluations/Confabulation_check.py b/App_Function_Libraries/Benchmarks_Evaluations/Confabulation_check.py index e2fc127b6..73ed98fd4 100644 --- a/App_Function_Libraries/Benchmarks_Evaluations/Confabulation_check.py +++ b/App_Function_Libraries/Benchmarks_Evaluations/Confabulation_check.py @@ -59,7 +59,8 @@ def simplified_geval(transcript: str, summary: str, api_name: str, api_key: str, Overall Assessment: [Your overall assessment of the summary's quality] """ - + # FIXME - Add g_eval_model to config.txt + # g_eval_model = loaded_config[][] try: result = chat_api_call( api_name, @@ -67,7 +68,11 @@ def simplified_geval(transcript: str, summary: str, api_name: str, api_key: str, prompt, "", temp=temp, - system_message="You are a helpful AI assistant tasked with evaluating summaries." + system_message="You are a helpful AI assistant tasked with evaluating summaries.", + streaming=False, + minp=None, + maxp=None, + model=None ) except Exception as e: return detailed_api_error(api_name, e) diff --git a/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py b/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py index a6c7651d3..2367f875a 100644 --- a/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py +++ b/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py @@ -329,7 +329,7 @@ def geval_summarization( temp = 0.7 logging.info(f"Debug - geval_summarization Function - API Endpoint: {api_endpoint}") try: - response = chat_api_call(api_endpoint, api_key, prompt_with_src_and_gen, "", temp, system_message) + response = chat_api_call(api_endpoint, api_key, prompt_with_src_and_gen, "", temp, system_message, streaming=False, minp=None, maxp=None, model=None) except Exception as e: raise ValueError(f"Unsupported API endpoint: {api_endpoint}") except RetryError: diff --git a/App_Function_Libraries/Character_Chat/Character_Chat_Lib.py b/App_Function_Libraries/Character_Chat/Character_Chat_Lib.py index 4546b4344..c57c73c61 100644 --- a/App_Function_Libraries/Character_Chat/Character_Chat_Lib.py +++ b/App_Function_Libraries/Character_Chat/Character_Chat_Lib.py @@ -87,13 +87,25 @@ def extract_character_id(choice: str) -> int: """Extract the character ID from the dropdown selection string.""" log_counter("extract_character_id_attempt") try: - character_id = int(choice.split('(ID: ')[1].rstrip(')')) + logging.debug(f"Choice received: {choice}") # Debugging line + if not choice: + raise ValueError("No choice provided.") + + if '(ID: ' not in choice or ')' not in choice: + raise ValueError(f"Invalid choice format: {choice}") + + # Extract the ID part + id_part = choice.split('(ID: ')[1] + character_id = int(id_part.rstrip(')')) + + logging.debug(f"Extracted character ID: {character_id}") # Debugging line log_counter("extract_character_id_success") return character_id except Exception as e: log_counter("extract_character_id_error", labels={"error": str(e)}) raise + def load_character_wrapper(character_id: int, user_name: str) -> Tuple[Dict[str, Any], List[Tuple[Optional[str], str]], Optional[Image.Image]]: """Wrapper function to load character and image using the extracted ID.""" log_counter("load_character_wrapper_attempt") diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index d099c7e43..4dd09e804 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -41,7 +41,7 @@ def approximate_token_count(history): # FIXME - add model parameter -def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming, model=None): +def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming, minp=None, maxp=None, model=None): logging.info(f"Debug - Chat API Call - API Endpoint: {api_endpoint}") log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() @@ -53,7 +53,7 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag logging.info(f"Debug - Chat API Call - API Key: {api_key}") logging.info(f"Debug - Chat chat_api_call - API Endpoint: {api_endpoint}") if api_endpoint.lower() == 'openai': - response = chat_with_openai(api_key, input_data, prompt, temp, system_message, streaming) + response = chat_with_openai(api_key, input_data, prompt, temp, system_message, streaming, minp, maxp, model) elif api_endpoint.lower() == 'anthropic': # Retrieve the model from config @@ -67,7 +67,8 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag custom_prompt_arg=prompt, max_retries=3, retry_delay=5, - system_prompt=system_message + system_prompt=system_message, + streaming=streaming, ) elif api_endpoint.lower() == "cohere": @@ -139,7 +140,7 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt, temperature, - system_message=None, streaming=False): + system_message=None, streaming=False, minp=None, maxp=None, model=None): log_counter("chat_attempt", labels={"api_endpoint": api_endpoint}) start_time = time.time() try: @@ -178,7 +179,7 @@ def chat(message, history, media_content, selected_parts, api_endpoint, api_key, logging.debug(f"Debug - Chat Function - Prompt: {prompt}") # Use the existing API request code based on the selected endpoint - response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming) + response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message, streaming, minp=None, maxp=None, model=None) if streaming: return response diff --git a/App_Function_Libraries/DB/Character_Chat_DB.py b/App_Function_Libraries/DB/Character_Chat_DB.py index 63bbac637..93114ad93 100644 --- a/App_Function_Libraries/DB/Character_Chat_DB.py +++ b/App_Function_Libraries/DB/Character_Chat_DB.py @@ -355,17 +355,20 @@ def add_character_card(card_data: Dict[str, Any]) -> Optional[int]: def get_character_cards() -> List[Dict]: """Retrieve all character cards from the database.""" - logging.debug(f"Fetching characters from DB: {chat_DB_PATH}") - conn = sqlite3.connect(chat_DB_PATH) - cursor = conn.cursor() - cursor.execute("SELECT * FROM CharacterCards") - rows = cursor.fetchall() - columns = [description[0] for description in cursor.description] - conn.close() - characters = [dict(zip(columns, row)) for row in rows] - #logging.debug(f"Characters fetched from DB: {characters}") - return characters - + try: + logging.debug(f"Fetching characters from DB: {chat_DB_PATH}") + conn = sqlite3.connect(chat_DB_PATH) + cursor = conn.cursor() + cursor.execute("SELECT * FROM CharacterCards") + rows = cursor.fetchall() + columns = [description[0] for description in cursor.description] + conn.close() + characters = [dict(zip(columns, row)) for row in rows] + logging.debug(f"Characters fetched from DB: {characters}") + return characters + except Exception as e: + logging.error(f"Error fetching character cards: {e}") + return [] def get_character_card_by_id(character_id: Union[int, Dict[str, Any]]) -> Optional[Dict[str, Any]]: """ diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index 83c6e57e0..ab2285514 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -2,6 +2,7 @@ # Description: Library for character card import functions # # Imports +import os from datetime import datetime import re import tempfile @@ -34,7 +35,7 @@ delete_character_card, update_character_card, search_character_chats, save_chat_history_to_character_db, ) -from App_Function_Libraries.TTS.TTS_Providers import generate_audio +from App_Function_Libraries.TTS.TTS_Providers import generate_audio, play_mp3, play_audio_file from App_Function_Libraries.Utils.Utils import sanitize_user_input, format_api_name, global_api_endpoints, \ default_api_endpoint, load_comprehensive_config @@ -267,958 +268,995 @@ def create_character_card_interaction_tab(): logging.error(f"Error setting default API endpoint: {str(e)}") default_value = None with gr.TabItem("Chat with a Character Card", visible=True): - gr.Markdown("# Chat with a Character Card") - with gr.Row(): - with gr.Column(scale=1): - # Checkbox to Decide Whether to Save Chats by Default - config = load_comprehensive_config() - auto_save_value = config.get('auto-save', 'save_character_chats', fallback='False') - auto_save_checkbox = gr.Checkbox(label="Save chats automatically", value=auto_save_value) - chat_media_name = gr.Textbox(label="Custom Chat Name (optional)", visible=True) - save_chat_history_to_db = gr.Button("Save Chat History to Database") - save_status = gr.Textbox(label="Status", interactive=False) - with gr.Column(scale=2): - gr.Markdown("## Search and Load Existing Chats") - chat_search_query = gr.Textbox( - label="Search Chats", - placeholder="Enter chat name or keywords to search" - ) - chat_search_button = gr.Button("Search Chats") - chat_search_dropdown = gr.Dropdown(label="Search Results", choices=[], visible=False) - load_chat_button = gr.Button("Load Selected Chat", visible=False) - - with gr.Row(): - with gr.Column(scale=1): - character_image = gr.Image(label="Character Image", type="pil") - character_card_upload = gr.File( - label="Upload Character Card (PNG, WEBP, JSON)", - file_types=[".png", ".webp", ".json"] - ) - import_card_button = gr.Button("Import Character Card") - load_characters_button = gr.Button("Load Existing Characters") - character_dropdown = gr.Dropdown(label="Select Character", choices=[]) - user_name_input = gr.Textbox(label="Your Name", placeholder="Enter your name here") - # Refactored API selection dropdown - api_name_input = gr.Dropdown( - choices=["None"] + [format_api_name(api) for api in global_api_endpoints], - value=default_value, - label="API for Interaction (Mandatory)" - ) - api_key_input = gr.Textbox( - label="API Key (if not set in Config_Files/config.txt)", - placeholder="Enter your API key here", type="password" - ) - temperature_slider = gr.Slider( - minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature" - ) - chat_file_upload = gr.File(label="Upload Chat History JSON", visible=True) - import_chat_button = gr.Button("Import Chat History") + with gr.Blocks() as interface: + gr.Markdown("# Chat with a Character Card") + with gr.Row(): + with gr.Column(scale=1): + # Checkbox to Decide Whether to Save Chats by Default + config = load_comprehensive_config() + auto_save_value = config.get('auto-save', 'save_character_chats', fallback='False') + auto_save_checkbox = gr.Checkbox(label="Save chats automatically", value=auto_save_value) + chat_media_name = gr.Textbox(label="Custom Chat Name (optional)", visible=True) + save_chat_history_to_db = gr.Button("Save Chat History to Database") + save_status = gr.Textbox(label="Status", interactive=False) + with gr.Column(scale=2): + gr.Markdown("## Search and Load Existing Chats") + chat_search_query = gr.Textbox( + label="Search Chats", + placeholder="Enter chat name or keywords to search" + ) + chat_search_button = gr.Button("Search Chats") + chat_search_dropdown = gr.Dropdown(label="Search Results", choices=[], visible=False) + load_chat_button = gr.Button("Load Selected Chat", visible=False) + + with gr.Row(): + with gr.Column(scale=1): + character_image = gr.Image(label="Character Image", type="pil") + character_card_upload = gr.File( + label="Upload Character Card (PNG, WEBP, JSON)", + file_types=[".png", ".webp", ".json"] + ) + import_card_button = gr.Button("Import Character Card") + load_characters_button = gr.Button("Load Existing Characters") + character_dropdown = gr.Dropdown(label="Select Character", choices=[]) + user_name_input = gr.Textbox(label="Your Name", placeholder="Enter your name here") + # Refactored API selection dropdown + api_name_input = gr.Dropdown( + choices=["None"] + [format_api_name(api) for api in global_api_endpoints], + value=default_value, + label="API for Interaction (Mandatory)" + ) + api_key_input = gr.Textbox( + label="API Key (if not set in Config_Files/config.txt)", + placeholder="Enter your API key here", type="password" + ) + temperature_slider = gr.Slider( + minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature" + ) + with gr.Row(): + minp_slider = gr.Slider( + minimum=0.0, maximum=1.0, value=0.00, step=0.01, label="Min-P" + ) + with gr.Row(): + maxp_slider = gr.Slider( + minimum=0.00, maximum=1.0, value=1.00, step=0.01, label="Top-P" + ) + chat_file_upload = gr.File(label="Upload Chat History JSON", visible=True) + import_chat_button = gr.Button("Import Chat History") + + with gr.Column(scale=2): + chat_history = gr.Chatbot(label="Conversation", height=800) + user_input = gr.Textbox(label="Your message") + with gr.Row(): + streaming = gr.Checkbox(label="Enable streaming", value=True, interactive=True) + auto_speak_checkbox = gr.Checkbox(label="Auto-speak response", value=False, interactive=True) + send_message_button = gr.Button("Send Message") + with gr.Row(): + speak_button = gr.Button("Speak Response") + tts_status = gr.Textbox(label="TTS Status", interactive=False) + with gr.Row(): + answer_for_me_button = gr.Button("Answer for Me") + continue_talking_button = gr.Button("Continue Talking") + regenerate_button = gr.Button("Regenerate Last Message") + with gr.Row(): + token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) + clear_chat_button = gr.Button("Clear Chat") + save_snapshot_button = gr.Button("Save Chat Snapshot") + update_chat_dropdown = gr.Dropdown(label="Select Chat to Update", choices=[], visible=False) + load_selected_chat_button = gr.Button("Load Selected Chat", visible=False) + update_chat_button = gr.Button("Update Selected Chat", visible=False) + + # States + character_data = gr.State(None) + user_name = gr.State("") + selected_chat_id = gr.State(None) # To track the selected chat for updates + + # Callback Functions + + def search_existing_chats(query): + logging.info(f"Searching for chats with query: {query}") + results, message = search_character_chats(query) + if results: + # Format search results for dropdown + formatted_results = [ + f"{chat['conversation_name']} (ID: {chat['id']})" for chat in results + ] + else: + formatted_results = [] + return formatted_results, message - with gr.Column(scale=2): - chat_history = gr.Chatbot(label="Conversation", height=800) - user_input = gr.Textbox(label="Your message") - send_message_button = gr.Button("Send Message") - with gr.Row(): - speak_button = gr.Button("Speak Response") - tts_status = gr.Textbox(label="TTS Status", interactive=False) - with gr.Row(): - answer_for_me_button = gr.Button("Answer for Me") - continue_talking_button = gr.Button("Continue Talking") - regenerate_button = gr.Button("Regenerate Last Message") - with gr.Row(): - token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False) - clear_chat_button = gr.Button("Clear Chat") - save_snapshot_button = gr.Button("Save Chat Snapshot") - update_chat_dropdown = gr.Dropdown(label="Select Chat to Update", choices=[], visible=False) - load_selected_chat_button = gr.Button("Load Selected Chat", visible=False) - update_chat_button = gr.Button("Update Selected Chat", visible=False) - - # States - character_data = gr.State(None) - user_name = gr.State("") - selected_chat_id = gr.State(None) # To track the selected chat for updates + def load_selected_chat_from_search(selected_chat, user_name): + logging.info(f"Loading selected chat: {selected_chat}") + if not selected_chat: + return None, [], None, "No chat selected." - # Callback Functions + try: + chat_id_match = re.search(r'\(ID:\s*(\d+)\)', selected_chat) + if not chat_id_match: + return None, [], None, "Invalid chat selection format." - def search_existing_chats(query): - results, message = search_character_chats(query) - if results: - # Format search results for dropdown - formatted_results = [ - f"{chat['conversation_name']} (ID: {chat['id']})" for chat in results - ] - else: - formatted_results = [] - return formatted_results, message + chat_id = int(chat_id_match.group(1)) - def load_selected_chat_from_search(selected_chat, user_name): - if not selected_chat: - return None, [], None, "No chat selected." + # Use the new function to load chat and character data + char_data, chat_history, img = load_chat_and_character(chat_id, user_name) - try: - chat_id_match = re.search(r'\(ID:\s*(\d+)\)', selected_chat) - if not chat_id_match: - return None, [], None, "Invalid chat selection format." + if not char_data: + return None, [], None, "Failed to load character data for the selected chat." - chat_id = int(chat_id_match.group(1)) + return char_data, chat_history, img, f"Chat '{selected_chat}' loaded successfully." + except Exception as e: + logging.error(f"Error loading selected chat: {e}") + return None, [], None, f"Error loading chat: {e}" - # Use the new function to load chat and character data - char_data, chat_history, img = load_chat_and_character(chat_id, user_name) - if not char_data: - return None, [], None, "Failed to load character data for the selected chat." + def import_chat_history(file, current_history, char_data, user_name_val): + """ + Imports chat history from a file, replacing '{{user}}' with the actual user name. - return char_data, chat_history, img, f"Chat '{selected_chat}' loaded successfully." - except Exception as e: - logging.error(f"Error loading selected chat: {e}") - return None, [], None, f"Error loading chat: {e}" + Args: + file (file): The uploaded chat history file. + current_history (list): The current chat history. + char_data (dict): The current character data. + user_name_val (str): The user's name. + Returns: + tuple: Updated chat history, updated character data, and a status message. + """ + logging.info(f"Importing chat history from file: {file.name}") + loaded_history, char_name = load_chat_history(file) + if loaded_history is None: + return current_history, char_data, "Failed to load chat history." - def import_chat_history(file, current_history, char_data, user_name_val): - """ - Imports chat history from a file, replacing '{{user}}' with the actual user name. + # Replace '{{user}}' in the loaded chat history + loaded_history = replace_user_placeholder(loaded_history, user_name_val) - Args: - file (file): The uploaded chat history file. - current_history (list): The current chat history. - char_data (dict): The current character data. - user_name_val (str): The user's name. - - Returns: - tuple: Updated chat history, updated character data, and a status message. - """ - loaded_history, char_name = load_chat_history(file) - if loaded_history is None: - return current_history, char_data, "Failed to load chat history." - - # Replace '{{user}}' in the loaded chat history - loaded_history = replace_user_placeholder(loaded_history, user_name_val) - - # Check if the loaded chat is for the current character - if char_data and char_data.get('name') != char_name: - return current_history, char_data, ( - f"Warning: Loaded chat is for character '{char_name}', " - f"but current character is '{char_data.get('name')}'. Chat not imported." - ) - - # If no character is selected, try to load the character from the chat - if not char_data: - characters = get_character_cards() - character = next((char for char in characters if char['name'] == char_name), None) - if character: - char_data = character - # Replace '{{user}}' in the first_message if necessary - if character.get('first_message'): - character['first_message'] = character['first_message'].replace("{{user}}", - user_name_val if user_name_val else "User") - else: + # Check if the loaded chat is for the current character + if char_data and char_data.get('name') != char_name: return current_history, char_data, ( - f"Warning: Character '{char_name}' not found. Please select the character manually." + f"Warning: Loaded chat is for character '{char_name}', " + f"but current character is '{char_data.get('name')}'. Chat not imported." ) - return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully." - - def load_character(name): - characters = get_character_cards() - character = next((char for char in characters if char['name'] == name), None) - if character: - first_message = character.get('first_message', "Hello! I'm ready to chat.") - return character, [(None, first_message)] if first_message else [], None - return None, [], None - - def load_character_image(name): - character = next((char for char in get_character_cards() if char['name'] == name), None) - if character and 'image' in character and character['image']: - try: - # Decode the base64 image - image_data = base64.b64decode(character['image']) - # Load as PIL Image - img = Image.open(io.BytesIO(image_data)).convert("RGBA") - return img - except Exception as e: - logging.error(f"Error loading image for character '{name}': {e}") - return None - return None - - def character_chat_wrapper( - message, history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save - ): - if not char_data: - return history, "Please select a character first." - - user_name_val = user_name_val or "User" - char_name = char_data.get('name', 'AI Assistant') - - # Prepare the character's background information - char_background = f""" - Name: {char_name} - Description: {char_data.get('description', 'N/A')} - Personality: {char_data.get('personality', 'N/A')} - Scenario: {char_data.get('scenario', 'N/A')} - """ - - # Prepare the system prompt - system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}""" - - # Prepare chat context - media_content = { - 'id': char_name, - 'title': char_name, - 'content': char_background, - 'description': char_data.get('description', ''), - 'personality': char_data.get('personality', ''), - 'scenario': char_data.get('scenario', '') - } - selected_parts = ['description', 'personality', 'scenario'] - - prompt = char_data.get('post_history_instructions', '') - - # Sanitize and format user message - user_message = sanitize_user_input(message) - user_message = replace_placeholders(user_message, char_name, user_name_val) - full_message = f"{user_name_val}: {user_message}" - - # Generate bot response - bot_message = chat( - full_message, - history, - media_content, - selected_parts, - api_endpoint, - api_key, - prompt, - temperature, - system_message - ) - - # Replace placeholders in bot message - bot_message = replace_placeholders(bot_message, char_name, user_name_val) - - # Update history - history.append((user_message, bot_message)) - - # Auto-save if enabled - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." - - return history, save_status - - def validate_chat_history(chat_history: List[Tuple[Optional[str], str]]) -> bool: - """ - Validate the chat history format and content. + # If no character is selected, try to load the character from the chat + if not char_data: + characters = get_character_cards() + character = next((char for char in characters if char['name'] == char_name), None) + if character: + char_data = character + # Replace '{{user}}' in the first_message if necessary + if character.get('first_message'): + character['first_message'] = character['first_message'].replace("{{user}}", + user_name_val if user_name_val else "User") + else: + return current_history, char_data, ( + f"Warning: Character '{char_name}' not found. Please select the character manually." + ) + + return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully." + + # def load_character(name): + # characters = get_character_cards() + # character = next((char for char in characters if char['name'] == name), None) + # if character: + # first_message = character.get('first_message', "Hello! I'm ready to chat.") + # return character, [(None, first_message)] if first_message else [], None + # return None, [], None + # + # def load_character_image(name): + # character = next((char for char in get_character_cards() if char['name'] == name), None) + # if character and 'image' in character and character['image']: + # try: + # # Decode the base64 image + # image_data = base64.b64decode(character['image']) + # # Load as PIL Image + # img = Image.open(io.BytesIO(image_data)).convert("RGBA") + # return img + # except Exception as e: + # logging.error(f"Error loading image for character '{name}': {e}") + # return None + # return None + + def character_chat_wrapper( + message, history, char_data, api_endpoint, api_key, + temperature, user_name_val, auto_save, streaming, minp, maxp + ): + if not char_data: + return history, "Please select a character first." + + user_name_val = user_name_val or "User" + char_name = char_data.get('name', 'AI Assistant') + + # Prepare the character's background information + char_background = f""" + Name: {char_name} + Description: {char_data.get('description', 'N/A')} + Personality: {char_data.get('personality', 'N/A')} + Scenario: {char_data.get('scenario', 'N/A')} + """ + + # Prepare the system prompt + system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}""" + + # Prepare chat context + media_content = { + 'id': char_name, + 'title': char_name, + 'content': char_background, + 'description': char_data.get('description', ''), + 'personality': char_data.get('personality', ''), + 'scenario': char_data.get('scenario', '') + } + selected_parts = ['description', 'personality', 'scenario'] + + prompt = char_data.get('post_history_instructions', '') + + # Sanitize and format user message + user_message = sanitize_user_input(message) + user_message = replace_placeholders(user_message, char_name, user_name_val) + full_message = f"{user_name_val}: {user_message}" + + # Generate bot response + logging.debug(f"Generating response; Calling chat function with message: {full_message}") + bot_message = chat( + full_message, + history, + media_content, + selected_parts, + api_endpoint, + api_key, + prompt, + temperature, + system_message, + streaming, + minp=minp, + maxp=maxp + ) - Args: - chat_history: List of message tuples (user_message, bot_message) + # Replace placeholders in bot message + bot_message = replace_placeholders(bot_message, char_name, user_name_val) - Returns: - bool: True if valid, False if invalid - """ - if not isinstance(chat_history, list): - return False + # Update history + history.append((user_message, bot_message)) - for entry in chat_history: - if not isinstance(entry, tuple) or len(entry) != 2: - return False - # First element can be None (for system messages) or str - if not (entry[0] is None or isinstance(entry[0], str)): - return False - # Second element (bot response) must be str and not empty - if not isinstance(entry[1], str) or not entry[1].strip(): + # Auto-save if enabled + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + + return history, save_status + + def validate_chat_history(chat_history: List[Tuple[Optional[str], str]]) -> bool: + """ + Validate the chat history format and content. + + Args: + chat_history: List of message tuples (user_message, bot_message) + + Returns: + bool: True if valid, False if invalid + """ + if not isinstance(chat_history, list): return False - return True - - def sanitize_conversation_name(name: str) -> str: - """ - Sanitize the conversation name. - - Args: - name: Raw conversation name - - Returns: - str: Sanitized conversation name - """ - # Remove any non-alphanumeric characters except spaces and basic punctuation - sanitized = re.sub(r'[^a-zA-Z0-9\s\-_.]', '', name) - # Limit length - sanitized = sanitized[:100] - # Ensure it's not empty - if not sanitized.strip(): - sanitized = f"Chat_{datetime.now().strftime('%Y%m%d_%H%M%S')}" - return sanitized - - def save_chat_history_to_db_wrapper( - chat_history: List[Tuple[Optional[str], str]], - conversation_id: str, - media_content: Dict, - chat_media_name: str, - char_data: Dict, - auto_save: bool - ) -> Tuple[str, str]: - """ - Save chat history to the database with validation. + for entry in chat_history: + if not isinstance(entry, tuple) or len(entry) != 2: + return False + # First element can be None (for system messages) or str + if not (entry[0] is None or isinstance(entry[0], str)): + return False + # Second element (bot response) must be str and not empty + if not isinstance(entry[1], str) or not entry[1].strip(): + return False + + return True + + def sanitize_conversation_name(name: str) -> str: + """ + Sanitize the conversation name. + + Args: + name: Raw conversation name + + Returns: + str: Sanitized conversation name + """ + # Remove any non-alphanumeric characters except spaces and basic punctuation + sanitized = re.sub(r'[^a-zA-Z0-9\s\-_.]', '', name) + # Limit length + sanitized = sanitized[:100] + # Ensure it's not empty + if not sanitized.strip(): + sanitized = f"Chat_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + return sanitized + + def save_chat_history_to_db_wrapper( + chat_history: List[Tuple[Optional[str], str]], + conversation_id: str, + media_content: Dict, + chat_media_name: str, + char_data: Dict, + auto_save: bool + ) -> Tuple[str, str]: + """ + Save chat history to the database with validation. + + Args: + chat_history: List of message tuples + conversation_id: Current conversation ID + media_content: Media content metadata + chat_media_name: Custom name for the chat + char_data: Character data dictionary + auto_save: Auto-save flag + + Returns: + Tuple[str, str]: (status message, detail message) + """ + try: + # Basic input validation + if not chat_history: + return "No chat history to save.", "" - Args: - chat_history: List of message tuples - conversation_id: Current conversation ID - media_content: Media content metadata - chat_media_name: Custom name for the chat - char_data: Character data dictionary - auto_save: Auto-save flag + if not validate_chat_history(chat_history): + return "Invalid chat history format.", "Please ensure the chat history is valid." - Returns: - Tuple[str, str]: (status message, detail message) - """ - try: - # Basic input validation - if not chat_history: - return "No chat history to save.", "" + if not char_data: + return "No character selected.", "Please select a character first." - if not validate_chat_history(chat_history): - return "Invalid chat history format.", "Please ensure the chat history is valid." + character_id = char_data.get('id') + if not character_id: + return "Invalid character data: No character ID found.", "" - if not char_data: - return "No character selected.", "Please select a character first." + # Sanitize and prepare conversation name + conversation_name = sanitize_conversation_name( + chat_media_name if chat_media_name.strip() + else f"Chat with {char_data.get('name', 'Unknown')} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + ) - character_id = char_data.get('id') - if not character_id: - return "Invalid character data: No character ID found.", "" + # Save to the database using your existing function + chat_id = save_chat_history_to_character_db( + character_id=character_id, + conversation_name=conversation_name, + chat_history=chat_history + ) - # Sanitize and prepare conversation name - conversation_name = sanitize_conversation_name( - chat_media_name if chat_media_name.strip() - else f"Chat with {char_data.get('name', 'Unknown')} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - ) + if chat_id: + success_message = ( + f"Chat saved successfully!\n" + f"ID: {chat_id}\n" + f"Name: {conversation_name}\n" + f"Messages: {len(chat_history)}" + ) + return success_message, "" + else: + return "Failed to save chat to database.", "Database operation failed." - # Save to the database using your existing function - chat_id = save_chat_history_to_character_db( - character_id=character_id, - conversation_name=conversation_name, - chat_history=chat_history + except Exception as e: + logging.error(f"Error saving chat history: {str(e)}", exc_info=True) + return f"Error saving chat: {str(e)}", "Please check the logs for more details." + + def update_character_info(name): + return load_character_and_image(name, user_name.value) + + def on_character_select(name, user_name_val): + logging.debug(f"Character selected: {name}") + char_data, chat_history, img = load_character_and_image(name, user_name_val) + return char_data, chat_history, img + + def clear_chat_history(char_data, user_name_val): + """ + Clears the chat history and initializes it with the character's first message, + replacing the '{{user}}' placeholder with the actual user name. + + Args: + char_data (dict): The current character data. + user_name_val (str): The user's name. + + Returns: + tuple: Updated chat history and the unchanged char_data. + """ + if char_data and 'first_message' in char_data and char_data['first_message']: + # Replace '{{user}}' in the first_message + first_message = char_data['first_message'].replace("{{user}}", + user_name_val if user_name_val else "User") + # Initialize chat history with the updated first_message + return [(None, first_message)], char_data + else: + # If no first_message is defined, simply clear the chat + return [], char_data + + def regenerate_last_message( + history, char_data, api_endpoint, api_key, + temperature, user_name_val, auto_save, streaming, minp, maxp + ): + """ + Regenerates the last bot message by removing it and resending the corresponding user message. + + Args: + history (list): The current chat history as a list of tuples (user_message, bot_message). + char_data (dict): The current character data. + api_endpoint (str): The API endpoint to use for the LLM. + api_key (str): The API key for authentication. + temperature (float): The temperature setting for the LLM. + user_name_val (str): The user's name. + auto_save (bool): Flag indicating whether to auto-save the chat. + + Returns: + tuple: Updated chat history and a save status message. + """ + if not history: + return history, "No messages to regenerate." + + last_entry = history[-1] + last_user_message, last_bot_message = last_entry + + # Check if the last bot message exists + if last_bot_message is None: + return history, "The last message is not from the bot." + + # Remove the last bot message + new_history = history[:-1] + + # Resend the last user message to generate a new bot response + if not last_user_message: + return new_history, "No user message to regenerate the bot response." + + # Prepare the character's background information + char_name = char_data.get('name', 'AI Assistant') + char_background = f""" + Name: {char_name} + Description: {char_data.get('description', 'N/A')} + Personality: {char_data.get('personality', 'N/A')} + Scenario: {char_data.get('scenario', 'N/A')} + """ + + # Prepare the system prompt for character impersonation + system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name_val}". + + {char_background} + + Additional instructions: {char_data.get('post_history_instructions', '')} + """ + + # Prepare media_content and selected_parts + media_content = { + 'id': char_name, + 'title': char_name, + 'content': char_background, + 'description': char_data.get('description', ''), + 'personality': char_data.get('personality', ''), + 'scenario': char_data.get('scenario', '') + } + selected_parts = ['description', 'personality', 'scenario'] + + prompt = char_data.get('post_history_instructions', '') + + # Prepare the input for the chat function + full_message = f"{user_name_val}: {last_user_message}" if last_user_message else f"{user_name_val}: " + + # Call the chat function to get a new bot message + bot_message = chat( + full_message, + new_history, + media_content, + selected_parts, + api_endpoint, + api_key, + prompt, + temperature, + system_message, + streaming, + minp=minp, + maxp=maxp ) - if chat_id: - success_message = ( - f"Chat saved successfully!\n" - f"ID: {chat_id}\n" - f"Name: {conversation_name}\n" - f"Messages: {len(chat_history)}" - ) - return success_message, "" + # Append the new bot message to the history + new_history.append((last_user_message, bot_message)) + + # Auto-save if enabled + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, new_history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." else: - return "Failed to save chat to database.", "Database operation failed." - - except Exception as e: - logging.error(f"Error saving chat history: {str(e)}", exc_info=True) - return f"Error saving chat: {str(e)}", "Please check the logs for more details." - - def update_character_info(name): - return load_character_and_image(name, user_name.value) - - def on_character_select(name, user_name_val): - logging.debug(f"Character selected: {name}") - char_data, chat_history, img = load_character_and_image(name, user_name_val) - return char_data, chat_history, img - - def clear_chat_history(char_data, user_name_val): - """ - Clears the chat history and initializes it with the character's first message, - replacing the '{{user}}' placeholder with the actual user name. + save_status = "" - Args: - char_data (dict): The current character data. - user_name_val (str): The user's name. - - Returns: - tuple: Updated chat history and the unchanged char_data. - """ - if char_data and 'first_message' in char_data and char_data['first_message']: - # Replace '{{user}}' in the first_message - first_message = char_data['first_message'].replace("{{user}}", - user_name_val if user_name_val else "User") - # Initialize chat history with the updated first_message - return [(None, first_message)], char_data - else: - # If no first_message is defined, simply clear the chat - return [], char_data + return new_history, save_status - def regenerate_last_message( - history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save - ): - """ - Regenerates the last bot message by removing it and resending the corresponding user message. + def toggle_chat_file_upload(): + return gr.update(visible=True) - Args: - history (list): The current chat history as a list of tuples (user_message, bot_message). - char_data (dict): The current character data. - api_endpoint (str): The API endpoint to use for the LLM. - api_key (str): The API key for authentication. - temperature (float): The temperature setting for the LLM. - user_name_val (str): The user's name. - auto_save (bool): Flag indicating whether to auto-save the chat. + def save_untracked_chat_action(history, char_data): + if not char_data or not history: + return "No chat to save or character not selected." - Returns: - tuple: Updated chat history and a save status message. - """ - if not history: - return history, "No messages to regenerate." - - last_entry = history[-1] - last_user_message, last_bot_message = last_entry - - # Check if the last bot message exists - if last_bot_message is None: - return history, "The last message is not from the bot." - - # Remove the last bot message - new_history = history[:-1] - - # Resend the last user message to generate a new bot response - if not last_user_message: - return new_history, "No user message to regenerate the bot response." - - # Prepare the character's background information - char_name = char_data.get('name', 'AI Assistant') - char_background = f""" - Name: {char_name} - Description: {char_data.get('description', 'N/A')} - Personality: {char_data.get('personality', 'N/A')} - Scenario: {char_data.get('scenario', 'N/A')} - """ - - # Prepare the system prompt for character impersonation - system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name_val}". - - {char_background} - - Additional instructions: {char_data.get('post_history_instructions', '')} - """ - - # Prepare media_content and selected_parts - media_content = { - 'id': char_name, - 'title': char_name, - 'content': char_background, - 'description': char_data.get('description', ''), - 'personality': char_data.get('personality', ''), - 'scenario': char_data.get('scenario', '') - } - selected_parts = ['description', 'personality', 'scenario'] - - prompt = char_data.get('post_history_instructions', '') - - # Prepare the input for the chat function - full_message = f"{user_name_val}: {last_user_message}" if last_user_message else f"{user_name_val}: " - - # Call the chat function to get a new bot message - bot_message = chat( - full_message, - new_history, - media_content, - selected_parts, - api_endpoint, - api_key, - prompt, - temperature, - system_message - ) - - # Append the new bot message to the history - new_history.append((last_user_message, bot_message)) - - # Auto-save if enabled - if auto_save: character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, new_history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." - else: - save_status = "" - - return new_history, save_status - - def toggle_chat_file_upload(): - return gr.update(visible=True) - - def save_untracked_chat_action(history, char_data): - if not char_data or not history: - return "No chat to save or character not selected." - - character_id = char_data.get('id') - if not character_id: - return "Character ID not found." - - conversation_name = f"Snapshot {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - chat_id = add_character_chat(character_id, conversation_name, history, is_snapshot=True) - if chat_id: - return f"Chat snapshot saved successfully with ID {chat_id}." - else: - return "Failed to save chat snapshot." - - def select_chat_for_update(): - # Fetch all chats for the selected character - if character_data.value: - character_id = character_data.value.get('id') - if character_id: - chats = get_character_chats(character_id) - chat_choices = [ - f"{chat['conversation_name']} (ID: {chat['id']})" for chat in chats - ] - return gr.update(choices=chat_choices), None - return gr.update(choices=[]), "No character selected." - - def load_selected_chat(chat_selection): - if not chat_selection: - return [], "No chat selected." + if not character_id: + return "Character ID not found." - try: - chat_id = int(chat_selection.split('(ID: ')[1].rstrip(')')) - chat = get_character_chat_by_id(chat_id) - if chat: - history = chat['chat_history'] - selected_chat_id.value = chat_id # Update the selected_chat_id state - return history, f"Loaded chat '{chat['conversation_name']}' successfully." + conversation_name = f"Snapshot {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + chat_id = add_character_chat(character_id, conversation_name, history, is_snapshot=True) + if chat_id: + return f"Chat snapshot saved successfully with ID {chat_id}." else: - return [], "Chat not found." - except Exception as e: - logging.error(f"Error loading selected chat: {e}") - return [], f"Error loading chat: {e}" - - def update_chat(chat_id, updated_history): - success = update_character_chat(chat_id, updated_history) - if success: - return "Chat updated successfully." - else: - return "Failed to update chat." - - def continue_talking( - history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save - ): - """ - Causes the character to continue the conversation or think out loud. - """ - if not char_data: - return history, "Please select a character first." - - user_name_val = user_name_val or "User" - char_name = char_data.get('name', 'AI Assistant') - - # Prepare the character's background information - char_background = f""" - Name: {char_name} - Description: {char_data.get('description', 'N/A')} - Personality: {char_data.get('personality', 'N/A')} - Scenario: {char_data.get('scenario', 'N/A')} - """ - - # Prepare the system prompt - system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')} - If the user does not respond, continue expressing your thoughts or continue the conversation by thinking out loud. If thinking out loud, prefix the message with "Thinking: ".""" - - # Prepare chat context - media_content = { - 'id': char_name, - 'title': char_name, - 'content': char_background, - 'description': char_data.get('description', ''), - 'personality': char_data.get('personality', ''), - 'scenario': char_data.get('scenario', '') - } - selected_parts = ['description', 'personality', 'scenario'] - - prompt = char_data.get('post_history_instructions', '') - - # Simulate empty user input - user_message = "" - - # Generate bot response - bot_message = chat( - user_message, - history, - media_content, - selected_parts, - api_endpoint, - api_key, - prompt, - temperature, - system_message - ) - - # Replace placeholders in bot message - bot_message = replace_placeholders(bot_message, char_name, user_name_val) + return "Failed to save chat snapshot." + + def select_chat_for_update(): + # Fetch all chats for the selected character + if character_data.value: + character_id = character_data.value.get('id') + if character_id: + chats = get_character_chats(character_id) + chat_choices = [ + f"{chat['conversation_name']} (ID: {chat['id']})" for chat in chats + ] + return gr.update(choices=chat_choices), None + return gr.update(choices=[]), "No character selected." + + def load_selected_chat(chat_selection): + if not chat_selection: + return [], "No chat selected." - # Update history - history.append((None, bot_message)) + try: + chat_id = int(chat_selection.split('(ID: ')[1].rstrip(')')) + chat = get_character_chat_by_id(chat_id) + if chat: + history = chat['chat_history'] + selected_chat_id.value = chat_id # Update the selected_chat_id state + return history, f"Loaded chat '{chat['conversation_name']}' successfully." + else: + return [], "Chat not found." + except Exception as e: + logging.error(f"Error loading selected chat: {e}") + return [], f"Error loading chat: {e}" - # Auto-save if enabled - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, history) - save_status = "Chat auto-saved." + def update_chat(chat_id, updated_history): + success = update_character_chat(chat_id, updated_history) + if success: + return "Chat updated successfully." else: - save_status = "Character ID not found; chat not saved." - - return history, save_status - - def answer_for_me( - history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save - ): - """ - Generates a likely user response and continues the conversation. - """ - if not char_data: - return history, "Please select a character first." - - user_name_val = user_name_val or "User" - char_name = char_data.get('name', 'AI Assistant') - - # Prepare the character's background information - char_background = f""" - Name: {char_name} - Description: {char_data.get('description', 'N/A')} - Personality: {char_data.get('personality', 'N/A')} - Scenario: {char_data.get('scenario', 'N/A')} - """ - - # Prepare system message for generating user's response - system_message_user = f"""You are simulating the user {user_name_val}. Based on the conversation so far, generate a natural and appropriate response that {user_name_val} might say next. The response should fit the context and flow of the conversation. ONLY SPEAK FOR {user_name_val}.""" - - # Prepare chat context - media_content = { - 'id': char_name, - 'title': char_name, - 'content': char_background, - 'description': char_data.get('description', ''), - 'personality': char_data.get('personality', ''), - 'scenario': char_data.get('scenario', '') - } - selected_parts = ['description', 'personality', 'scenario'] - - # Generate user response - user_response = chat( - "", # No new message - history, - media_content, - selected_parts, - api_endpoint, - api_key, - prompt="", - temperature=temperature, - system_message=system_message_user - ) - - # Append the generated user response to history - history.append((user_response, None)) - - # Now generate the character's response to this user response - # Prepare the system message for the character - system_message_bot = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}""" - - bot_message = chat( - f"{user_name_val}: {user_response}", - history[:-1], - media_content, - selected_parts, - api_endpoint, - api_key, - prompt=char_data.get('post_history_instructions', ''), - temperature=temperature, - system_message=system_message_bot - ) - - # Replace placeholders in bot message - bot_message = replace_placeholders(bot_message, char_name, user_name_val) + return "Failed to update chat." + + def continue_talking( + history, char_data, api_endpoint, api_key, + temperature, user_name_val, auto_save, minp, maxp + ): + """ + Causes the character to continue the conversation or think out loud. + """ + if not char_data: + return history, "Please select a character first." + + user_name_val = user_name_val or "User" + char_name = char_data.get('name', 'AI Assistant') + + # Prepare the character's background information + char_background = f""" + Name: {char_name} + Description: {char_data.get('description', 'N/A')} + Personality: {char_data.get('personality', 'N/A')} + Scenario: {char_data.get('scenario', 'N/A')} + """ + + # Prepare the system prompt + system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')} + If the user does not respond, continue expressing your thoughts or continue the conversation by thinking out loud. If thinking out loud, prefix the message with "Thinking: ".""" + + # Prepare chat context + media_content = { + 'id': char_name, + 'title': char_name, + 'content': char_background, + 'description': char_data.get('description', ''), + 'personality': char_data.get('personality', ''), + 'scenario': char_data.get('scenario', '') + } + selected_parts = ['description', 'personality', 'scenario'] + + prompt = char_data.get('post_history_instructions', '') + + # Simulate empty user input + user_message = "" + + # Generate bot response + bot_message = chat( + user_message, + history, + media_content, + selected_parts, + api_endpoint, + api_key, + prompt, + temperature, + system_message, + streaming, + minp=minp, + maxp=maxp + ) - # Update history with bot's response - history[-1] = (user_response, bot_message) + # Replace placeholders in bot message + bot_message = replace_placeholders(bot_message, char_name, user_name_val) - # Auto-save if enabled - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." + # Update history + history.append((None, bot_message)) - return history, save_status + # Auto-save if enabled + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + + return history, save_status + + def answer_for_me( + history, char_data, api_endpoint, api_key, + temperature, user_name_val, auto_save, minp, maxp): + """ + Generates a likely user response and continues the conversation. + """ + if not char_data: + return history, "Please select a character first." + + user_name_val = user_name_val or "User" + char_name = char_data.get('name', 'AI Assistant') + + # Prepare the character's background information + char_background = f""" + Name: {char_name} + Description: {char_data.get('description', 'N/A')} + Personality: {char_data.get('personality', 'N/A')} + Scenario: {char_data.get('scenario', 'N/A')} + """ + + # Prepare system message for generating user's response + system_message_user = f"""You are simulating the user {user_name_val}. Based on the conversation so far, generate a natural and appropriate response that {user_name_val} might say next. The response should fit the context and flow of the conversation. ONLY SPEAK FOR {user_name_val}.""" + + # Prepare chat context + media_content = { + 'id': char_name, + 'title': char_name, + 'content': char_background, + 'description': char_data.get('description', ''), + 'personality': char_data.get('personality', ''), + 'scenario': char_data.get('scenario', '') + } + selected_parts = ['description', 'personality', 'scenario'] + + # Generate user response + user_response = chat( + "", # No new message + history, + media_content, + selected_parts, + api_endpoint, + api_key, + prompt="", + temperature=temperature, + system_message=system_message_user, + streaming=streaming, + minp=minp, + maxp=maxp + ) + # Append the generated user response to history + history.append((user_response, None)) + + # Now generate the character's response to this user response + # Prepare the system message for the character + system_message_bot = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}""" + + bot_message = chat( + f"{user_name_val}: {user_response}", + history[:-1], + media_content, + selected_parts, + api_endpoint, + api_key, + prompt=char_data.get('post_history_instructions', ''), + temperature=temperature, + system_message=system_message_bot, + streaming=streaming, + minp=minp, + maxp=maxp + ) - # Define States for conversation_id and media_content, which are required for saving chat history - conversation_id = gr.State(str(uuid.uuid4())) - media_content = gr.State({}) - # TTS Generation and Playback - def speak_last_response(chat_history): - """Handle speaking the last chat response.""" - logging.debug("Starting speak_last_response") - try: - # If there's no chat history, return - if not chat_history or len(chat_history) == 0: - logging.debug("No messages in chat history") - return gr.update(value="No messages to speak", visible=True) + # Replace placeholders in bot message + bot_message = replace_placeholders(bot_message, char_name, user_name_val) - # Log the chat history content for debugging - logging.debug(f"Chat history: {chat_history}") + # Update history with bot's response + history[-1] = (user_response, bot_message) - # Get the last message from the assistant - last_message = chat_history[-1][1] # Second element of the last tuple - logging.debug(f"Last message to speak: {last_message}") + # Auto-save if enabled + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + + return history, save_status + + + # Define States for conversation_id and media_content, which are required for saving chat history + conversation_id = gr.State(str(uuid.uuid4())) + media_content = gr.State({}) + # TTS Generation and Playback + def speak_last_response(chat_history): + """Handle speaking the last chat response.""" + logging.debug("Starting speak_last_response") + try: + # If there's no chat history, return + if not chat_history or len(chat_history) == 0: + logging.debug("No messages in chat history") + return gr.update(value="No messages to speak", visible=True) - # Update status to generating - yield gr.update(value="Generating audio...", visible=True) + # Log the chat history content for debugging + logging.debug(f"Chat history: {chat_history}") - # Generate audio using your preferred TTS provider - try: - audio_file = generate_audio( - text=last_message, - provider="openai", # or get from config - output_file="last_response.mp3" - ) - logging.debug(f"Generated audio file: {audio_file}") - except Exception as e: - logging.error(f"Failed to generate audio: {e}") - yield gr.update(value=f"Failed to generate audio: {str(e)}", visible=True) - return + # Get the last message from the assistant + last_message = chat_history[-1][1] # Second element of the last tuple + logging.debug(f"Last message to speak: {last_message}") - # Update status to playing - yield gr.update(value="Playing audio...", visible=True) + # Update status to generating + yield gr.update(value="Generating audio...", visible=True) - # Play the audio - if audio_file and os.path.exists(audio_file): + # Generate audio using your preferred TTS provider try: - play_mp3(audio_file) - yield gr.update(value="Finished playing audio", visible=True) + audio_file = generate_audio( + text=last_message, + provider="openai", # or get from config + output_file="last_response.mp3", + api_key=None + ) + logging.debug(f"Generated audio file: {audio_file}") except Exception as e: - logging.error(f"Failed to play audio: {e}") - yield gr.update(value=f"Failed to play audio: {str(e)}", visible=True) - else: - logging.error("Audio file not found") - yield gr.update(value="Failed: Audio file not found", visible=True) + logging.error(f"Failed to generate audio: {e}") + yield gr.update(value=f"Failed to generate audio: {str(e)}", visible=True) + return + + # Update status to playing + yield gr.update(value="Playing audio...", visible=True) + + # Play the audio + if audio_file and os.path.exists(audio_file): + try: + play_audio_file(audio_file) + yield gr.update(value="Finished playing audio", visible=True) + except Exception as e: + logging.error(f"Failed to play audio: {e}") + yield gr.update(value=f"Failed to play audio: {str(e)}", visible=True) + else: + logging.error("Audio file not found") + yield gr.update(value="Failed: Audio file not found", visible=True) - except Exception as e: - logging.error(f"Error in speak_last_response: {str(e)}") - yield gr.update(value=f"Error: {str(e)}", visible=True) + except Exception as e: + logging.error(f"Error in speak_last_response: {str(e)}") + yield gr.update(value=f"Error: {str(e)}", visible=True) - # Update the click handler + # Button Callbacks speak_button.click( fn=speak_last_response, inputs=[chat_history], # Use chat_history instead of chatbot outputs=[tts_status], api_name="speak_response" ) - # Button Callbacks - - # Add the new button callbacks here - answer_for_me_button.click( - fn=answer_for_me, - inputs=[ - chat_history, - character_data, - api_name_input, - api_key_input, - temperature_slider, - user_name_input, - auto_save_checkbox - ], - outputs=[chat_history, save_status] - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) - - continue_talking_button.click( - fn=continue_talking, - inputs=[ - chat_history, - character_data, - api_name_input, - api_key_input, - temperature_slider, - user_name_input, - auto_save_checkbox - ], - outputs=[chat_history, save_status] - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) + # Add the new button callbacks here + answer_for_me_button.click( + fn=answer_for_me, + inputs=[ + chat_history, + character_data, + api_name_input, + api_key_input, + temperature_slider, + user_name_input, + auto_save_checkbox, + minp_slider, + maxp_slider + ], + outputs=[chat_history, save_status] + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) - import_card_button.click( - fn=import_character_card, - inputs=[character_card_upload], - outputs=[character_data, character_dropdown, save_status] - ) + continue_talking_button.click( + fn=continue_talking, + inputs=[ + chat_history, + character_data, + api_name_input, + api_key_input, + temperature_slider, + user_name_input, + auto_save_checkbox, + minp_slider, + maxp_slider + ], + outputs=[chat_history, save_status] + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) - load_characters_button.click( - fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]), - outputs=character_dropdown - ) + import_card_button.click( + fn=import_character_card, + inputs=[character_card_upload], + outputs=[character_data, character_dropdown, save_status] + ) - # FIXME user_name_val = validate_user_name(user_name_val) - clear_chat_button.click( - fn=clear_chat_history, - inputs=[character_data, user_name_input], - outputs=[chat_history, character_data] - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) + load_characters_button.click( + fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]), + outputs=character_dropdown + ) - character_dropdown.change( - fn=extract_character_id, - inputs=[character_dropdown], - outputs=character_data - ).then( - fn=load_character_wrapper, - inputs=[character_data, user_name_input], - outputs=[character_data, chat_history, character_image] - ) + # FIXME user_name_val = validate_user_name(user_name_val) + clear_chat_button.click( + fn=clear_chat_history, + inputs=[character_data, user_name_input], + outputs=[chat_history, character_data] + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) - send_message_button.click( - fn=character_chat_wrapper, - inputs=[ - user_input, - chat_history, - character_data, - api_name_input, - api_key_input, - temperature_slider, - user_name_input, - auto_save_checkbox - ], - outputs=[chat_history, save_status] - ).then( - lambda: "", outputs=user_input - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) + character_dropdown.change( + fn=extract_character_id, + inputs=[character_dropdown], + outputs=character_data + ).then( + fn=load_character_wrapper, + inputs=[character_data, user_name_input], + outputs=[character_data, chat_history, character_image] + ) - regenerate_button.click( - fn=regenerate_last_message, - inputs=[ - chat_history, - character_data, - api_name_input, - api_key_input, - temperature_slider, - user_name_input, - auto_save_checkbox - ], - outputs=[chat_history, save_status] - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) + send_message_button.click( + fn=character_chat_wrapper, + inputs=[ + user_input, + chat_history, + character_data, + api_name_input, + api_key_input, + temperature_slider, + user_name_input, + auto_save_checkbox, + minp_slider, + maxp_slider + ], + outputs=[chat_history, save_status] + ).then( + lambda: "", outputs=user_input + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) - import_chat_button.click( - fn=lambda: gr.update(visible=True), - outputs=chat_file_upload - ) + regenerate_button.click( + fn=regenerate_last_message, + inputs=[ + chat_history, + character_data, + api_name_input, + api_key_input, + temperature_slider, + user_name_input, + auto_save_checkbox, + streaming, + minp_slider, + maxp_slider + ], + outputs=[chat_history, save_status] + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) - chat_file_upload.change( - fn=import_chat_history, - inputs=[chat_file_upload, chat_history, character_data, user_name_input], - outputs=[chat_history, character_data, save_status] - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) + import_chat_button.click( + fn=lambda: gr.update(visible=True), + outputs=chat_file_upload + ) - save_chat_history_to_db.click( - fn=save_chat_history_to_db_wrapper, - inputs=[ - chat_history, - conversation_id, - media_content, - chat_media_name, - character_data, - auto_save_checkbox # Pass the auto_save state - ], - outputs=[conversation_id, save_status] - ) + chat_file_upload.change( + fn=import_chat_history, + inputs=[chat_file_upload, chat_history, character_data, user_name_input], + outputs=[chat_history, character_data, save_status] + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) - # Populate the update_chat_dropdown based on selected character - character_dropdown.change( - fn=select_chat_for_update, - inputs=[], - outputs=[update_chat_dropdown, save_status] - ) + save_chat_history_to_db.click( + fn=save_chat_history_to_db_wrapper, + inputs=[ + chat_history, + conversation_id, + media_content, + chat_media_name, + character_data, + auto_save_checkbox # Pass the auto_save state + ], + outputs=[conversation_id, save_status] + ) - load_selected_chat_button.click( - fn=load_selected_chat, - inputs=[update_chat_dropdown], - outputs=[chat_history, save_status] - ) + # Populate the update_chat_dropdown based on selected character + character_dropdown.change( + fn=select_chat_for_update, + inputs=[], + outputs=[update_chat_dropdown, save_status] + ) - save_snapshot_button.click( - fn=save_untracked_chat_action, - inputs=[chat_history, character_data], - outputs=save_status - ) + load_selected_chat_button.click( + fn=load_selected_chat, + inputs=[update_chat_dropdown], + outputs=[chat_history, save_status] + ) - update_chat_button.click( - fn=update_chat, - inputs=[selected_chat_id, chat_history], - outputs=save_status - ) + save_snapshot_button.click( + fn=save_untracked_chat_action, + inputs=[chat_history, character_data], + outputs=save_status + ) - # Search Chats - chat_search_button.click( - fn=search_existing_chats, - inputs=[chat_search_query], - outputs=[chat_search_dropdown, save_status] - ).then( - fn=lambda choices, msg: gr.update(choices=choices, visible=True) if choices else gr.update(visible=False), - inputs=[chat_search_dropdown, save_status], - outputs=[chat_search_dropdown] - ) + update_chat_button.click( + fn=update_chat, + inputs=[selected_chat_id, chat_history], + outputs=save_status + ) - # Load Selected Chat from Search - load_chat_button.click( - fn=load_selected_chat_from_search, - inputs=[chat_search_dropdown, user_name_input], - outputs=[character_data, chat_history, character_image, save_status] - ).then( - lambda history: approximate_token_count(history), - inputs=[chat_history], - outputs=[token_count_display] - ) + # Search Chats + chat_search_button.click( + fn=search_existing_chats, + inputs=[chat_search_query], + outputs=[chat_search_dropdown, save_status] + ).then( + fn=lambda choices, msg: gr.update(choices=choices, visible=True) if choices else gr.update(visible=False), + inputs=[chat_search_dropdown, save_status], + outputs=[chat_search_dropdown] + ) - # Show Load Chat Button when a chat is selected - chat_search_dropdown.change( - fn=lambda selected: gr.update(visible=True) if selected else gr.update(visible=False), - inputs=[chat_search_dropdown], - outputs=[load_chat_button] - ) + # Load Selected Chat from Search + load_chat_button.click( + fn=load_selected_chat_from_search, + inputs=[chat_search_dropdown, user_name_input], + outputs=[character_data, chat_history, character_image, save_status] + ).then( + lambda history: approximate_token_count(history), + inputs=[chat_history], + outputs=[token_count_display] + ) + # Show Load Chat Button when a chat is selected + chat_search_dropdown.change( + fn=lambda selected: gr.update(visible=True) if selected else gr.update(visible=False), + inputs=[chat_search_dropdown], + outputs=[load_chat_button] + ) - return character_data, chat_history, user_input, user_name, character_image + return character_data, chat_history, user_input, user_name, character_image def create_character_chat_mgmt_tab(): @@ -1653,7 +1691,11 @@ def export_all_character_conversations(character_selection): load_characters_button.click( fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]), outputs=select_character - ) + ).then( + lambda choices: print(f"Dropdown choices: {choices}"), # Debugging line + inputs=[select_character], + outputs=None + ) export_chat_button.click( fn=export_current_conversation, diff --git a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py index 7c1e14ed0..3ad08922e 100644 --- a/App_Function_Libraries/Gradio_UI/WebSearch_tab.py +++ b/App_Function_Libraries/Gradio_UI/WebSearch_tab.py @@ -24,33 +24,35 @@ def create_websearch_tab(): search_state = gr.State(value=None) # Basic styling gr.HTML(""" - """) @@ -108,7 +110,12 @@ def create_websearch_tab(): # Final Output Section with gr.Column(visible=False) as output_section: - answer = gr.Markdown(label="Analysis") + # Single markdown box for all analysis text + answer = gr.Markdown( + label="Analysis Results", + elem_classes="analysis-text" + ) + # Sources box sources = gr.JSON(label="Sources") def format_results(results: list) -> str: @@ -252,12 +259,20 @@ async def analyze_handler(state): if not analysis.get("final_answer"): raise ValueError("Analysis did not produce a final answer") - formatted_answer = f""" - ### Analysis Result - {analysis["final_answer"]["Report"]} + # Format the raw report with proper markdown + raw_report = analysis["final_answer"]["Report"] - ### Sources - """ + # Ensure proper markdown formatting + formatted_answer = raw_report.replace('\n', '\n\n') # Double line breaks + formatted_answer = formatted_answer.replace('•', '\n•') # Bullet points on new lines + formatted_answer = formatted_answer.replace('- ', '\n- ') # Dashed lists on new lines + + # Handle numbered lists (assumes numbers followed by period or parenthesis) + import re + formatted_answer = re.sub(r'(\d+[\)\.]) ', r'\n\1 ', formatted_answer) + + # Clean up any triple+ line breaks + formatted_answer = re.sub(r'\n{3,}', '\n\n', formatted_answer) yield ( gr.HTML( diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 73600a85e..9ec91362a 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -116,7 +116,7 @@ def get_openai_embeddings(input_data: str, model: str) -> List[float]: raise ValueError(f"OpenAI: Unexpected error occurred: {str(e)}") -def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_message, streaming): +def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_message, streaming, minp, maxp, model): loaded_config_data = load_and_log_configs() openai_api_key = api_key try: @@ -126,7 +126,7 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_messag logging.info("OpenAI: Attempting to use API key from config file") openai_api_key = loaded_config_data['openai_api']['api_key'] - if not openai_api_key: + if not openai_api_key or openai_api_key == "": logging.error("OpenAI: API key not found or is empty") return "OpenAI: API Key Not Provided/Found in Config file or is empty" @@ -134,12 +134,21 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_messag if isinstance(streaming, str): streaming = streaming.lower() == "true" + elif isinstance(streaming, int): + streaming = bool(streaming) # Convert integers (1/0) to boolean elif streaming is None: - streaming = loaded_config_data['openai_api']['streaming'] - if streaming: + streaming = loaded_config_data.get('openai_api', {}).get('streaming', False) logging.debug("OpenAI: Streaming mode enabled") else: logging.debug("OpenAI: Streaming mode disabled") + if not isinstance(streaming, bool): + raise ValueError(f"Invalid type for 'streaming': Expected a boolean, got {type(streaming).__name__}") + + if maxp is None: + maxp = loaded_config_data['openai_api']['max_p'] + if model is None: + openai_model = loaded_config_data['openai_api']['model'] or "gpt-4o" + logging.debug(f"OpenAI: Using model: {openai_model}") # Input data handling logging.debug(f"OpenAI: Raw input data type: {type(input_data)}") @@ -186,9 +195,6 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_messag logging.debug(f"OpenAI: Extracted text (first 500 chars): {text[:500]}...") logging.debug(f"OpenAI: Custom prompt: {custom_prompt_arg}") - openai_model = loaded_config_data['openai_api']['model'] or "gpt-4o" - logging.debug(f"OpenAI: Using model: {openai_model}") - headers = { 'Authorization': f'Bearer {openai_api_key}', 'Content-Type': 'application/json' @@ -209,9 +215,10 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_messag {"role": "system", "content": system_message}, {"role": "user", "content": openai_prompt} ], - "max_tokens": 4096, + "max_completion_tokens": 4096, "temperature": temp, - "stream": streaming + "stream": streaming, + "top_p": maxp } if streaming: logging.debug("OpenAI: Posting request (streaming") @@ -276,7 +283,7 @@ def stream_generator(): return f"OpenAI: Unexpected error occurred: {str(e)}" -def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5, system_prompt=None, temp=None, streaming=False): +def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5, system_prompt=None, temp=None, streaming=False, minp=None, maxp=None): try: loaded_config_data = load_and_log_configs() @@ -334,10 +341,27 @@ def chat_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retri if temp is None: temp = loaded_config_data['anthropic_api']['temperature'] temp = float(temp) + logging.debug(f"Anthropic: Using temperature from config.txt: {temp}") else: temp = 0.7 logging.debug(f"Anthropic: Using default temperature: {temp}") + if minp is None: + minp = loaded_config_data['anthropic_api']['min_p'] + minp = float(minp) + logging.debug(f"Anthropic: Using min_p from config.txt: {minp}") + else: + minp = 0.0 + logging.debug(f"Anthropic: Using default min_p: {minp}") + + if maxp is None: + maxp = loaded_config_data['anthropic_api']['top_p'] + maxp = float(maxp) + logging.debug(f"Anthropic: Using max_p from config.txt: {maxp}") + else: + maxp = 1.0 + logging.debug(f"Anthropic: Using default maxp: {maxp}") + headers = { 'x-api-key': anthropic_api_key, 'anthropic-version': '2023-06-01', diff --git a/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py b/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py index b24a3f2b7..c1b6a5d69 100644 --- a/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py +++ b/App_Function_Libraries/Prompt_Engineering/Prompt_Engineering.py @@ -495,7 +495,7 @@ def generate_prompt(api_endpoint, api_key, task, variables_str, temperature): # Call chat API to generate the prompt response = chat_api_call(api_endpoint=api_endpoint, api_key=api_key, input_data="", prompt=metaprompt, - temp=temperature) + temp=temperature, streaming=False, minp=None, maxp=None, model=None) return response def extract_between_tags(tag: str, string: str, strip: bool = False) -> list[str]: @@ -540,7 +540,7 @@ def test_generated_prompt(api_endpoint, api_key, generated_prompt, variable_valu prompt_with_values = prompt_with_values.replace(var, value) # Send the filled-in prompt to the chat API - response = chat_api_call(api_endpoint=api_endpoint, api_key=api_key, input_data="", prompt=prompt_with_values, temp=temperature) + response = chat_api_call(api_endpoint=api_endpoint, api_key=api_key, input_data="", prompt=prompt_with_values, temp=temperature, system_message=None, streaming=False, minp=None, maxp=None, model=None) return response # diff --git a/App_Function_Libraries/TTS/TTS_Providers.py b/App_Function_Libraries/TTS/TTS_Providers.py index 49b1bea96..dcca142d4 100644 --- a/App_Function_Libraries/TTS/TTS_Providers.py +++ b/App_Function_Libraries/TTS/TTS_Providers.py @@ -27,17 +27,27 @@ def play_mp3(file_path): """Play an MP3 file using the pydub library.""" try: from pydub.utils import which - print(f"Debug: ffmpeg path: {which('ffmpeg')}") - print(f"Debug: ffplay path: {which('ffplay')}") + logging.debug(f"Debug: ffmpeg path: {which('ffmpeg')}") + logging.debug(f"Debug: ffplay path: {which('ffplay')}") absolute_path = os.path.abspath(file_path) audio = AudioSegment.from_mp3(absolute_path) - print("Debug: File loaded successfully") + logging.debug("Debug: File loaded successfully") play(audio) except Exception as e: - print(f"Debug: Exception type: {type(e)}") - print(f"Debug: Exception args: {e.args}") - print(f"Error playing the audio file: {e}") + logging.debug(f"Debug: Exception type: {type(e)}") + logging.debug(f"Debug: Exception args: {e.args}") + logging.error(f"Error playing the audio file: {e}") + + +def play_audio_file(file_path): + """Play an audio file using the pydub library.""" + try: + absolute_path = os.path.abspath(file_path) + audio = AudioSegment.from_file(absolute_path) + play(audio) + except Exception as e: + logging.error(f"Error playing the audio file: {e}") def generate_audio(api_key, text, provider, voice=None, model=None, voice2=None, output_file=None, response_format=None, streaming=False): diff --git a/App_Function_Libraries/Utils/Utils.py b/App_Function_Libraries/Utils/Utils.py index fcaf707c7..3c4354d2d 100644 --- a/App_Function_Libraries/Utils/Utils.py +++ b/App_Function_Libraries/Utils/Utils.py @@ -243,7 +243,6 @@ def load_and_log_configs(): anthropic_min_p = config.get('API', 'anthropic_min_p', fallback='0.05') cohere_streaming = config.get('API', 'cohere_streaming', fallback='False') cohere_temperature = config.get('API', 'cohere_temperature', fallback='0.7') - cohere_top_p = config.get('API', 'cohere_top_p', fallback='0.95') cohere_min_p = config.get('API', 'cohere_min_p', fallback='0.05') groq_streaming = config.get('API', 'groq_streaming', fallback='False') groq_temperature = config.get('API', 'groq_temperature', fallback='0.7') @@ -252,7 +251,6 @@ def load_and_log_configs(): openai_streaming = config.get('API', 'openai_streaming', fallback='False') openai_temperature = config.get('API', 'openai_temperature', fallback='0.7') openai_top_p = config.get('API', 'openai_top_p', fallback='0.95') - openai_min_p = config.get('API', 'openai_min_p', fallback='0.05') huggingface_streaming = config.get('API', 'huggingface_streaming', fallback='False') huggingface_temperature = config.get('API', 'huggingface_temperature', fallback='0.7') huggingface_top_p = config.get('API', 'huggingface_top_p', fallback='0.95') @@ -481,7 +479,6 @@ def load_and_log_configs(): 'model': cohere_model, 'streaming': cohere_streaming, 'temperature': cohere_temperature, - 'top_p': cohere_top_p, 'min_p': cohere_min_p }, 'deepseek_api': { @@ -535,7 +532,6 @@ def load_and_log_configs(): 'streaming': openai_streaming, 'temperature': openai_temperature, 'top_p': openai_top_p, - 'min_p': openai_min_p }, 'elevenlabs_api': { 'api_key': elevenlabs_api_key diff --git a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py index ba241f882..48bb4f0c0 100644 --- a/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py +++ b/App_Function_Libraries/Web_Scraping/WebSearch_APIs.py @@ -296,7 +296,7 @@ def analyze_question(question: str, api_endpoint) -> Dict: try: logging.info(f"Generating sub-questions (attempt {attempt + 1})") - response = chat_api_call(api_endpoint, None, input_data, sub_question_generation_prompt, temp=0.7, system_message=None, streaming=False) + response = chat_api_call(api_endpoint, None, input_data, sub_question_generation_prompt, temp=0.7, system_message=None, streaming=False, minp=None, maxp=None, model=None) if response: try: # Try to parse as JSON first diff --git a/Docs/Design/Creative_Writing.md b/Docs/Design/Creative_Writing.md index fd2a9206f..34fb55daf 100644 --- a/Docs/Design/Creative_Writing.md +++ b/Docs/Design/Creative_Writing.md @@ -12,6 +12,6 @@ https://github.com/FartyPants/StoryCrafter/tree/main https://github.com/datacrystals/AIStoryWriter https://perchance.org/ai-story-generator https://perchance.org/tutorial#repeating_things - +https://github.com/Cloud-Code-AI/AkiraDocs diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 6b072eb13..58f2293aa 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -47,6 +47,10 @@ https://github.com/microsoft/SpeechT5 https://www.lightnote.co/?utm_source=www.hivefive.community&utm_medium=newsletter&utm_campaign=hive-five-202-a-bias-to-action https://github.com/smellslikeml/dolla_llama https://github.com/dnhkng/GlaDOS +https://arxiv.org/abs/2501.01384 +https://sharechatx.github.io/ +https://github.com/vipchengrui/traditional-speech-enhancement + STT https://github.com/KoljaB/RealtimeSTT @@ -58,6 +62,7 @@ STT TTS https://github.com/KoljaB/RealtimeTTS + https://si.inc/hertz-dev/ 101 https://www.inferless.com/learn/comparing-different-text-to-speech---tts--models-for-different-use-cases diff --git a/Docs/Design/Text2SQL.md b/Docs/Design/Text2SQL.md index 66213b240..029ec72f8 100644 --- a/Docs/Design/Text2SQL.md +++ b/Docs/Design/Text2SQL.md @@ -18,3 +18,6 @@ https://medium.com/intel-tech/tabular-data-rag-llms-improve-results-through-data https://github.com/TAG-Research/TAG-Bench https://arxiv.org/pdf/2407.14482 https://spider2-sql.github.io/ +https://departmentofproduct.substack.com/p/how-to-write-sql-queries-using-ai?utm_medium=email&triedRedirect=true + + diff --git a/Docs/Design/UX.md b/Docs/Design/UX.md index 9fbd8e2af..19b6e1130 100644 --- a/Docs/Design/UX.md +++ b/Docs/Design/UX.md @@ -83,6 +83,8 @@ https://lovable.dev/ https://markwhen.com/ https://kando.menu/ https://deepseek-artifacts.vercel.app/ +https://darkpatternsgame.productartistry.com/ + Not waifus, but clippy: https://github.com/fghrsh/live2d_demo diff --git a/Docs/Design/VLMs.md b/Docs/Design/VLMs.md index 9fce2d2e0..728ad1cb2 100644 --- a/Docs/Design/VLMs.md +++ b/Docs/Design/VLMs.md @@ -23,6 +23,10 @@ https://huggingface.co/Qwen/QVQ-72B-Preview https://huggingface.co/Qwen/Qwen2-VL-72B https://huggingface.co/Qwen/Qwen2-VL-7B https://huggingface.co/Qwen/Qwen2-VL-2B +https://github.com/Lokesh-Chimakurthi/vision-rag +https://github.com/tjmlabs/ColiVara + + https://www.reddit.com/r/StableDiffusion/comments/1h7hunp/how_to_run_hunyuanvideo_on_a_single_24gb_vram_card/ https://arxiv.org/abs/2412.05185 From 3fe8cabeceb7fd8ad4f4d5405d265de7ffe0f7a8 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 3 Jan 2025 20:24:46 -0800 Subject: [PATCH 84/88] f --- .../Gradio_UI/Character_Chat_tab.py | 84 +++++++------- App_Function_Libraries/LLM_API_Calls.py | 3 +- Docs/Design/Researcher.md | 1 + Docs/Design/TTS_STT.md | 15 ++- Docs/Issues/ToDo.md | 106 +++++++++++++++++- 5 files changed, 168 insertions(+), 41 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index ab2285514..7b24e2719 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -434,28 +434,6 @@ def import_chat_history(file, current_history, char_data, user_name_val): return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully." - # def load_character(name): - # characters = get_character_cards() - # character = next((char for char in characters if char['name'] == name), None) - # if character: - # first_message = character.get('first_message', "Hello! I'm ready to chat.") - # return character, [(None, first_message)] if first_message else [], None - # return None, [], None - # - # def load_character_image(name): - # character = next((char for char in get_character_cards() if char['name'] == name), None) - # if character and 'image' in character and character['image']: - # try: - # # Decode the base64 image - # image_data = base64.b64decode(character['image']) - # # Load as PIL Image - # img = Image.open(io.BytesIO(image_data)).convert("RGBA") - # return img - # except Exception as e: - # logging.error(f"Error loading image for character '{name}': {e}") - # return None - # return None - def character_chat_wrapper( message, history, char_data, api_endpoint, api_key, temperature, user_name_val, auto_save, streaming, minp, maxp @@ -463,6 +441,21 @@ def character_chat_wrapper( if not char_data: return history, "Please select a character first." + # Sanitize the initial history to ensure no None values + sanitized_history = [] + for entry in history: + if entry is None: + sanitized_history.append(("", "")) # Replace None with an empty tuple + elif isinstance(entry, (list, tuple)) and len(entry) == 2: + # Ensure both elements are strings + user_msg = entry[0] if entry[0] is not None else "" + bot_msg = entry[1] if entry[1] is not None else "" + sanitized_history.append((user_msg, bot_msg)) + else: + # If the entry is invalid, replace it with an empty tuple + sanitized_history.append(("", "")) + history = sanitized_history + user_name_val = user_name_val or "User" char_name = char_data.get('name', 'AI Assistant') @@ -512,22 +505,36 @@ def character_chat_wrapper( maxp=maxp ) - # Replace placeholders in bot message - bot_message = replace_placeholders(bot_message, char_name, user_name_val) + # Handle streaming response + if streaming: + # For streaming, append the user message to the history with a placeholder for the bot's response + history.append((user_message, "")) # Append user message with an empty bot response + logging.debug(f"Updated history (streaming): {history}") - # Update history - history.append((user_message, bot_message)) + # Yield the bot's response incrementally + def stream_response(): + for chunk in bot_message: + # Update the last entry in the history with the partial response + history[-1] = (user_message, chunk) + yield history, "" # History and empty status message - # Auto-save if enabled - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." + return stream_response() + else: + # For non-streaming, append the full bot response to the history + bot_message = replace_placeholders(bot_message, char_name, user_name_val) + history.append((user_message, bot_message)) + logging.debug(f"Updated history (non-streaming): {history}") + + # Auto-save if enabled + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." return history, save_status @@ -1141,12 +1148,13 @@ def speak_last_response(chat_history): temperature_slider, user_name_input, auto_save_checkbox, + streaming, minp_slider, maxp_slider ], - outputs=[chat_history, save_status] + outputs=[chat_history, save_status] # Output to both history and chatbot ).then( - lambda: "", outputs=user_input + lambda: "", outputs=user_input # Clear the input box after sending ).then( lambda history: approximate_token_count(history), inputs=[chat_history], diff --git a/App_Function_Libraries/LLM_API_Calls.py b/App_Function_Libraries/LLM_API_Calls.py index 9ec91362a..1c8a7e3b8 100644 --- a/App_Function_Libraries/LLM_API_Calls.py +++ b/App_Function_Libraries/LLM_API_Calls.py @@ -145,7 +145,8 @@ def chat_with_openai(api_key, input_data, custom_prompt_arg, temp, system_messag raise ValueError(f"Invalid type for 'streaming': Expected a boolean, got {type(streaming).__name__}") if maxp is None: - maxp = loaded_config_data['openai_api']['max_p'] + maxp = loaded_config_data['openai_api']['top_p'] + maxp = float(maxp) if model is None: openai_model = loaded_config_data['openai_api']['model'] or "gpt-4o" logging.debug(f"OpenAI: Using model: {openai_model}") diff --git a/Docs/Design/Researcher.md b/Docs/Design/Researcher.md index f4f467ea9..c4203c00f 100644 --- a/Docs/Design/Researcher.md +++ b/Docs/Design/Researcher.md @@ -147,6 +147,7 @@ https://github.com/rashadphz/farfalle/tree/main/src/backend https://github.com/rashadphz/farfalle/blob/main/src/backend/agent_search.py https://github.com/rashadphz/farfalle/blob/main/src/backend/prompts.py +https://github.com/stanford-oval/storm/ https://github.com/stanford-oval/storm/blob/main/examples/storm_examples/run_storm_wiki_claude.py https://learning.google.com/experiments/learn-about diff --git a/Docs/Design/TTS_STT.md b/Docs/Design/TTS_STT.md index 58f2293aa..7ac47c0cc 100644 --- a/Docs/Design/TTS_STT.md +++ b/Docs/Design/TTS_STT.md @@ -50,6 +50,14 @@ https://github.com/dnhkng/GlaDOS https://arxiv.org/abs/2501.01384 https://sharechatx.github.io/ https://github.com/vipchengrui/traditional-speech-enhancement +https://github.com/ictnlp/LLaMA-Omni +https://github.com/lamm-mit/PDF2Audio +https://github.com/Purfview/whisper-standalone-win +https://github.com/livekit/agents +https://huggingface.co/papers/2410.02678 +https://github.com/Picovoice/speech-to-text-benchmark +https://huggingface.co/spaces/bencser/episodegen + STT @@ -191,9 +199,14 @@ https://docs.inferless.com/cookbook/serverless-customer-service-bot https://huggingface.co/spaces/lamm-mit/PDF2Audio https://huggingface.co/spaces/bencser/episodegen - +https://github.com/myshell-ai/MeloTTS +https://github.com/idiap/coqui-ai-TTS +https://docs.inferless.com/cookbook/serverless-customer-service-bot https://github.com/Picovoice/speech-to-text-benchmark +Train using: https://github.com/Mangio621/Mangio-RVC-Fork/releases, +import the .pth into https://huggingface.co/wok000/vcclient000/tree/main to convert your voice in near real time with about a .25s delay +https://www.hackster.io/lhl/voicechat2-local-ai-voice-chat-4c48f2 https://huggingface.co/papers/2410.02678 https://github.com/livekit/agents diff --git a/Docs/Issues/ToDo.md b/Docs/Issues/ToDo.md index 460d4bd13..9b363380b 100644 --- a/Docs/Issues/ToDo.md +++ b/Docs/Issues/ToDo.md @@ -39,4 +39,108 @@ List of stuff I don't feel like filing an issue for (yet). https://ai.gopubby.com/demystifying-pdf-parsing-06-representative-industry-solutions-5d4a1cfe311b - Prompts - - https://github.com/ProfSynapse/Synapse_CoR \ No newline at end of file + - https://github.com/ProfSynapse/Synapse_CoR + + +Links to sort +https://github.com/dylanashley/story-distiller +https://ieeexplore.ieee.org/document/10734853 +https://towardsdatascience.com/agentic-chunking-for-rags-091beccd94b1 +https://www.youtube.com/watch?v=NVp9jiMDdXc + +https://huggingface.co/NeuML/pubmedbert-base-embeddings +https://huggingface.co/datasets/qiaojin/PubMedQA +https://huggingface.co/datasets/armanc/scientific_papers +https://www.notion.com/blog/how-we-sped-up-notion-in-the-browser-with-wasm-sqlite +https://alexgarcia.xyz/blog/2024/sqlite-vec-hybrid-search/index.html +https://needle-threading.github.io/ +https://vitron-llm.github.io/ +https://huggingface.co/papers/2411.04335 +https://github.com/severian42/Cascade-of-Semantically-Integrated-Layers +https://github.com/THUDM/LongReward +RAG +https://towardsdatascience.com/improve-your-rag-context-recall-by-40-with-an-adapted-embedding-model-5d4a8f583f32 +https://towardsdatascience.com/beyond-na%C3%AFve-rag-advanced-techniques-for-building-smarter-and-reliable-ai-systems-c4fbcf8718b8 +https://ai.gopubby.com/rag-on-steroids-self-rag-69505f9d8238 +https://github.com/dongguanting/DPA-RAG +https://arxiv.org/html/2411.02959v1 +https://arxiv.org/pdf/2409.05591 + +UI +https://ui.shadcn.com/docs +https://www.typingmind.com/ + +ASR +https://github.com/revdotcom/reverb + +Hallucinations +https://arxiv.org/abs/2410.22071 + +Scraping +https://github.com/devflowinc/firecrawl-simple + +Prompts +https://colab.research.google.com/drive/1fw7ge47ymnznsz3rWlXVcyPC9PKk6_xH#scrollTo=-9mw9XLfj_vD + +Finetuning +https://towardsdatascience.com/i-fine-tuned-the-tiny-llama-3-2-1b-to-replace-gpt-4o-7ce1e5619f3d + +Smaller models +https://towardsdatascience.com/leveraging-smaller-llms-for-enhanced-retrieval-augmented-generation-rag-bc320e71223d + +Multimodal +https://magazine.sebastianraschka.com/p/understanding-multimodal-llms?utm_source=substack&utm_medium=email +https://arxiv.org/abs/2410.21943 + +Ranking +https://archive.is/hXXK6 + +Evals +https://towardsdatascience.com/llm-evaluation-techniques-and-costs-3147840afc53 +https://archive.is/IA4UR +https://github.com/huggingface/lighteval +https://github.com/tianyi-lab/BenTo + +Prompts +https://huggingface.co/spaces/baconnier/prompt-plus-plus +https://github.com/google-deepmind/opro + + +https://arxiv.org/pdf/2410.11795 +https://dynamic-city.github.io/ +https://generative-infinite-game.github.io/ +https://arxiv.org/abs/2410.18417 +https://arxiv.org/html/2410.18745v1 +https://arxiv.org/html/2410.13293v1 +https://huggingface.co/papers/2410.13852 +https://huggingface.co/papers/2410.09584 +https://huggingface.co/papers/2410.10813 +https://arxiv.org/pdf/2409.12640 +https://huggingface.co/papers/2410.06634 +https://huggingface.co/papers/2410.10594 +https://github.com/ShayekhBinIslam/openrag +https://huggingface.co/papers/2410.08815 +https://huggingface.co/papers/2408.11875 +https://huggingface.co/papers/2409.19753 +https://arxiv.org/abs/2410.08815 +https://arxiv.org/abs/2410.07035 +https://arxiv.org/abs/2311.04954 +https://lmql.ai/ +https://artificialintelligencemadesimple.substack.com/p/how-amazon-is-rethinking-human-evaluation +https://arxiv.org/abs/2410.08037 +https://github.com/Rolandjg/skool4free +https://arxiv.org/abs/2410.05229 +https://arxiv.org/abs/2410.07176 +https://arxiv.org/abs/2410.05983 +https://huggingface.co/papers/2410.04199 +https://arxiv.org/html/2409.14924v1 +https://huggingface.co/papers/2407.13101 +https://huggingface.co/spaces/Xenova/the-tokenizer-playground +https://github.com/ml-jku/RA-DT +https://arxiv.org/abs/2410.07071 +https://huggingface.co/papers/2410.03017 +https://huggingface.co/papers/2409.18943 +https://arxiv.org/pdf/2409.16493 + + + From 4af4c682c7f834adc61e75687aeafd7a445ed96b Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 3 Jan 2025 20:45:23 -0800 Subject: [PATCH 85/88] Update Character_Chat_tab.py --- .../Gradio_UI/Character_Chat_tab.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index 7b24e2719..c50cadbd2 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -507,18 +507,24 @@ def character_chat_wrapper( # Handle streaming response if streaming: - # For streaming, append the user message to the history with a placeholder for the bot's response history.append((user_message, "")) # Append user message with an empty bot response - logging.debug(f"Updated history (streaming): {history}") + full_response = "" + for chunk in bot_message: + full_response += chunk + history[-1] = (user_message, full_response) + yield history, "" # Yield updated history and empty status - # Yield the bot's response incrementally - def stream_response(): - for chunk in bot_message: - # Update the last entry in the history with the partial response - history[-1] = (user_message, chunk) - yield history, "" # History and empty status message - - return stream_response() + # After streaming is complete, handle auto-save + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + yield history, save_status else: # For non-streaming, append the full bot response to the history bot_message = replace_placeholders(bot_message, char_name, user_name_val) @@ -536,7 +542,7 @@ def stream_response(): else: save_status = "Character ID not found; chat not saved." - return history, save_status + return history, save_status def validate_chat_history(chat_history: List[Tuple[Optional[str], str]]) -> bool: """ @@ -1152,7 +1158,7 @@ def speak_last_response(chat_history): minp_slider, maxp_slider ], - outputs=[chat_history, save_status] # Output to both history and chatbot + outputs=[chat_history, save_status], ).then( lambda: "", outputs=user_input # Clear the input box after sending ).then( From 5677b322d836751955689d19edf30d79e13689fa Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 3 Jan 2025 20:53:11 -0800 Subject: [PATCH 86/88] Update Character_Chat_tab.py --- .../Gradio_UI/Character_Chat_tab.py | 124 +++++++++++++++--- 1 file changed, 107 insertions(+), 17 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index c50cadbd2..883eff04a 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -768,20 +768,41 @@ def regenerate_last_message( maxp=maxp ) - # Append the new bot message to the history - new_history.append((last_user_message, bot_message)) + # Handle streaming response + if streaming: + new_history.append((last_user_message, "")) # Append user message with an empty bot response + full_response = "" + for chunk in bot_message: + full_response += chunk + new_history[-1] = (last_user_message, full_response) + yield new_history, "" # Yield updated history and empty status - # Auto-save if enabled - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, new_history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." + # After streaming is complete, handle auto-save + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, new_history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + yield new_history, save_status else: + # For non-streaming, append the full bot response to the history + bot_message = replace_placeholders(bot_message, char_name, user_name_val) + new_history.append((last_user_message, bot_message)) + + # Auto-save if enabled save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, new_history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." return new_history, save_status @@ -841,11 +862,12 @@ def update_chat(chat_id, updated_history): def continue_talking( history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save, minp, maxp + temperature, user_name_val, auto_save, streaming_checkbox, minp, maxp ): """ Causes the character to continue the conversation or think out loud. """ + streaming = streaming_checkbox.value if hasattr(streaming_checkbox, 'value') else streaming_checkbox if not char_data: return history, "Please select a character first." @@ -896,8 +918,29 @@ def continue_talking( maxp=maxp ) - # Replace placeholders in bot message - bot_message = replace_placeholders(bot_message, char_name, user_name_val) + # Handle streaming response + if streaming: + history.append((None, "")) # Append empty user message with an empty bot response + full_response = "" + for chunk in bot_message: + full_response += chunk + history[-1] = (None, full_response) + yield history, "" # Yield updated history and empty status + + # After streaming is complete, handle auto-save + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + yield history, save_status + else: + # Replace placeholders in bot message + bot_message = replace_placeholders(bot_message, char_name, user_name_val) # Update history history.append((None, bot_message)) @@ -917,10 +960,11 @@ def continue_talking( def answer_for_me( history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save, minp, maxp): + temperature, user_name_val, auto_save, streaming_checkbox, minp, maxp): """ Generates a likely user response and continues the conversation. """ + streaming = streaming_checkbox.value if hasattr(streaming_checkbox, 'value') else streaming_checkbox if not char_data: return history, "Please select a character first." @@ -965,8 +1009,52 @@ def answer_for_me( maxp=maxp ) - # Append the generated user response to history - history.append((user_response, None)) + if streaming: + history.append(("", "")) # Append empty user message + full_user_response = "" + for chunk in user_response: + full_user_response += chunk + history[-1] = (full_user_response, "") + yield history, "" # Yield updated history and empty status + + # Now generate the character's response to this user response + system_message_bot = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}""" + + bot_message = chat( + f"{user_name_val}: {full_user_response}", + history[:-1], + media_content, + selected_parts, + api_endpoint, + api_key, + prompt=char_data.get('post_history_instructions', ''), + temperature=temperature, + system_message=system_message_bot, + streaming=streaming, + minp=minp, + maxp=maxp + ) + + full_bot_response = "" + for chunk in bot_message: + full_bot_response += chunk + history[-1] = (full_user_response, full_bot_response) + yield history, "" # Yield updated history and empty status + + # After streaming is complete, handle auto-save + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + yield history, save_status + else: + # Append the generated user response to history + history.append((user_response, None)) # Now generate the character's response to this user response # Prepare the system message for the character @@ -1081,6 +1169,7 @@ def speak_last_response(chat_history): temperature_slider, user_name_input, auto_save_checkbox, + streaming, minp_slider, maxp_slider ], @@ -1101,6 +1190,7 @@ def speak_last_response(chat_history): temperature_slider, user_name_input, auto_save_checkbox, + streaming, minp_slider, maxp_slider ], From 872d6ad16cc459177ac391ab29ae9fb930ec6d3c Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 3 Jan 2025 21:11:49 -0800 Subject: [PATCH 87/88] finallly --- .../Gradio_UI/Character_Chat_tab.py | 220 ++++++++---------- 1 file changed, 103 insertions(+), 117 deletions(-) diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index 883eff04a..f73797f48 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -685,8 +685,7 @@ def clear_chat_history(char_data, user_name_val): def regenerate_last_message( history, char_data, api_endpoint, api_key, - temperature, user_name_val, auto_save, streaming, minp, maxp - ): + temperature, user_name_val, auto_save, streaming_checkbox, minp, maxp): """ Regenerates the last bot message by removing it and resending the corresponding user message. @@ -702,109 +701,106 @@ def regenerate_last_message( Returns: tuple: Updated chat history and a save status message. """ - if not history: - return history, "No messages to regenerate." - - last_entry = history[-1] - last_user_message, last_bot_message = last_entry - - # Check if the last bot message exists - if last_bot_message is None: - return history, "The last message is not from the bot." - - # Remove the last bot message - new_history = history[:-1] - - # Resend the last user message to generate a new bot response - if not last_user_message: - return new_history, "No user message to regenerate the bot response." - - # Prepare the character's background information - char_name = char_data.get('name', 'AI Assistant') - char_background = f""" - Name: {char_name} - Description: {char_data.get('description', 'N/A')} - Personality: {char_data.get('personality', 'N/A')} - Scenario: {char_data.get('scenario', 'N/A')} - """ - - # Prepare the system prompt for character impersonation - system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name_val}". - - {char_background} - - Additional instructions: {char_data.get('post_history_instructions', '')} - """ - - # Prepare media_content and selected_parts - media_content = { - 'id': char_name, - 'title': char_name, - 'content': char_background, - 'description': char_data.get('description', ''), - 'personality': char_data.get('personality', ''), - 'scenario': char_data.get('scenario', '') - } - selected_parts = ['description', 'personality', 'scenario'] - - prompt = char_data.get('post_history_instructions', '') - - # Prepare the input for the chat function - full_message = f"{user_name_val}: {last_user_message}" if last_user_message else f"{user_name_val}: " - - # Call the chat function to get a new bot message - bot_message = chat( - full_message, - new_history, - media_content, - selected_parts, - api_endpoint, - api_key, - prompt, - temperature, - system_message, - streaming, - minp=minp, - maxp=maxp - ) + try: + streaming = streaming_checkbox.value if hasattr(streaming_checkbox, 'value') else streaming_checkbox + if not history: + return history, "No messages to regenerate." + + last_entry = history[-1] + last_user_message, last_bot_message = last_entry + + # Check if the last bot message exists + if last_bot_message is None: + return history, "The last message is not from the bot." + + # Remove the last bot message + new_history = history[:-1] + + # Resend the last user message to generate a new bot response + if not last_user_message: + return new_history, "No user message to regenerate the bot response." + + # Prepare the character's background information + char_name = char_data.get('name', 'AI Assistant') + char_background = f""" + Name: {char_name} + Description: {char_data.get('description', 'N/A')} + Personality: {char_data.get('personality', 'N/A')} + Scenario: {char_data.get('scenario', 'N/A')} + """ + + # Prepare the system prompt for character impersonation + system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name_val}". + + {char_background} + + Additional instructions: {char_data.get('post_history_instructions', '')} + """ + + # Prepare media_content and selected_parts + media_content = { + 'id': char_name, + 'title': char_name, + 'content': char_background, + 'description': char_data.get('description', ''), + 'personality': char_data.get('personality', ''), + 'scenario': char_data.get('scenario', '') + } + selected_parts = ['description', 'personality', 'scenario'] + + prompt = char_data.get('post_history_instructions', '') + + # Prepare the input for the chat function + full_message = f"{user_name_val}: {last_user_message}" if last_user_message else f"{user_name_val}: " + + # Call the chat function to get a new bot message + bot_message = chat( + full_message, + new_history, + media_content, + selected_parts, + api_endpoint, + api_key, + prompt, + temperature, + system_message, + streaming, + minp=minp, + maxp=maxp + ) - # Handle streaming response - if streaming: - new_history.append((last_user_message, "")) # Append user message with an empty bot response - full_response = "" - for chunk in bot_message: - full_response += chunk - new_history[-1] = (last_user_message, full_response) - yield new_history, "" # Yield updated history and empty status + # Handle streaming response + if streaming: + full_response = "" + for chunk in bot_message: + full_response += chunk + yield new_history + [(last_user_message, full_response)], "" # Yield updated history and empty status - # After streaming is complete, handle auto-save - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, new_history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." - yield new_history, save_status - else: - # For non-streaming, append the full bot response to the history - bot_message = replace_placeholders(bot_message, char_name, user_name_val) - new_history.append((last_user_message, bot_message)) + # After streaming is complete, update history and handle auto-save + new_history.append((last_user_message, full_response)) + else: + # Replace placeholders in bot message + full_response = replace_placeholders(bot_message, char_name, user_name_val) + # Update history + new_history.append((last_user_message, full_response)) + + # Auto-save if enabled + save_status = "" + if auto_save: + character_id = char_data.get('id') + if character_id: + conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" + add_character_chat(character_id, conversation_name, new_history) + save_status = "Chat auto-saved." + else: + save_status = "Character ID not found; chat not saved." + + return new_history, save_status - # Auto-save if enabled - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, new_history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." + except Exception as e: + save_status = f"Error regenerating message: {str(e)}" + return history, save_status # Return original history if an error occurs - return new_history, save_status def toggle_chat_file_upload(): return gr.update(visible=True) @@ -924,26 +920,16 @@ def continue_talking( full_response = "" for chunk in bot_message: full_response += chunk - history[-1] = (None, full_response) - yield history, "" # Yield updated history and empty status + yield history + [(None, full_response)], "" # Yield updated history and empty status - # After streaming is complete, handle auto-save - save_status = "" - if auto_save: - character_id = char_data.get('id') - if character_id: - conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" - add_character_chat(character_id, conversation_name, history) - save_status = "Chat auto-saved." - else: - save_status = "Character ID not found; chat not saved." - yield history, save_status + # After streaming is complete, update history and handle auto-save + history.append((None, full_response)) else: # Replace placeholders in bot message - bot_message = replace_placeholders(bot_message, char_name, user_name_val) + full_response = replace_placeholders(bot_message, char_name, user_name_val) # Update history - history.append((None, bot_message)) + history.append((None, full_response)) # Auto-save if enabled save_status = "" @@ -1273,7 +1259,7 @@ def speak_last_response(chat_history): ], outputs=[chat_history, save_status] ).then( - lambda history: approximate_token_count(history), + lambda history: approximate_token_count(history) if history is not None else 0, inputs=[chat_history], outputs=[token_count_display] ) From 5e2c2d540928d931e8fdcd9af58aca2ece84e9d7 Mon Sep 17 00:00:00 2001 From: Robert Date: Fri, 3 Jan 2025 21:18:36 -0800 Subject: [PATCH 88/88] BREAKING CONFIG CHANGE!!! --- App_Function_Libraries/Chat/Chat_Functions.py | 22 +++--- .../Gradio_UI/Character_Chat_tab.py | 4 - Config_Files/Backup_Config.txt | 38 +++++++--- Docs/Design/WebSearch.md | 76 +++++++++++++++++++ 4 files changed, 114 insertions(+), 26 deletions(-) diff --git a/App_Function_Libraries/Chat/Chat_Functions.py b/App_Function_Libraries/Chat/Chat_Functions.py index 4dd09e804..aeea31071 100644 --- a/App_Function_Libraries/Chat/Chat_Functions.py +++ b/App_Function_Libraries/Chat/Chat_Functions.py @@ -30,14 +30,18 @@ # Functions: def approximate_token_count(history): - total_text = '' - for user_msg, bot_msg in history: - if user_msg: - total_text += user_msg + ' ' - if bot_msg: - total_text += bot_msg + ' ' - total_tokens = len(total_text.split()) - return total_tokens + try: + total_text = '' + for user_msg, bot_msg in history: + if user_msg: + total_text += user_msg + ' ' + if bot_msg: + total_text += bot_msg + ' ' + total_tokens = len(total_text.split()) + return total_tokens + except Exception as e: + logging.error(f"Error calculating token count: {str(e)}") + return 0 # FIXME - add model parameter @@ -384,7 +388,6 @@ def update_chat_content(selected_item, use_content, use_summary, use_prompt, ite CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.json') - def save_character(character_data): log_counter("save_character_attempt") start_time = time.time() @@ -445,7 +448,6 @@ def load_characters(): return {} - def get_character_names(): log_counter("get_character_names_attempt") start_time = time.time() diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index f73797f48..ebb498bc3 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -801,10 +801,6 @@ def regenerate_last_message( save_status = f"Error regenerating message: {str(e)}" return history, save_status # Return original history if an error occurs - - def toggle_chat_file_upload(): - return gr.update(visible=True) - def save_untracked_chat_action(history, char_data): if not char_data or not history: return "No chat to save or character not selected." diff --git a/Config_Files/Backup_Config.txt b/Config_Files/Backup_Config.txt index e56be9be8..eb5b40e56 100644 --- a/Config_Files/Backup_Config.txt +++ b/Config_Files/Backup_Config.txt @@ -3,6 +3,8 @@ anthropic_api_key = anthropic_model = claude-3-5-sonnet-20240620 anthropic_streaming = True anthropic_temperature = 0.7 +anthropic_top_p = 0.95 +anthropic_min_p = 0.05 # cohere_api_key = cohere_model = command-r-plus @@ -18,6 +20,7 @@ openai_api_key = openai_model = gpt-4o openai_streaming = False openai_temperature = 0.7 +openai_top_p = 0.95 # huggingface_api_key = huggingface_model = meta-llama/Llama-3.1-70B-Instruct @@ -142,16 +145,17 @@ default_openai_tts_output_format = mp3 # # ElevenLabs TTS Settings default_eleven_tts_voice = pNInz6obpgDQGcFmaJgB -default_eleven_tts_model = FIXME -default_eleven_tts_language_code = FIXME -default_eleven_tts_voice_stability = FIXME -default_eleven_tts_voice_similiarity_boost = FIXME -default_eleven_tts_voice_style = FIXME -default_eleven_tts_voice_use_speaker_boost = FIXME -default_eleven_tts_voice_pronunciation_dictionary_locators_dict_id = FIXME -default_eleven_tts_voice_pronunciation_dictionary_locators_version_id = FIXME +default_eleven_tts_model = +default_eleven_tts_language_code = +default_eleven_tts_voice_stability = +default_eleven_tts_voice_similiarity_boost = +default_eleven_tts_voice_style = +default_eleven_tts_voice_use_speaker_boost = +default_eleven_tts_voice_pronunciation_dictionary_locators_dict_id = +default_eleven_tts_voice_pronunciation_dictionary_locators_version_id = default_eleven_tts_speed = 1 -default_eleven_tts_output_format = FIXME +# Output options: 'mp3_22050_32', 'mp3_44100_32', 'mp3_44100_64', 'mp3_44100_96', 'mp3_44100_128', 'mp3_44100_192', 'pcm_16000', 'pcm_22050', 'pcm_24000', 'pcm_44100', 'ulaw_8000' +default_eleven_tts_output_format = mp3_44100_128 # Google TTS Settings default_google_tts_model = FIXME default_google_tts_voice = FIXME @@ -160,6 +164,14 @@ default_google_tts_speed = 1 # MS Edge TTS Settings edge_tts_voice = FIXME # +# GPT-Sovits +# +# AllTalk TTS Settings +default_alltalk_tts_speed = 1.0 +default_alltalk_tts_voice = alloy +default_alltalk_tts_model = alltalk +default_alltalk_tts_output_format = mp3 +alltalk_api_ip = http://127.0.0.1:7851/v1/audio/speech [Search-Engines] # Search Defaults @@ -179,9 +191,11 @@ search_result_display_metadata = False search_result_save_to_db = True # How you want the results to be written, think 'style' or voice search_result_analysis_tone = +relevance_analysis_llm = openai +final_answer_llm = openai #### Search Engines ##### # Baidu -search_engine_api_key_baidu = > +search_engine_api_key_baidu = 1e1b1b1b1b1b1b1b1 # # Bing search_engine_api_url_bing = https://api.bing.microsoft.com/v7.0/search @@ -224,8 +238,8 @@ search_engine_searx_api = https://search.rhscz.eu/ # Tavily search_engine_api_key_tavily = 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 # Yandex -search_engine_api_key_yandex = > -search_engine_id_yandex = > +search_engine_api_key_yandex = 1e1b1b1b1b1b1b1b1 +search_engine_id_yandex = 1e1b1b1b1b1b1b1b1 [Prompts] prompt_sample = "What is the meaning of life?" diff --git a/Docs/Design/WebSearch.md b/Docs/Design/WebSearch.md index 183116f3f..79f0a140b 100644 --- a/Docs/Design/WebSearch.md +++ b/Docs/Design/WebSearch.md @@ -265,3 +265,79 @@ Structure/approach taken from https://github.com/deedy5/duckduckgo_search #### PubMedCentral Search - https://www.ncbi.nlm.nih.gov/home/develop/api/ + + + +### Prompts used: + +sub_question_generation_prompt = +``` +You are an AI assistant that helps generate search queries. Given an original query, suggest alternative search queries that could help find relevant information. Your goal is to generate queries that are diverse, specific, and highly relevant to the original query, ensuring comprehensive coverage of the topic. + Important instructions: + 1. Generate between 2 and 6 queries unless a fixed count is specified. Generate more queries for complex or multifaceted topics and fewer for simple or straightforward ones. + 2. Ensure the queries are diverse, covering different aspects or perspectives of the original query, while remaining highly relevant to its core intent. + 3. Prefer specific queries over general ones, as they are more likely to yield targeted and useful results. + 4. If the query involves comparing two topics, generate separate queries for each topic. + 5. If previous queries and an answer are provided, generate new queries that address the shortcomings of the previous answer and avoid repeating the previous queries. + 6. If the original query is broad or ambiguous, generate queries that explore specific subtopics or clarify the intent. + 7. If the query is too specific or unclear, generate queries that explore related or broader topics to ensure useful results. + 8. Return the queries as a JSON array in the format ["query_1", "query_2", ...]. + Examples: + 1. For the query "What are the benefits of exercise?", generate queries like: + ["health benefits of physical activity", "mental health benefits of exercise", "long-term effects of regular exercise", "how exercise improves cardiovascular health", "role of exercise in weight management"] + 2. For the query "Compare Python and JavaScript", generate queries like: + ["key features of Python programming language", "advantages of JavaScript for web development", "use cases for Python vs JavaScript", "performance comparison of Python and JavaScript", "ease of learning Python vs JavaScript"] + 3. For the query "How does climate change affect biodiversity?", generate queries like: + ["impact of climate change on species extinction", "effects of global warming on ecosystems", "role of climate change in habitat loss", "how rising temperatures affect marine biodiversity", "climate change and its impact on migratory patterns"] + 4. For the query "Best practices for remote work", generate queries like: + ["tips for staying productive while working from home", "how to maintain work-life balance in remote work", "tools for effective remote team collaboration", "managing communication in remote teams", "ergonomic setup for home offices"] + 5. For the query "What is quantum computing?", generate queries like: + ["basic principles of quantum computing", "applications of quantum computing in real-world problems", "difference between classical and quantum computing", "key challenges in developing quantum computers", "future prospects of quantum computing"] + Original query: {original_query} +# +search_result_relevance_eval_prompt = Given the following search results for the user's question: "{original_question}" and the generated sub-questions: {sub_questions}, evaluate the relevance of the search result to the user's question. + Explain your reasoning for selection. + Search Results: + {content} + Instructions: + 1. You MUST only answer TRUE or False while providing your reasoning for your answer. + 2. A result is relevant if the result most likely contains comprehensive and relevant information to answer the user's question. + 3. Provide a brief reason for selection. + You MUST respond using EXACTLY this format and nothing else: + Selected Answer: [True or False] + Reasoning: [Your reasoning for the selections] + +``` + +analyze_search_results_prompt = +``` +Generate a comprehensive, well-structured, and informative answer for a given question, + using ONLY the information found in the provided web Search Results (URL, Page Title, Summary). + Use an unbiased, journalistic tone, adapting the level of formality to match the user’s question. + • Cite your statements using [number] notation, placing citations at the end of the relevant sentence. + • Only cite the most relevant results. If multiple sources support the same point, cite all relevant sources [e.g., 1, 2, 3]. + • If sources conflict, present both perspectives clearly and cite the respective sources. + • If different sources refer to different entities with the same name, provide separate answers. + • Do not add any external or fabricated information. + • Do not include URLs or a reference section; cite inline with [number] format only. + • Do not repeat the question or include unnecessary redundancy. + • Use markdown formatting (e.g., **bold**, bullet points, ## headings) to organize the information. + • If the provided results are insufficient to answer the question, explicitly state what information is missing or unclear. + Structure your answer like this: + 1. **Short introduction**: Briefly summarize the topic (1–2 sentences). + 2. **Bulleted points**: Present key details, each with appropriate citations. + 3. **Conclusion**: Summarize the findings or restate the core answer (with citations if needed). + Example: + 1. **Short introduction**: This topic explores the impact of climate change on agriculture. + 2. **Bulleted points**: + - Rising temperatures have reduced crop yields in some regions [1]. + - Changes in rainfall patterns are affecting irrigation practices [2, 3]. + 3. **Conclusion**: Climate change poses significant challenges to global agriculture [1, 2, 3]. + + {concatenated_texts} + + --------------------- + Make sure to match the language of the user's question. + Question: {question} + Answer (in the language of the user's question): +```