Merge pull request #31 from Firdous2307/implement-document-chunking

Implement document chunking

src/rag_functions.py

@@ -7,6 +7,7 @@
 from werkzeug.utils import secure_filename
 from azure.ai.formrecognizer import DocumentAnalysisClient
 from azure.core.credentials import AzureKeyCredential
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 # Setting up logging
 logging.basicConfig(level=logging.INFO)
@@ -54,6 +55,16 @@ def file_check_num(uploaded_file):
         return -1
 
 
+
+def chunk_document(text, chunk_size=1000, chunk_overlap=200):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+    )
+    chunks = text_splitter.split_text(text)
+    return chunks
+
 def extract_contents_from_doc(files, temp_dir):
     """
     Azure Document Intelligence