Improve Algolia indexing

docs/plugins/build_index.py → docs/plugins/algolia.py

@@ -13,6 +13,8 @@
 ALGOLIA_INDEX_NAME = 'logfire-docs'
 ALGOLIA_APP_ID = 'KPPUDTIAVX'
 ALGOLIA_WRITE_API_KEY = os.environ.get('ALGOLIA_WRITE_API_KEY')
+# Algolia accepts 100k, leaaving some room for other fields
+MAX_CONTENT_SIZE = 90_000
 
 
 def on_page_content(html: str, page: Page, config: Config, files: Files) -> str:
@@ -24,6 +26,30 @@ def on_page_content(html: str, page: Page, config: Config, files: Files) -> str:
 
     soup = BeautifulSoup(html, 'html.parser')
 
+    # Clean up presentational and UI elements
+    for element in soup.find_all(['autoref']):
+        element.decompose()
+
+    # this removes the large source code embeds from Github
+    for element in soup.find_all('details'):
+        element.decompose()
+
+    for el_with_class in soup.find_all(class_=['doc-section-item', 'doc-section-title', 'doc-md-description', 'doc']):
+        # delete the class attribute
+        del el_with_class['class']
+
+    # Cleanup code examples
+    for extra in soup.find_all('div', attrs={'class': ['language-py highlight', 'language-python highlight']}):
+        extra.replace_with(BeautifulSoup(f'<pre>{extra.find("code").get_text()}</pre>', 'html.parser'))
+
+    # Cleanup code examples, part 2
+    for extra in soup.find_all('div', attrs={'class': 'language-python doc-signature highlight'}):
+        extra.replace_with(BeautifulSoup(f'<pre>{extra.find("code").get_text()}</pre>', 'html.parser'))
+
+    # The API reference generates HTML tables with line numbers, this strips the line numbers cell and goes back to a code block
+    for extra in soup.find_all('table', attrs={'class': 'highlighttable'}):
+        extra.replace_with(BeautifulSoup(f'<pre>{extra.find("code").get_text()}</pre>', 'html.parser'))
+
     # Find all h1 and h2 headings
     headings = soup.find_all(['h1', 'h2'])
 
@@ -65,7 +91,12 @@ def on_post_build(config: Config) -> None:
 
     client = SearchClient.create(ALGOLIA_APP_ID, ALGOLIA_WRITE_API_KEY)
     index = client.init_index(ALGOLIA_INDEX_NAME)
-    # temporary filter the records from the index if the content is bigger than 10k characters
-    filtered_records = list(filter(lambda record: len(record['content']) < 9000, records))
+
+    for large_record in list(filter(lambda record: len(record['content']) >= MAX_CONTENT_SIZE, records)):
+        print(f'Content for {large_record["abs_url"]} is too large to be indexed. Skipping...')
+        print(f'Content : {large_record["content"]} characters')
+
+    # filter the records from the index if the content is bigger than 10k characters
+    filtered_records = list(filter(lambda record: len(record['content']) < MAX_CONTENT_SIZE, records))
     print(f'Uploading {len(filtered_records)} out of {len(records)} records to Algolia...')
     index.replace_all_objects(filtered_records, {'createIfNotExists': True}).wait()  # type: ignore[reportUnknownMemberType]

docs/plugins/build_llms_txt.py

@@ -10,7 +10,7 @@
 
 
 def on_config(config: MkDocsConfig):
-    os.mkdir(config.site_dir)
+    os.makedirs(config.site_dir, exist_ok=True)
     llms_path = os.path.join(config.site_dir, 'llms.txt')
     with open(llms_path, 'w') as f:
         f.write('')

mkdocs.yml

@@ -309,4 +309,4 @@ plugins:
 hooks:
   - docs/plugins/main.py
   - docs/plugins/build_llms_txt.py
-  - docs/plugins/build_index.py
+  - docs/plugins/algolia.py