docs: add example of Dataset.insert (#3534)

* Generate API docs automatically from plugin
* Add example of `LanceDataset.insert()` and `write_dataset`

5/N of #2423

---------

Co-authored-by: Will Jones <willjones127@gmail.com>

Author: eddyxu

docs/api/api.rst

@@ -2,6 +2,7 @@ APIs
 ----
 
 .. toctree::
+   :maxdepth: 1
 
-  Rust <https://docs.rs/crate/lance/latest>
-  Python <./python.rst>
+   Rust <https://docs.rs/crate/lance/latest>
+   Python <./python.rst>

docs/conf.py

@@ -1,18 +1,5 @@
 # Configuration file for the Sphinx documentation builder.
 
-import shutil
-
-
-def run_apidoc(_):
-    from sphinx.ext.apidoc import main
-
-    shutil.rmtree("api/python", ignore_errors=True)
-    main(["-f", "-o", "api/python", "../python/python/lance"])
-
-
-def setup(app):
-    app.connect("builder-inited", run_apidoc)
-
 
 # -- Project information -----------------------------------------------------
 
@@ -29,6 +16,7 @@ def setup(app):
 extensions = [
     "breathe",
     "sphinx_immaterial",
+    "sphinx_immaterial.apidoc.python.apigen",
     "sphinx.ext.autodoc",
     "sphinx.ext.doctest",
     "sphinx.ext.githubpages",
@@ -58,6 +46,19 @@ def setup(app):
     "ray": ("https://docs.ray.io/en/latest/", None),
 }
 
+python_apigen_modules = {
+    "lance": "api/python/",
+}
+object_description_options = [
+    (
+        "py:.*",
+        dict(
+            include_object_type_in_xref_tooltip=False,
+            include_in_toc=False,
+            include_fields_in_toc=False,
+        ),
+    ),
+]
 
 # -- Options for HTML output -------------------------------------------------
 
@@ -96,7 +97,7 @@ def setup(app):
         },
     ],
 }
-include_in_toc = False
+
 
 # -- doctest configuration ---------------------------------------------------
 

docs/introduction/read_and_write.rst

@@ -50,6 +50,43 @@ You will need to provide a :py:class:`pyarrow.Schema` for the dataset in this ca
 :py:meth:`lance.write_dataset` supports writing :py:class:`pyarrow.Table`, :py:class:`pandas.DataFrame`,
 :py:class:`pyarrow.dataset.Dataset`, and ``Iterator[pyarrow.RecordBatch]``.
 
+Adding Rows
+-----------
+
+To insert data into your dataset, you can use either :py:meth:`LanceDataset.insert <lance.LanceDataset.insert>`
+or :py:meth:`~lance.write_dataset` with ``mode=append``.
+
+.. testsetup::
+
+  shutil.rmtree("./insert_example.lance", ignore_errors=True)
+
+.. doctest::
+
+  >>> import lance
+  >>> import pyarrow as pa
+
+  >>> table = pa.Table.from_pylist([{"name": "Alice", "age": 20},
+  ...                               {"name": "Bob", "age": 30}])
+  >>> ds = lance.write_dataset(table, "./insert_example.lance")
+
+  >>> new_table = pa.Table.from_pylist([{"name": "Carla", "age": 37}])
+  >>> ds.insert(new_table)
+  >>> ds.to_table().to_pandas()
+      name  age
+  0  Alice   20
+  1    Bob   30
+  2  Carla   37
+
+  >>> new_table2 = pa.Table.from_pylist([{"name": "David", "age": 42}])
+  >>> ds = lance.write_dataset(new_table2, ds, mode="append")
+  >>> ds.to_table().to_pandas()
+      name  age
+  0  Alice   20
+  1    Bob   30
+  2  Carla   37
+  3  David   42
+
+
 Deleting rows
 -------------
 
@@ -123,7 +160,7 @@ more efficient to use the merge insert operation described below.
     dataset.update({"age": new_age}, where=f"name='{name}'")
 
 Merge Insert
-~~~~~~~~~~~~
+------------
 
 Lance supports a merge insert operation.  This can be used to add new data in bulk
 while also (potentially) matching against existing data.  This operation can be used