a few small updates

build_parse_docs.py

@@ -12,7 +12,7 @@
 
 # Full CLI Reference
 
-This is the reference for the dlt CLI.
+This page contains all commands available in the dlt CLI and is generated automatically from the argparse object.
 
 """
 
@@ -35,8 +35,11 @@ def get_parser_help_recursive(
         parser: argparse.ArgumentParser, cmd: str = "", root: bool = True, nesting: int = 0
     ):
         markdown = ""
+
+        if not parser.description:
+            raise ValueError(f"Parser at command '{cmd}' has no description. This means you have probably added a new parser without adding a description property.")
 
-        heading = 2 * "#"
+        heading = "##" if nesting < 2 else "###"
         markdown += f"{heading} `{cmd}`\n\n"
         markdown += f"{parser.description}\n\n"
         markdown += f"**Help command**\n"
@@ -83,7 +86,7 @@ def get_parser_help_recursive(
         return markdown
 
 if __name__ == "__main__":
-    parser = _build_parser()
+    parser, _ = _build_parser()
 
     with open("docs/website/docs/reference/cli.md", "w") as f:
         f.write(HEADER + get_argparse_help_string("dlt", parser))

dlt/cli/_dlt.py

@@ -1,4 +1,4 @@
-from typing import Any, Sequence, Type, cast, List, Dict
+from typing import Any, Sequence, Type, cast, List, Dict, Tuple
 import argparse
 import click
 
@@ -99,7 +99,7 @@ def __call__(
         debug.enable_debug()
 
 
-def _build_parser() -> argparse.ArgumentParser:
+def _build_parser() -> Tuple[argparse.ArgumentParser, Dict[str, SupportsCliCommand]]:
     parser = argparse.ArgumentParser(
         description="Creates, adds, inspects and deploys dlt pipelines.",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@@ -142,15 +142,17 @@ def _build_parser() -> argparse.ArgumentParser:
         command = c()
         if command.command in installed_commands.keys():
             continue
-        command_parser = subparsers.add_parser(command.command, help=command.help_string)
+        command_parser = subparsers.add_parser(
+            command.command, help=command.help_string, description=command.help_string
+        )
         command.configure_parser(command_parser)
         installed_commands[command.command] = command
 
-    return parser
+    return parser, installed_commands
 
 
 def main() -> int:
-    parser = _build_parser()
+    parser, installed_commands = _build_parser()
 
     args = parser.parse_args()
 

dlt/cli/plugins.py

@@ -150,40 +150,63 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None:
         )
 
         pipeline_subparsers.add_parser(
-            "info", help="Displays state of the pipeline, use -v or -vv for more info"
+            "info",
+            help="Displays state of the pipeline, use -v or -vv for more info",
+            description="Displays state of the pipeline",
         )
         pipeline_subparsers.add_parser(
             "show",
             help=(
                 "Generates and launches Streamlit app with the loading status and dataset explorer"
             ),
+            description=(
+                "Generates and launches Streamlit app with the loading status and dataset explorer"
+            ),
         )
         pipeline_subparsers.add_parser(
             "failed-jobs",
             help=(
                 "Displays information on all the failed loads in all completed packages, failed"
                 " jobs and associated error messages"
             ),
+            description=(
+                "Displays information on all the failed loads in all completed packages, "
+                "failed jobs and associated error messages"
+            ),
         )
         pipeline_subparsers.add_parser(
             "drop-pending-packages",
             help=(
                 "Deletes all extracted and normalized packages including those that are partially"
                 " loaded."
             ),
+            description=(
+                "Deletes all extracted and normalized packages including those that are partially"
+                " loaded."
+            ),
         )
         pipeline_subparsers.add_parser(
             "sync",
             help=(
                 "Drops the local state of the pipeline and resets all the schemas and restores it"
                 " from destination. The destination state, data and schemas are left intact."
             ),
+            description=(
+                "Drops the local state of the pipeline and resets all the schemas and restores it"
+                " from destination. The destination state, data and schemas are left intact."
+            ),
             parents=[pipe_cmd_sync_parent],
         )
         pipeline_subparsers.add_parser(
-            "trace", help="Displays last run trace, use -v or -vv for more info"
+            "trace",
+            help="Displays last run trace, use -v or -vv for more info",
+            description="Displays last run trace",
+        )
+        pipe_cmd_schema = pipeline_subparsers.add_parser(
+            "schema",
+            help="Displays default schema",
+            description="Displays default schema of pipeline",
         )
-        pipe_cmd_schema = pipeline_subparsers.add_parser("schema", help="Displays default schema")
         pipe_cmd_schema.add_argument(
             "--format",
             choices=["json", "yaml"],
@@ -200,6 +223,7 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None:
         pipe_cmd_drop = pipeline_subparsers.add_parser(
             "drop",
             help="Selectively drop tables and reset state",
+            description="Selectively drop tables and reset state",
             parents=[pipe_cmd_sync_parent],
             epilog=(
                 f"See {DLT_PIPELINE_COMMAND_DOCS_URL}#selectively-drop-tables-and-reset-state for"
@@ -236,7 +260,9 @@ def configure_parser(self, pipe_cmd: argparse.ArgumentParser) -> None:
         )
 
         pipe_cmd_package = pipeline_subparsers.add_parser(
-            "load-package", help="Displays information on load package, use -v or -vv for more info"
+            "load-package",
+            help="Displays information on load package, use -v or -vv for more info",
+            description="Displays information on load package",
         )
         pipe_cmd_package.add_argument(
             "load_id",
@@ -334,8 +360,8 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None:
         # deploy github actions
         deploy_github_cmd = deploy_sub_parsers.add_parser(
             DeploymentMethods.github_actions.value,
-            description="Deploys the pipeline to Github Actions",
             help="Deploys the pipeline to Github Actions",
+            description="Deploys the pipeline to Github Actions",
             parents=[deploy_comm],
         )
         deploy_github_cmd.add_argument(
@@ -364,6 +390,7 @@ def configure_parser(self, parser: argparse.ArgumentParser) -> None:
         deploy_airflow_cmd = deploy_sub_parsers.add_parser(
             DeploymentMethods.airflow_composer.value,
             help="Deploys the pipeline to Airflow",
+            description="Deploys the pipeline to Airflow",
             parents=[deploy_comm],
         )
         deploy_airflow_cmd.add_argument(