Inline all methods for Qiskit API (#552)

The generation script was not inlining all the methods in the class
pages because we weren't assigning the correct value to the
`python_api_name` metadata. With this PR, we change the way we search
for the id we set as the name in the `sphinxHtmlToMarkdown.ts`

The problem comes from we are missing the class `sig-object` in the `dt`
tags we used to extract the `python_api_name` metadata in the HTML used
to generate the markdown API files. We actually don't need to specify
the class because the `dt` tag we are looking for is unique, and this PR
fixes that.

Moreover, the changes introduced in #543 are reverted to recover the
mergeClassMembers.ts script that we need to inline all the methods in
historical versions.

Tested the generation of Qiskit 0.33, where we will inline the methods
in a follow-up, and also the generation of Qiskit 0.45 to validate we
are not modifying any version that was working previously.

Commands used (Notice we need to have the API docs downloaded in .out):

> npm run gen-api -- -p qiskit -v 0.33.0 -a "" --historical
> npm run gen-api -- -p qiskit -v 0.45.0 -a ""

Part of #542

scripts/commands/updateApiDocs.ts

@@ -23,6 +23,7 @@ import { WebCrawler } from "../lib/WebCrawler";
 import { downloadImages } from "../lib/downloadImages";
 import { generateToc } from "../lib/sphinx/generateToc";
 import { SphinxToMdResult } from "../lib/sphinx/SphinxToMdResult";
+import { mergeClassMembers } from "../lib/sphinx/mergeClassMembers";
 import { flatFolders } from "../lib/sphinx/flatFolders";
 import { updateLinks } from "../lib/sphinx/updateLinks";
 import { renameUrls } from "../lib/sphinx/renameUrls";
@@ -305,6 +306,7 @@ async function convertHtmlToMarkdown(
     await mkdirp(dir);
   }
 
+  results = await mergeClassMembers(results);
   flatFolders(results);
   renameUrls(results);
   await updateLinks(results, pkg.transformLink);

scripts/lib/sphinx/mergeClassMembers.test.ts

@@ -0,0 +1,131 @@
+// This code is a Qiskit project.
+//
+// (C) Copyright IBM 2023.
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE file in the root directory
+// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+import { describe, expect, test } from "@jest/globals";
+import { mergeClassMembers } from "./mergeClassMembers";
+
+describe("mergeClassMembers", () => {
+  test("merge class members", async () => {
+    const results: Parameters<typeof mergeClassMembers>[0] = [
+      {
+        markdown: `## Attributes
+
+|                                                                                                                                                                                                         |                 |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- |
+| [\`RuntimeOptions.backend\`](qiskit_ibm_runtime.RuntimeOptions.backend#qiskit_ibm_runtime.RuntimeOptions.backend "qiskit_ibm_runtime.RuntimeOptions.backend")                                             |                 |
+
+## Methods
+
+|                                                                                                                                                                          |                   |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------- |
+| [\`RuntimeOptions.validate\`](qiskit_ibm_runtime.RuntimeOptions.validate#qiskit_ibm_runtime.RuntimeOptions.validate "qiskit_ibm_runtime.RuntimeOptions.validate")(channel) | Validate options. |`,
+        meta: {
+          python_api_type: "class",
+          python_api_name: "RuntimeOptions",
+        },
+        url: "/docs/api/qiskit-ibm-runtime/stubs/qiskit_ibm_runtime.RuntimeOptions",
+        images: [],
+        isReleaseNotes: false,
+      },
+      {
+        markdown: `# RuntimeOptions.backend
+
+\`Optional[str] = None\`
+`,
+        meta: {
+          python_api_type: "attribute",
+          python_api_name: "RuntimeOptions.backend",
+        },
+        url: "/docs/api/qiskit-ibm-runtime/stubs/qiskit_ibm_runtime.RuntimeOptions.backend",
+        images: [],
+        isReleaseNotes: false,
+      },
+      {
+        markdown: `# RuntimeOptions.circuits
+
+\`Optional[str] = None\`
+`,
+        meta: {
+          python_api_type: "property",
+          python_api_name: "RuntimeOptions.circuits",
+        },
+        url: "/docs/api/qiskit-ibm-runtime/stubs/qiskit_ibm_runtime.RuntimeOptions.circuits",
+        images: [],
+        isReleaseNotes: false,
+      },
+      {
+        markdown: `
+# RuntimeOptions.validate
+
+\`RuntimeOptions.validate(channel)\`
+
+Validate options.
+
+*   Parameters:
+
+    **channel** (\`str\`) – channel type.
+
+*   Raises:
+
+    **IBMInputValueError** – If one or more option is invalid.
+
+*   Return type:
+
+    \`None\`
+          `,
+        meta: {
+          python_api_type: "method",
+          python_api_name: "RuntimeOptions.backend",
+        },
+        url: "/docs/api/qiskit-ibm-runtime/stubs/qiskit_ibm_runtime.RuntimeOptions.validate",
+        images: [],
+        isReleaseNotes: false,
+      },
+    ];
+    const merged = await mergeClassMembers(results);
+    expect(
+      merged.find((item) => item.meta.python_api_type === "class")?.markdown,
+    ).toMatchInlineSnapshot(`
+        "## Attributes
+
+        ### RuntimeOptions.backend
+
+        \`Optional[str] = None\`
+
+        ### RuntimeOptions.circuits
+
+        \`Optional[str] = None\`
+
+        ## Methods
+
+        ### RuntimeOptions.validate
+
+        \`RuntimeOptions.validate(channel)\`
+
+        Validate options.
+
+        *   Parameters:
+
+            **channel** (\`str\`) – channel type.
+
+        *   Raises:
+
+            **IBMInputValueError** – If one or more option is invalid.
+
+        *   Return type:
+
+            \`None\`
+        "
+      `);
+    expect(merged.length).toEqual(1);
+  });
+});

scripts/lib/sphinx/mergeClassMembers.ts

@@ -0,0 +1,154 @@
+// This code is a Qiskit project.
+//
+// (C) Copyright IBM 2023.
+//
+// This code is licensed under the Apache License, Version 2.0. You may
+// obtain a copy of this license in the LICENSE file in the root directory
+// of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+//
+// Any modifications or derivative works of this code must retain this
+// copyright notice, and modified files need to carry a notice indicating
+// that they have been altered from the originals.
+
+import { includes, isEmpty, orderBy, reject } from "lodash";
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+import remarkMdx from "remark-mdx";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import remarkStringify from "remark-stringify";
+import { Content, Root } from "mdast";
+import { visit } from "unist-util-visit";
+import { SphinxToMdResultWithUrl } from "./SphinxToMdResult";
+import { remarkStringifyOptions } from "./unifiedParser";
+
+export async function mergeClassMembers<T extends SphinxToMdResultWithUrl>(
+  results: T[],
+): Promise<T[]> {
+  const resultsWithName = results.filter(
+    (result) => !isEmpty(result.meta.python_api_name),
+  );
+  const classes = resultsWithName.filter(
+    (result) => result.meta.python_api_type === "class",
+  );
+
+  for (const clazz of classes) {
+    const members = orderBy(
+      resultsWithName.filter((result) => {
+        if (
+          !includes(
+            ["method", "property", "attribute", "function"],
+            result.meta.python_api_type,
+          )
+        )
+          return false;
+        return result.meta.python_api_name?.startsWith(
+          `${clazz.meta.python_api_name}.`,
+        );
+      }),
+      (result) => result.meta.python_api_name,
+    );
+
+    const attributesAndProps = members.filter(
+      (member) =>
+        member.meta.python_api_type === "attribute" ||
+        member.meta.python_api_type === "property",
+    );
+    const methods = members.filter(
+      (member) => member.meta.python_api_type === "method",
+    );
+
+    try {
+      // inject members markdown
+      clazz.markdown = (
+        await unified()
+          .use(remarkParse)
+          .use(remarkMdx)
+          .use(remarkGfm)
+          .use(remarkMath)
+          .use(() => {
+            return async (tree) => {
+              for (const node of tree.children) {
+                await replaceMembersAfterTitle(
+                  tree,
+                  node,
+                  "Attributes",
+                  attributesAndProps,
+                );
+                await replaceMembersAfterTitle(tree, node, "Methods", methods);
+                await replaceMembersAfterTitle(
+                  tree,
+                  node,
+                  "Methods Defined Here",
+                  methods,
+                );
+              }
+            };
+          })
+          .use(remarkStringify, remarkStringifyOptions)
+          .process(clazz.markdown)
+      ).toString();
+    } catch (e) {
+      console.log("Error found in", clazz.meta.python_api_name);
+      console.log(clazz.markdown);
+      throw e;
+    }
+  }
+
+  // remove merged results
+  const finalResults = reject(results, (result) =>
+    includes(["method", "attribute", "property"], result.meta.python_api_type),
+  );
+
+  return finalResults;
+}
+
+async function replaceMembersAfterTitle(
+  tree: Root,
+  node: Content,
+  title: string,
+  members: SphinxToMdResultWithUrl[],
+) {
+  if (node.type !== "heading") return;
+  const nodeIndex = tree.children.indexOf(node);
+  if (nodeIndex === -1) return;
+
+  const nextNode = tree.children[nodeIndex + 1];
+  const firstChild = node.children[0];
+
+  if (
+    firstChild?.type === "text" &&
+    firstChild?.value === title &&
+    nextNode?.type === "table"
+  ) {
+    const children: any[] = [];
+    for (const member of members) {
+      const updated = await parseMarkdownIncreasingHeading(member.markdown, 2);
+      children.push(...updated.children);
+    }
+    tree.children.splice(nodeIndex + 1, 1, ...children);
+  }
+}
+
+async function parseMarkdownIncreasingHeading(
+  md: string,
+  depthIncrease: number,
+): Promise<Root> {
+  const root = await unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkMath)
+    .use(remarkMdx)
+    .parse(md);
+  const changedTree = await unified()
+    .use(remarkMath)
+    .use(remarkGfm)
+    .use(remarkMdx)
+    .use(() => (root) => {
+      visit(root, "heading", (node: any) => {
+        node.depth = node.depth + depthIncrease;
+      });
+    })
+    .run(root);
+  return changedTree as Root;
+}

scripts/lib/sphinx/sphinxHtmlToMarkdown.ts

@@ -199,7 +199,7 @@ export async function sphinxHtmlToMarkdown(options: {
       .map((child) => {
         const $child = $page(child);
         $child.find(".viewcode-link").closest("a").remove();
-        const id = $dl.find("dt.sig-object").attr("id") || "";
+        const id = $dl.find("dt").attr("id") || "";
 
         if (child.name === "dt" && $dl.hasClass("class")) {
           if (!meta.python_api_type) {