Support hidden docs cells and TS validation in notebooks (#6259)

docs/core_docs/.gitignore

@@ -20,6 +20,7 @@ src/supabase.d.ts
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+/scripts/tmp
 
 # ESLint
 .eslintcache
@@ -33,8 +34,6 @@ yarn-error.log*
 
 /.quarto/
 # AUTO_GENERATED_DOCS
-docs/tutorials/test.md
-docs/tutorials/test.mdx
 docs/tutorials/rag.md
 docs/tutorials/rag.mdx
 docs/tutorials/query_analysis.md

docs/core_docs/README.md

@@ -47,3 +47,11 @@ Some common defaults for linting/formatting have been set for you. If you integr
 ```
 $ yarn ci
 ```
+
+### Validating Notebooks
+
+You can validate that notebooks build and compile TypeScript using the following command:
+
+```bash
+$ yarn validate <PATH_TO_FILE>
+```

docs/core_docs/docs/how_to/streaming.ipynb

@@ -70,6 +70,21 @@
     "```"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "// @ls-docs-hide-cell\n",
+    "import { ChatOpenAI } from \"@langchain/openai\";\n",
+    "\n",
+    "const model = new ChatOpenAI({\n",
+    "  model: \"gpt-4o\",\n",
+    "  temperature: 0,\n",
+    "});"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 3,

docs/core_docs/package.json

@@ -28,7 +28,8 @@
     "gen:supabase": "npx supabase gen types typescript --project-id 'xsqpnijvmbodcxyapnyq' --schema public > ./src/supabase.d.ts",
     "broken-links": "node ./scripts/check-broken-links.js",
     "check:broken-links": "yarn quarto && yarn broken-links",
-    "check:broken-links:ci": "yarn quarto:vercel && yarn broken-links"
+    "check:broken-links:ci": "yarn quarto:vercel && yarn broken-links",
+    "validate": "tsx --experimental-wasm-modules -r dotenv/config ./scripts/validate_notebook.ts"
   },
   "dependencies": {
     "@docusaurus/core": "2.4.3",
@@ -70,8 +71,10 @@
     "rimraf": "^5.0.1",
     "supabase": "^1.148.6",
     "swc-loader": "^0.2.3",
+    "tsx": "^3.12.3",
     "typedoc": "^0.24.4",
     "typedoc-plugin-markdown": "next",
+    "typescript": "~5.1.6",
     "yaml-loader": "^0.8.0"
   },
   "packageManager": "yarn@3.4.1",

docs/core_docs/scripts/quarto-build.js

@@ -2,6 +2,8 @@ const fs = require("node:fs");
 const { glob } = require("glob");
 const { execSync } = require("node:child_process");
 
+const IGNORED_CELL_REGEX = /```\w*?\n\/\/ ?@ls-docs-hide-cell\n[\s\S]*?```/g;
+
 async function main() {
   const allIpynb = await glob("./docs/**/*.ipynb");
 
@@ -15,6 +17,15 @@ async function main() {
   gitignore += "# AUTO_GENERATED_DOCS\n";
   gitignore += allRenames.join("\n");
   fs.writeFileSync(pathToRootGitignore, gitignore);
+  for (const renamedFilepath of allRenames) {
+    if (fs.existsSync(renamedFilepath)) {
+      let content = fs.readFileSync(renamedFilepath).toString();
+      if (content.match(IGNORED_CELL_REGEX)) {
+        content = content.replace(IGNORED_CELL_REGEX, "");
+        fs.writeFileSync(renamedFilepath, content);
+      }
+    }
+  }
 
   try {
     /**

docs/core_docs/scripts/validate_notebook.ts

@@ -0,0 +1,85 @@
+import * as fs from "node:fs/promises";
+import * as ts from "typescript";
+import { v4 as uuidv4 } from "uuid";
+
+export async function extract(filepath: string) {
+  const cells = JSON.parse((await fs.readFile(filepath)).toString()).cells;
+  const code = cells
+    .map((cell: Record<string, any>) => {
+      if (cell.cell_type === "code") {
+        return cell.source.join("");
+      }
+      return "";
+    })
+    .join("\n");
+  return code;
+}
+
+let [pathname, ...args] = process.argv.slice(2);
+
+if (!pathname) {
+  throw new Error("No pathname provided.");
+}
+
+const run = async () => {
+  if (pathname.startsWith("docs/core_docs/")) {
+    pathname = "./" + pathname.slice("docs/core_docs/".length);
+  }
+  if (!pathname.endsWith(".ipynb")) {
+    throw new Error("Only .ipynb files are supported.");
+  }
+  const filename = pathname
+    .split("/")
+    [pathname.split("/").length - 1].replace(".ipynb", ".mts");
+  const tempFilepath = `./tmp/${filename}`;
+  try {
+    const typescriptSource = await extract(pathname);
+    try {
+      await fs.access("./tmp", fs.constants.F_OK);
+    } catch (err) {
+      await fs.mkdir("./tmp");
+    }
+    await fs.writeFile(tempFilepath, typescriptSource);
+    const program = ts.createProgram([tempFilepath], {
+      module: ts.ModuleKind.NodeNext,
+      moduleResolution: ts.ModuleResolutionKind.NodeNext,
+      target: ts.ScriptTarget.ES2021,
+      alwaysStrict: true,
+      skipLibCheck: true,
+    });
+    const diagnostics = ts.getPreEmitDiagnostics(program);
+    if (diagnostics.length === 0) {
+      console.log("No type errors found.");
+    } else {
+      diagnostics.forEach((diagnostic) => {
+        if (diagnostic.file) {
+          const { line, character } =
+            diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start!);
+          const message = ts.flattenDiagnosticMessageText(
+            diagnostic.messageText,
+            "\n"
+          );
+          console.log(
+            `${diagnostic.file.fileName} (${line + 1},${
+              character + 1
+            }): ${message}`
+          );
+        } else {
+          console.log(
+            ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
+          );
+        }
+      });
+    }
+  } catch (e) {
+    console.log(e);
+  } finally {
+    try {
+      await fs.rm(tempFilepath);
+    } catch (e) {
+      // Do nothing
+    }
+  }
+};
+
+run();

yarn.lock

@@ -22866,8 +22866,10 @@ __metadata:
     rimraf: ^5.0.1
     supabase: ^1.148.6
     swc-loader: ^0.2.3
+    tsx: ^3.12.3
     typedoc: ^0.24.4
     typedoc-plugin-markdown: next
+    typescript: ~5.1.6
     uuid: ^10.0.0
     webpack: ^5.75.0
     yaml-loader: ^0.8.0