feat(rari): support rari as beta feature and new popularities (#12051)

* feat(rari): support rari as beta feature
* feat(rari): add rari server to bin
* update popularities format in yari to match rari implementation

---------

Co-authored-by: Andi Pieper <apieper@mozilla.com>

.github/workflows/stage-build.yml

@@ -151,6 +151,13 @@ jobs:
           mv mdn/translated-content-de/files/de mdn/translated-content/files/
           rm -rf mdn/translated-content-de
 
+      - name: Clean and commit de
+        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
+        working-directory: mdn/translated-content
+        run: |
+          git add files/de
+          git -c user.name='MDN' -c user.email='mdn-dev@mozilla.com' commit -m 'de'
+
       - uses: actions/checkout@v4
         if: ${{ ! vars.SKIP_BUILD }}
         with:
@@ -168,7 +175,8 @@ jobs:
         if: ${{ ! vars.SKIP_BUILD }}
         run: yarn --frozen-lockfile
         env:
-          # https://github.com/microsoft/vscode-ripgrep#github-api-limit-note
+          # Use a GITHUB_TOKEN to bypass rate limiting for ripgrep and rari.
+          # See https://github.com/microsoft/vscode-ripgrep#github-api-limit-note
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install Python
@@ -214,6 +222,13 @@ jobs:
           CURRICULUM_ROOT: ${{ github.workspace }}/mdn/curriculum
           BASE_URL: "https://developer.allizom.org"
 
+          # rari
+          BUILD_OUT_ROOT: "client/build"
+          GENERIC_CONTENT_ROOT: "copy"
+          LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev
+          INTERACTIVE_EXAMPLES_BASE_URL: https://interactive-examples.mdn.allizom.net
+          ADDITIONAL_LOCALES_FOR_GENERICS_AND_SPAS: de
+
           # The default for this environment variable is geared for writers
           # (aka. local development). Usually defaults are supposed to be for
           # secure production but this is an exception and default
@@ -290,31 +305,14 @@ jobs:
           echo "BLOG_ROOT=$BLOG_ROOT"
           # Build the ServiceWorker first
           yarn build:sw
-          yarn build:prepare
-
-          yarn tool sync-translated-content es fr ja ko pt-br ru zh-cn zh-tw
-
-          # Build using one process per locale.
-          # Note: We have 4 cores, but 9 processes is a reasonable number.
-          for locale in en-us de es fr ja ko pt-br ru zh-cn zh-tw; do
-            yarn build:docs --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
-            pids+=($!)
-          done
-
-          for pid in "${pids[@]}"; do
-            wait $pid
-          done
-
-          du -sh client/build
-
-          # Build the blog
-          yarn build:blog
+          yarn build:client
+          yarn build:ssr
+          yarn tool build-robots-txt
 
-          # Build the curriculum
-          yarn build:curriculum
+          yarn rari content sync-translated-content
+          yarn rari git-history
 
-          # Generate sitemap index file
-          yarn build --sitemap-index
+          yarn rari build --issues client/build/issues.json --templ-stats
 
           # SSR all pages
           yarn render:html

.github/workflows/test-build.yml

@@ -72,6 +72,26 @@ jobs:
           # See matching warning for mdn/content checkout step
           fetch-depth: 0
 
+      - name: Checkout (translated-content-de)
+        uses: actions/checkout@v4
+        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
+        with:
+          repository: mdn/translated-content-de
+          path: mdn/translated-content-de
+
+      - name: Move de into translated-content
+        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
+        run: |
+          mv mdn/translated-content-de/files/de mdn/translated-content/files/
+          rm -rf mdn/translated-content-de
+
+      - name: Clean and commit de
+        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
+        working-directory: mdn/translated-content
+        run: |
+          git add files/de
+          git -c user.name='MDN' -c user.email='mdn-dev@mozilla.com' commit -m 'de'
+
       - uses: actions/checkout@v4
         if: ${{ ! vars.SKIP_BUILD }}
         with:
@@ -89,7 +109,8 @@ jobs:
         if: ${{ ! vars.SKIP_BUILD }}
         run: yarn --frozen-lockfile
         env:
-          # https://github.com/microsoft/vscode-ripgrep#github-api-limit-note
+          # Use a GITHUB_TOKEN to bypass rate limiting for ripgrep and rari.
+          # See https://github.com/microsoft/vscode-ripgrep#github-api-limit-note
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Print information about build
@@ -111,6 +132,13 @@ jobs:
           CURRICULUM_ROOT: ${{ github.workspace }}/mdn/curriculum
           BASE_URL: "https://test.developer.allizom.org"
 
+          # rari
+          BUILD_OUT_ROOT: "client/build"
+          GENERIC_CONTENT_ROOT: "copy"
+          LIVE_SAMPLES_BASE_URL: https://live.test.mdnyalp.dev
+          INTERACTIVE_EXAMPLES_BASE_URL: https://interactive-examples.mdn.allizom.net
+          ADDITIONAL_LOCALES_FOR_GENERICS_AND_SPAS: de
+
           # The default for this environment variable is geared for writers
           # (aka. local development). Usually defaults are supposed to be for
           # secure production but this is an exception and default
@@ -155,7 +183,7 @@ jobs:
           REACT_APP_PLACEMENT_ENABLED: true
 
           # Playground
-          REACT_APP_PLAYGROUND_BASE_HOST: play.test.mdn.allizom.net
+          REACT_APP_PLAYGROUND_BASE_HOST: test.mdnyalp.dev
 
           # Observatory
           REACT_APP_OBSERVATORY_API_URL: https://observatory-api.mdn.allizom.net
@@ -168,31 +196,14 @@ jobs:
           echo "BLOG_ROOT=$BLOG_ROOT"
           # Build the ServiceWorker first
           yarn build:sw
-          yarn build:prepare
-
-          #yarn tool sync-translated-content
-
-          # Build using one process per locale.
-          # Note: We have 4 cores, but 9 processes is a reasonable number.
-          for locale in en-us fr; do
-            yarn build:docs --locale $locale 2>&1 | sed "s/^/[$locale] /"   &
-            pids+=($!)
-          done
-
-          for pid in "${pids[@]}"; do
-            wait $pid
-          done
-
-          du -sh client/build
-
-          # Build the blog
-          yarn build:blog
+          yarn build:client
+          yarn build:ssr
+          yarn tool build-robots-txt
 
-          # Build the curriculum
-          yarn build:curriculum
+          yarn rari content sync-translated-content
+          yarn rari git-history
 
-          # Generate sitemap index file
-          yarn build --sitemap-index
+          yarn rari build --issues client/build/issues.json --templ-stats
 
           # SSR all pages
           yarn render:html
@@ -241,6 +252,7 @@ jobs:
         run: |
           npm ci
           npm run build-redirects
+          npm run build-canonicals
 
       - name: Deploy Function
         if: ${{ ! vars.SKIP_FUNCTION }}

Procfile.rari

@@ -0,0 +1,2 @@
+server: yarn start:rari-server
+web: yarn start:client

README.md

@@ -269,6 +269,14 @@ this, you can pick any unused port (e.g., 6000) and run the following:
 
     echo SERVER_PORT=6000 >> .env
 
+### Problems running with rari on Windows
+
+Download and install:
+
+Microsoft Visual C++ Redistributable for Visual Studio 2019 (
+[x86](https://aka.ms/vs/16/release/VC_redist.x86.exe),
+[ARM64](https://aka.ms/vs/16/release/VC_redist.arm64.exe) )
+
 ### Yarn install errors
 
 If you get errors while installing dependencies via yarn on a Mac, you may need

client/src/document/toolbar/flaws.tsx

@@ -331,7 +331,7 @@ function Flaws({
               />
             );
           default:
-            throw new Error(`Unknown flaw check '${flaw.name}'`);
+            return <Unknown key="unknown" flaws={doc.flaws.unknown || []} />;
         }
       })}
     </div>
@@ -531,6 +531,20 @@ function BrokenLinks({
   );
 }
 
+function Unknown({ flaws }: { flaws: GenericFlaw[] }) {
+  return (
+    <div className="flaw flaw__unkown">
+      <h3>{humanizeFlawName("unknown")}</h3>
+      <ul>
+        {flaws.map((flaw) => (
+          <li key={flaw.id}>
+            <code>{flaw.explanation}</code>
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+}
 function BadBCDQueries({ flaws }: { flaws: BadBCDQueryFlaw[] }) {
   return (
     <div className="flaw flaw__bad_bcd_queries">
@@ -692,7 +706,9 @@ function Macros({
     <div className="flaw flaw__macros">
       <h3>{humanizeFlawName("macros")}</h3>
       {flaws.map((flaw) => {
-        const inPrerequisiteMacro = !flaw.filepath.includes(sourceFilePath);
+        const inPrerequisiteMacro = flaw.filepath
+          ? !flaw.filepath.includes(sourceFilePath)
+          : false;
         return (
           <details
             key={flaw.id}

content/popularities.ts

@@ -7,7 +7,11 @@ export function getPopularities() {
   if (!popularities.size) {
     // This is the file that's *not* checked into git.
     const filePath = new URL("../popularities.json", import.meta.url);
-    Object.entries(JSON.parse(fs.readFileSync(filePath, "utf-8"))).forEach(
+    const json = JSON.parse(fs.readFileSync(filePath, "utf-8"));
+    if (!("popularities" in json)) {
+      throw Error("run: yarn tool popularities --refresh");
+    }
+    Object.entries(json.popularities).forEach(
       ([url, value]: [string, unknown]) => {
         popularities.set(url, value as number);
       }

copy/community/community.md

@@ -1,3 +1,7 @@
+---
+title: Contribute to MDN
+---
+
 # MDN Community
 
 Where web enthusiasts learn, collaborate, and create

copy/config.json

@@ -0,0 +1,29 @@
+{
+  "pages": {
+    "community": {
+      "titleSuffix": "Contribute to MDN"
+    },
+    "plus": {
+      "slugPrefix": "plus/docs",
+      "titleSuffix": "MDN Plus"
+    },
+    "observatory": {
+      "slugPrefix": "observatory/docs",
+      "titleSuffix": "HTTP Observatory"
+    }
+  },
+  "spas": {
+    "about": {
+      "slug": "about",
+      "pageTitle": "About MDN"
+    },
+    "advertising": {
+      "slug": "advertising",
+      "pageTitle": "Advertise with us"
+    },
+    "plus": {
+      "slug": "plus",
+      "pageTitle": "MDN Plus"
+    }
+  }
+}

deployer/src/deployer/search/__init__.py

@@ -248,7 +248,7 @@ def to_search(file, _index=None):
             )
         ),
         popularity=doc["popularity"],
-        summary=doc["summary"],
+        summary=doc.get("summary", ""),
         # Note! We're always lowercasing the 'slug'. This way we can search on it,
         # still as a `keyword` index, but filtering by prefix.
         # E.g. in kuma; ?slug_prefix=weB/Css

libs/env/index.d.ts

@@ -1,5 +1,6 @@
 export const BUILD_OUT_ROOT: string;
 export const DEFAULT_FLAW_LEVELS: string;
+export const RARI: boolean;
 export const BASE_URL: string;
 export const FILES: string;
 export const FOLDERSEARCH: string;
@@ -27,6 +28,7 @@ export const STATIC_ROOT: string;
 export const PROXY_HOSTNAME: string;
 export const CONTENT_HOSTNAME: string;
 export const FAKE_V1_API: boolean;
+export const EXTERNAL_DEV_SERVER: sting;
 export const SENTRY_DSN_BUILD: string;
 export const OPENAI_KEY: string;
 export const PG_URI: string;

libs/env/index.js

@@ -28,6 +28,7 @@ dotenv.config({
 // build
 // -----
 
+export const RARI = Boolean(parse(process.env.RARI || "false"));
 export const BASE_URL = process.env.BASE_URL || "https://developer.mozilla.org";
 
 export const BUILD_OUT_ROOT =
@@ -178,6 +179,8 @@ export const PROXY_HOSTNAME =
 export const CONTENT_HOSTNAME = process.env.SERVER_CONTENT_HOST;
 
 export const FAKE_V1_API = parse(process.env.SERVER_FAKE_V1_API || false);
+export const EXTERNAL_DEV_SERVER =
+  process.env.EXTERNAL_DEV_SERVER || "http://localhost:8083";
 
 // ----
 // tool

libs/types/document.ts

@@ -110,6 +110,7 @@ export type Flaws = Partial<{
   heading_links: HeadingLinksFlaw[];
   translation_differences: TranslationDifferenceFlaw[];
   unsafe_html: UnsafeHTMLFlaw[];
+  unknown: GenericFlaw[];
 }>;
 
 export type Translation = {