[Segment Cache] Support output: "export" mode

Adds support for output: "export" mode to the Segment Cache
implementation. We output an additional `.txt` data file per segment per
page. When the client issues a per-segment request, it appends the
segment path to the end of the page URL, rather than passing it as a
request header.

The segment file output follows this convention:

```
/a/b/c.html
/a/b/c/__next.a.txt         <- corresponds to segment /a
/a/b/c/__next.a.b.txt       <- corresponds to segment /a/b
/a/b/c/__next.a.b.c.txt     <- corresponds to segment /a/b/c

... and so on
```

This scheme is designed so that the server can implement patterns like
protection rules or rewrites using just the original path. i.e. by
blocking access to `/a/b`, you also block access to all of its
associated segment data.

Technically it's possible for the segment files to clash with a nested
segment config. We add a `__next` prefix to make a clash less likely.
It's unlikely this will ever be an issue in practice but if needed we
could make this prefix configurable at build time.

packages/next/src/client/components/router-reducer/fetch-server-response.ts

@@ -160,6 +160,20 @@ export async function fetchServerResponse(
         : 'low'
       : 'auto'
 
+    if (process.env.NODE_ENV === 'production') {
+      url = new URL(url)
+      if (process.env.__NEXT_CONFIG_OUTPUT === 'export') {
+        // In "output: export" mode, we can't rely on headers to distinguish
+        // between HTML and RSC requests. Instead, we append an extra prefix
+        // to the request.
+        if (url.pathname.endsWith('/')) {
+          url.pathname += 'index.txt'
+        } else {
+          url.pathname += '.txt'
+        }
+      }
+    }
+
     const res = await createFetch(
       url,
       headers,
@@ -255,16 +269,9 @@ export function createFetch(
 ) {
   const fetchUrl = new URL(url)
 
-  if (process.env.NODE_ENV === 'production') {
-    if (process.env.__NEXT_CONFIG_OUTPUT === 'export') {
-      if (fetchUrl.pathname.endsWith('/')) {
-        fetchUrl.pathname += 'index.txt'
-      } else {
-        fetchUrl.pathname += '.txt'
-      }
-    }
-  }
-
+  // TODO: In output: "export" mode, the headers do nothing. Omit them (and the
+  // cache busting search param) from the request so they're
+  // maximally cacheable.
   setCacheBustingSearchParam(fetchUrl, headers)
 
   if (process.env.__NEXT_TEST_MODE && fetchPriority !== null) {

packages/next/src/client/components/segment-cache/cache.ts

@@ -24,7 +24,6 @@ import {
 import {
   createFetch,
   createFromNextReadableStream,
-  urlToUrlWithoutFlightMarker,
   type RequestHeaders,
 } from '../router-reducer/fetch-server-response'
 import {
@@ -42,6 +41,7 @@ import type {
 import { createTupleMap, type TupleMap, type Prefix } from './tuple-map'
 import { createLRU } from './lru'
 import {
+  convertSegmentPathToStaticExportFilename,
   encodeChildSegmentKey,
   encodeSegment,
   ROOT_SEGMENT_KEY,
@@ -52,6 +52,7 @@ import type {
 } from '../../../server/app-render/types'
 import { normalizeFlightData } from '../../flight-data-helpers'
 import { STATIC_STALETIME_MS } from '../router-reducer/prefetch-cache-utils'
+import { RSC_SEGMENTS_DIR_SUFFIX } from '../../../lib/constants'
 
 // A note on async/await when working in the prefetch cache:
 //
@@ -205,6 +206,10 @@ export type NonEmptySegmentCacheEntry = Exclude<
   EmptySegmentCacheEntry
 >
 
+const isOutputExportMode =
+  process.env.NODE_ENV === 'production' &&
+  process.env.__NEXT_CONFIG_OUTPUT === 'export'
+
 // Route cache entries vary on multiple keys: the href and the Next-Url. Each of
 // these parts needs to be included in the internal cache key. Rather than
 // concatenate the keys into a single key, we use a multi-level map, where the
@@ -825,7 +830,9 @@ export async function fetchRouteOnCacheMiss(
     // This is a bit convoluted but it's taken from router-reducer and
     // fetch-server-response
     const canonicalUrl = response.redirected
-      ? createHrefFromUrl(urlToUrlWithoutFlightMarker(response.url))
+      ? createHrefFromUrl(
+          new URL(removeSegmentPathFromURLInOutputExportMode(response.url))
+        )
       : href
 
     // Check whether the response varies based on the Next-Url header.
@@ -839,9 +846,13 @@ export async function fetchRouteOnCacheMiss(
     // This checks whether the response was served from the per-segment cache,
     // rather than the old prefetching flow. If it fails, it implies that PPR
     // is disabled on this route.
-    // TODO: Add support for non-PPR routes.
     const routeIsPPREnabled =
-      response.headers.get(NEXT_DID_POSTPONE_HEADER) === '2'
+      response.headers.get(NEXT_DID_POSTPONE_HEADER) === '2' ||
+      // In output: "export" mode, we can't rely on response headers. But if we
+      // receive a well-formed response, we can assume it's a static response,
+      // because all data is static in this mode.
+      isOutputExportMode
+
     if (routeIsPPREnabled) {
       const prefetchStream = createPrefetchResponseStream(
         response.body,
@@ -973,7 +984,11 @@ export async function fetchSegmentOnCacheMiss(
       // is disabled on this route. Theoretically this should never happen
       // because we only issue requests for segments once we've verified that
       // the route supports PPR.
-      response.headers.get(NEXT_DID_POSTPONE_HEADER) !== '2' ||
+      (response.headers.get(NEXT_DID_POSTPONE_HEADER) !== '2' &&
+        // In output: "export" mode, we can't rely on response headers. But if
+        // we receive a well-formed response, we can assume it's a static
+        // response, because all data is static in this mode.
+        !isOutputExportMode) ||
       !response.body
     ) {
       // Server responded with an error, or with a miss. We should still cache
@@ -1334,21 +1349,38 @@ async function fetchSegmentPrefetchResponse(
   if (nextUrl !== null) {
     headers[NEXT_URL] = nextUrl
   }
-  return fetchPrefetchResponse(href, headers)
+  return fetchPrefetchResponse(
+    isOutputExportMode
+      ? addSegmentPathToUrlInOutputExportMode(href, segmentPath)
+      : href,
+    headers
+  )
 }
 
 async function fetchPrefetchResponse(
-  href: NormalizedHref,
+  href: string,
   headers: RequestHeaders
 ): Promise<Response | null> {
   const fetchPriority = 'low'
   const response = await createFetch(new URL(href), headers, fetchPriority)
-  const contentType = response.headers.get('content-type')
-  const isFlightResponse =
-    contentType && contentType.startsWith(RSC_CONTENT_TYPE_HEADER)
-  if (!response.ok || !isFlightResponse) {
+  if (!response.ok) {
     return null
   }
+
+  // Check the content type
+  if (isOutputExportMode) {
+    // In output: "export" mode, we relaxed about the content type, since it's
+    // not Next.js that's serving the response. If the status is OK, assume the
+    // response is valid. If it's not a valid response, the Flight client won't
+    // be able to decode it, and we'll treat it as a miss.
+  } else {
+    const contentType = response.headers.get('content-type')
+    const isFlightResponse =
+      contentType && contentType.startsWith(RSC_CONTENT_TYPE_HEADER)
+    if (!isFlightResponse) {
+      return null
+    }
+  }
   return response
 }
 
@@ -1399,6 +1431,43 @@ function createPrefetchResponseStream(
   })
 }
 
+function addSegmentPathToUrlInOutputExportMode(
+  url: string,
+  segmentPath: string
+) {
+  if (isOutputExportMode) {
+    // In output: "export" mode, we cannot use a header to encode the segment
+    // path. Instead, we append it to the end of the pathname.
+    const staticUrl = new URL(url)
+    const routeDir = staticUrl.pathname.endsWith('/')
+      ? staticUrl.pathname.substring(0, -1)
+      : staticUrl.pathname
+    const staticExportFilename =
+      convertSegmentPathToStaticExportFilename(segmentPath)
+    staticUrl.pathname = `${routeDir}/${staticExportFilename}`
+    return staticUrl.href
+  }
+  return url
+}
+
+function removeSegmentPathFromURLInOutputExportMode(redirectUrl: string) {
+  if (isOutputExportMode) {
+    // This is the inverse of addSegmentPathToUrlInOutputExportMode. Used when
+    // a prefetch request is redirected so we can obtain the canonical URL.
+    // TODO: How do redirects work in output: "export" mode? Do we support
+    // arbitrary redirects, like we do in the normal mode? Investigate.
+    // Remove everything that follows the final occurrence
+    // of RSC_SEGMENTS_DIR_SUFFIX
+    const staticUrl = new URL(redirectUrl)
+    const suffixIndex = staticUrl.pathname.lastIndexOf(RSC_SEGMENTS_DIR_SUFFIX)
+    if (suffixIndex !== -1) {
+      staticUrl.pathname = staticUrl.pathname.slice(0, suffixIndex)
+      return staticUrl.href
+    }
+  }
+  return redirectUrl
+}
+
 function createPromiseWithResolvers<T>(): PromiseWithResolvers<T> {
   // Shim of Stage 4 Promise.withResolvers proposal
   let resolve: (value: T | PromiseLike<T>) => void

packages/next/src/export/index.ts

@@ -12,11 +12,16 @@ import { existsSync, promises as fs } from 'fs'
 
 import '../server/require-hook'
 
-import { dirname, join, resolve, sep } from 'path'
+import { dirname, join, resolve, sep, relative } from 'path'
 import { formatAmpMessages } from '../build/output/index'
 import type { AmpPageStatus } from '../build/output/index'
 import * as Log from '../build/output/log'
-import { RSC_SUFFIX, SSG_FALLBACK_EXPORT_ERROR } from '../lib/constants'
+import {
+  RSC_SEGMENT_SUFFIX,
+  RSC_SEGMENTS_DIR_SUFFIX,
+  RSC_SUFFIX,
+  SSG_FALLBACK_EXPORT_ERROR,
+} from '../lib/constants'
 import { recursiveCopy } from '../lib/recursive-copy'
 import {
   BUILD_ID_FILE,
@@ -55,6 +60,7 @@ import { TurborepoAccessTraceResult } from '../build/turborepo-access-trace'
 import { createProgress } from '../build/progress'
 import type { DeepReadonly } from '../shared/lib/deep-readonly'
 import { isInterceptionRouteRewrite } from '../lib/generate-interception-routes-rewrites'
+import { convertSegmentPathToStaticExportFilename } from '../server/app-render/segment-value-encoding'
 
 export class ExportError extends Error {
   code = 'NEXT_EXPORT_ERROR'
@@ -739,6 +745,27 @@ async function exportAppImpl(
           await fs.mkdir(dirname(ampHtmlDest), { recursive: true })
           await fs.copyFile(`${orig}.amp.html`, ampHtmlDest)
         }
+
+        const segmentsDir = `${orig}${RSC_SEGMENTS_DIR_SUFFIX}`
+        if (isAppPath && existsSync(segmentsDir)) {
+          // Output a data file for each of this page's segments
+          const segmentsDirDest = join(outDir, route)
+          const segmentPaths = await collectSegmentPaths(segmentsDir)
+          await Promise.all(
+            segmentPaths.map(async (segmentFileSrc) => {
+              const segmentPath =
+                '/' + segmentFileSrc.slice(0, -RSC_SEGMENT_SUFFIX.length)
+              const segmentFilename =
+                convertSegmentPathToStaticExportFilename(segmentPath)
+              const segmentFileDest = join(segmentsDirDest, segmentFilename)
+              await fs.mkdir(dirname(segmentFileDest), { recursive: true })
+              await fs.copyFile(
+                join(segmentsDir, segmentFileSrc),
+                segmentFileDest
+              )
+            })
+          )
+        }
       })
     )
   }
@@ -780,6 +807,40 @@ async function exportAppImpl(
   return collector
 }
 
+async function collectSegmentPaths(segmentsDirectory: string) {
+  const results: Array<string> = []
+  await collectSegmentPathsImpl(segmentsDirectory, segmentsDirectory, results)
+  return results
+}
+
+async function collectSegmentPathsImpl(
+  segmentsDirectory: string,
+  directory: string,
+  results: Array<string>
+) {
+  const segmentFiles = await fs.readdir(directory, {
+    withFileTypes: true,
+  })
+  await Promise.all(
+    segmentFiles.map(async (segmentFile) => {
+      if (segmentFile.isDirectory()) {
+        await collectSegmentPathsImpl(
+          segmentsDirectory,
+          join(directory, segmentFile.name),
+          results
+        )
+        return
+      }
+      if (!segmentFile.name.endsWith(RSC_SEGMENT_SUFFIX)) {
+        return
+      }
+      results.push(
+        relative(segmentsDirectory, join(directory, segmentFile.name))
+      )
+    })
+  )
+}
+
 export default async function exportApp(
   dir: string,
   options: ExportAppOptions,

packages/next/src/server/app-render/segment-value-encoding.ts

@@ -73,3 +73,9 @@ function encodeToFilesystemAndURLSafeString(value: string) {
     .replace(/=+$/, '') // Remove trailing '='
   return '!' + base64url
 }
+
+export function convertSegmentPathToStaticExportFilename(
+  segmentPath: string
+): string {
+  return `__next${segmentPath.replace(/\//g, '.')}.txt`
+}

test/e2e/app-dir/segment-cache/export/app/layout.tsx

@@ -0,0 +1,11 @@
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode
+}) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

test/e2e/app-dir/segment-cache/export/app/page.tsx

@@ -0,0 +1,28 @@
+import { LinkAccordion } from '../components/link-accordion'
+
+export default function OutputExport() {
+  return (
+    <>
+      <p>
+        Demonstrates that per-segment prefetching works in{' '}
+        <code>output: export</code> mode.
+      </p>
+      <ul>
+        <li>
+          <LinkAccordion href="/target-page">Target</LinkAccordion>
+        </li>
+      </ul>
+      <p>
+        The following link is rewritten on the server to the same page as the
+        link above:
+      </p>
+      <ul>
+        <li>
+          <LinkAccordion href="/rewrite-to-target-page">
+            Rewrite to target page
+          </LinkAccordion>
+        </li>
+      </ul>
+    </>
+  )
+}

test/e2e/app-dir/segment-cache/export/app/target-page/page.tsx

@@ -0,0 +1,3 @@
+export default function Target() {
+  return <div id="target-page">Target page</div>
+}

test/e2e/app-dir/segment-cache/export/components/link-accordion.tsx

@@ -0,0 +1,23 @@
+'use client'
+
+import Link from 'next/link'
+import { useState } from 'react'
+
+export function LinkAccordion({ href, children }) {
+  const [isVisible, setIsVisible] = useState(false)
+  return (
+    <>
+      <input
+        type="checkbox"
+        checked={isVisible}
+        onChange={() => setIsVisible(!isVisible)}
+        data-link-accordion={href}
+      />
+      {isVisible ? (
+        <Link href={href}>{children}</Link>
+      ) : (
+        `${children} (link is hidden)`
+      )}
+    </>
+  )
+}

test/e2e/app-dir/segment-cache/export/next.config.js

@@ -0,0 +1,13 @@
+/**
+ * @type {import('next').NextConfig}
+ */
+const nextConfig = {
+  output: 'export',
+  experimental: {
+    ppr: false,
+    dynamicIO: true,
+    clientSegmentCache: true,
+  },
+}
+
+module.exports = nextConfig