[Segment Cache] Support output: "export" mode

Adds support for output: "export" mode to the Segment Cache
implementation. The main change is that we now output an additional
`.txt` data per segment per page. When the client issues a per-segment
request, it appends the segment path to the end of the page URL, rather
than passing it as a request header.

As a follow up, I need to investigate how redirects and rewrites should
work in this mode.

packages/next/src/client/components/router-reducer/fetch-server-response.ts

@@ -160,6 +160,20 @@ export async function fetchServerResponse(
         : 'low'
       : 'auto'
 
+    if (process.env.NODE_ENV === 'production') {
+      url = new URL(url)
+      if (process.env.__NEXT_CONFIG_OUTPUT === 'export') {
+        // In "output: export" mode, we can't rely on headers to distinguish
+        // between HTML and RSC requests. Instead, we append an extra prefix
+        // to the request.
+        if (url.pathname.endsWith('/')) {
+          url.pathname += 'index.txt'
+        } else {
+          url.pathname += '.txt'
+        }
+      }
+    }
+
     const res = await createFetch(
       url,
       headers,
@@ -254,17 +268,6 @@ export function createFetch(
   signal?: AbortSignal
 ) {
   const fetchUrl = new URL(url)
-
-  if (process.env.NODE_ENV === 'production') {
-    if (process.env.__NEXT_CONFIG_OUTPUT === 'export') {
-      if (fetchUrl.pathname.endsWith('/')) {
-        fetchUrl.pathname += 'index.txt'
-      } else {
-        fetchUrl.pathname += '.txt'
-      }
-    }
-  }
-
   setCacheBustingSearchParam(fetchUrl, headers)
 
   if (process.env.__NEXT_TEST_MODE && fetchPriority !== null) {

packages/next/src/client/components/segment-cache/cache.ts

@@ -24,7 +24,6 @@ import {
 import {
   createFetch,
   createFromNextReadableStream,
-  urlToUrlWithoutFlightMarker,
   type RequestHeaders,
 } from '../router-reducer/fetch-server-response'
 import {
@@ -52,6 +51,7 @@ import type {
 } from '../../../server/app-render/types'
 import { normalizeFlightData } from '../../flight-data-helpers'
 import { STATIC_STALETIME_MS } from '../router-reducer/prefetch-cache-utils'
+import { RSC_SEGMENTS_DIR_SUFFIX } from '../../../lib/constants'
 
 // A note on async/await when working in the prefetch cache:
 //
@@ -205,6 +205,10 @@ export type NonEmptySegmentCacheEntry = Exclude<
   EmptySegmentCacheEntry
 >
 
+const isOutputExportMode =
+  process.env.NODE_ENV === 'production' &&
+  process.env.__NEXT_CONFIG_OUTPUT === 'export'
+
 // Route cache entries vary on multiple keys: the href and the Next-Url. Each of
 // these parts needs to be included in the internal cache key. Rather than
 // concatenate the keys into a single key, we use a multi-level map, where the
@@ -825,7 +829,9 @@ export async function fetchRouteOnCacheMiss(
     // This is a bit convoluted but it's taken from router-reducer and
     // fetch-server-response
     const canonicalUrl = response.redirected
-      ? createHrefFromUrl(urlToUrlWithoutFlightMarker(response.url))
+      ? createHrefFromUrl(
+          new URL(removeSegmentPathFromURLInOutputExportMode(response.url))
+        )
       : href
 
     // Check whether the response varies based on the Next-Url header.
@@ -839,9 +845,13 @@ export async function fetchRouteOnCacheMiss(
     // This checks whether the response was served from the per-segment cache,
     // rather than the old prefetching flow. If it fails, it implies that PPR
     // is disabled on this route.
-    // TODO: Add support for non-PPR routes.
     const routeIsPPREnabled =
-      response.headers.get(NEXT_DID_POSTPONE_HEADER) === '2'
+      response.headers.get(NEXT_DID_POSTPONE_HEADER) === '2' ||
+      // In output: "export" mode, we can't rely on response headers. But if we
+      // receive a well-formed response, we can assume it's a static response,
+      // because all data is static in this mode.
+      isOutputExportMode
+
     if (routeIsPPREnabled) {
       const prefetchStream = createPrefetchResponseStream(
         response.body,
@@ -973,7 +983,11 @@ export async function fetchSegmentOnCacheMiss(
       // is disabled on this route. Theoretically this should never happen
       // because we only issue requests for segments once we've verified that
       // the route supports PPR.
-      response.headers.get(NEXT_DID_POSTPONE_HEADER) !== '2' ||
+      (response.headers.get(NEXT_DID_POSTPONE_HEADER) !== '2' &&
+        // In output: "export" mode, we can't rely on response headers. But if
+        // we receive a well-formed response, we can assume it's a static
+        // response, because all data is static in this mode.
+        !isOutputExportMode) ||
       !response.body
     ) {
       // Server responded with an error, or with a miss. We should still cache
@@ -1334,21 +1348,38 @@ async function fetchSegmentPrefetchResponse(
   if (nextUrl !== null) {
     headers[NEXT_URL] = nextUrl
   }
-  return fetchPrefetchResponse(href, headers)
+  return fetchPrefetchResponse(
+    isOutputExportMode
+      ? addSegmentPathToUrlInOutputExportMode(href, segmentPath)
+      : href,
+    headers
+  )
 }
 
 async function fetchPrefetchResponse(
-  href: NormalizedHref,
+  href: string,
   headers: RequestHeaders
 ): Promise<Response | null> {
   const fetchPriority = 'low'
   const response = await createFetch(new URL(href), headers, fetchPriority)
-  const contentType = response.headers.get('content-type')
-  const isFlightResponse =
-    contentType && contentType.startsWith(RSC_CONTENT_TYPE_HEADER)
-  if (!response.ok || !isFlightResponse) {
+  if (!response.ok) {
     return null
   }
+
+  // Check the content type
+  if (isOutputExportMode) {
+    // In output: "export" mode, we relaxed about the content type, since it's
+    // not Next.js that's serving the response. If the status is OK, assume the
+    // response is valid. If it's not a valid response, the Flight client won't
+    // be able to decode it, and we'll treat it as a miss.
+  } else {
+    const contentType = response.headers.get('content-type')
+    const isFlightResponse =
+      contentType && contentType.startsWith(RSC_CONTENT_TYPE_HEADER)
+    if (!isFlightResponse) {
+      return null
+    }
+  }
   return response
 }
 
@@ -1399,6 +1430,40 @@ function createPrefetchResponseStream(
   })
 }
 
+function addSegmentPathToUrlInOutputExportMode(
+  url: string,
+  segmentPath: string
+) {
+  if (isOutputExportMode) {
+    // In output: "export" mode, we cannot use a header to encode the segment
+    // path. Instead, we append it to the end of the pathname.
+    const staticUrl = new URL(url)
+    staticUrl.pathname = staticUrl.pathname.endsWith('/')
+      ? `${staticUrl.pathname.substring(0, -1)}${RSC_SEGMENTS_DIR_SUFFIX}${segmentPath}.txt`
+      : `${staticUrl.pathname}${RSC_SEGMENTS_DIR_SUFFIX}${segmentPath}.txt`
+    return staticUrl.href
+  }
+  return url
+}
+
+function removeSegmentPathFromURLInOutputExportMode(redirectUrl: string) {
+  if (isOutputExportMode) {
+    // This is the inverse of addSegmentPathToUrlInOutputExportMode. Used when
+    // a prefetch request is redirected so we can obtain the canonical URL.
+    // TODO: How do redirects work in output: "export" mode? Do we support
+    // arbitrary redirects, like we do in the normal mode? Investigate.
+    // Remove everything that follows the final occurrence
+    // of RSC_SEGMENTS_DIR_SUFFIX
+    const staticUrl = new URL(redirectUrl)
+    const suffixIndex = staticUrl.pathname.lastIndexOf(RSC_SEGMENTS_DIR_SUFFIX)
+    if (suffixIndex !== -1) {
+      staticUrl.pathname = staticUrl.pathname.slice(0, suffixIndex)
+      return staticUrl.href
+    }
+  }
+  return redirectUrl
+}
+
 function createPromiseWithResolvers<T>(): PromiseWithResolvers<T> {
   // Shim of Stage 4 Promise.withResolvers proposal
   let resolve: (value: T | PromiseLike<T>) => void

packages/next/src/export/index.ts

@@ -12,11 +12,16 @@ import { existsSync, promises as fs } from 'fs'
 
 import '../server/require-hook'
 
-import { dirname, join, resolve, sep } from 'path'
+import { dirname, join, resolve, sep, relative } from 'path'
 import { formatAmpMessages } from '../build/output/index'
 import type { AmpPageStatus } from '../build/output/index'
 import * as Log from '../build/output/log'
-import { RSC_SUFFIX, SSG_FALLBACK_EXPORT_ERROR } from '../lib/constants'
+import {
+  RSC_SEGMENT_SUFFIX,
+  RSC_SEGMENTS_DIR_SUFFIX,
+  RSC_SUFFIX,
+  SSG_FALLBACK_EXPORT_ERROR,
+} from '../lib/constants'
 import { recursiveCopy } from '../lib/recursive-copy'
 import {
   BUILD_ID_FILE,
@@ -732,6 +737,28 @@ async function exportAppImpl(
           await fs.mkdir(dirname(ampHtmlDest), { recursive: true })
           await fs.copyFile(`${orig}.amp.html`, ampHtmlDest)
         }
+
+        const segmentsDir = `${orig}${RSC_SEGMENTS_DIR_SUFFIX}`
+        if (isAppPath && existsSync(segmentsDir)) {
+          const segmentsDirDest = join(
+            outDir,
+            `${route}${RSC_SEGMENTS_DIR_SUFFIX}`
+          )
+          const segmentPaths = await collectSegmentPaths(segmentsDir)
+          await Promise.all(
+            segmentPaths.map(async (segmentFileSrc) => {
+              const segmentFileDest = join(
+                segmentsDirDest,
+                segmentFileSrc.slice(0, -RSC_SEGMENT_SUFFIX.length) + '.txt'
+              )
+              await fs.mkdir(dirname(segmentFileDest), { recursive: true })
+              await fs.copyFile(
+                join(segmentsDir, segmentFileSrc),
+                segmentFileDest
+              )
+            })
+          )
+        }
       })
     )
   }
@@ -773,6 +800,40 @@ async function exportAppImpl(
   return collector
 }
 
+async function collectSegmentPaths(segmentsDirectory: string) {
+  const results: Array<string> = []
+  await collectSegmentPathsImpl(segmentsDirectory, segmentsDirectory, results)
+  return results
+}
+
+async function collectSegmentPathsImpl(
+  segmentsDirectory: string,
+  directory: string,
+  results: Array<string>
+) {
+  const segmentFiles = await fs.readdir(directory, {
+    withFileTypes: true,
+  })
+  await Promise.all(
+    segmentFiles.map(async (segmentFile) => {
+      if (segmentFile.isDirectory()) {
+        await collectSegmentPathsImpl(
+          segmentsDirectory,
+          join(directory, segmentFile.name),
+          results
+        )
+        return
+      }
+      if (!segmentFile.name.endsWith(RSC_SEGMENT_SUFFIX)) {
+        return
+      }
+      results.push(
+        relative(segmentsDirectory, join(directory, segmentFile.name))
+      )
+    })
+  )
+}
+
 export default async function exportApp(
   dir: string,
   options: ExportAppOptions,

test/e2e/app-dir/segment-cache/export/app/layout.tsx

@@ -0,0 +1,11 @@
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode
+}) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

test/e2e/app-dir/segment-cache/export/app/page.tsx

@@ -0,0 +1,17 @@
+import { LinkAccordion } from '../components/link-accordion'
+
+export default function OutputExport() {
+  return (
+    <>
+      <p>
+        Demonstrates that per-segment prefetching works in{' '}
+        <code>output: export</code> mode.
+      </p>
+      <ul>
+        <li>
+          <LinkAccordion href="/target-page">Target</LinkAccordion>
+        </li>
+      </ul>
+    </>
+  )
+}

test/e2e/app-dir/segment-cache/export/app/target-page/page.tsx

@@ -0,0 +1,3 @@
+export default function Target() {
+  return <div id="target-page">Target page</div>
+}

test/e2e/app-dir/segment-cache/export/components/link-accordion.tsx

@@ -0,0 +1,23 @@
+'use client'
+
+import Link from 'next/link'
+import { useState } from 'react'
+
+export function LinkAccordion({ href, children }) {
+  const [isVisible, setIsVisible] = useState(false)
+  return (
+    <>
+      <input
+        type="checkbox"
+        checked={isVisible}
+        onChange={() => setIsVisible(!isVisible)}
+        data-link-accordion={href}
+      />
+      {isVisible ? (
+        <Link href={href}>{children}</Link>
+      ) : (
+        `${children} (link is hidden)`
+      )}
+    </>
+  )
+}

test/e2e/app-dir/segment-cache/export/next.config.js

@@ -0,0 +1,12 @@
+/**
+ * @type {import('next').NextConfig}
+ */
+const nextConfig = {
+  output: 'export',
+  experimental: {
+    dynamicIO: true,
+    clientSegmentCache: true,
+  },
+}
+
+module.exports = nextConfig