From 1f99c3009f3a6173cfa580e4c248f5f4e3ca4e2a Mon Sep 17 00:00:00 2001 From: Gerald Monaco Date: Fri, 10 Sep 2021 17:17:56 -0700 Subject: [PATCH] Use `Writable` instead of `Observable` (#29007) Use `Writable` instead of `Observable` and remove the `zen-observable` dependencies. I initially opted to use `Observable` for simplicity and fast iteration, but we should really just use `Writable` directly (or some other stream in the future). React's streaming SSR has some [specific requirements](https://github.com/reactwg/react-18/discussions/66#discussioncomment-944266) on the stream API. Rather than trying to also squeeze a `Readable` in here, which might be more standard for node apps, I've just followed React's lead. By limiting ourselves to just `Writable`, it ought to be easier to adopt a different stream type in the future if desired. The React `pipeToNodeWritable` API requires us to pass a stream immediately, but we don't actually have a `ServerResponse` to give it until `RenderResult.pipe(...)` is called later. For that reason, we pass React a `Writable` that we will simply forward to `res` later. This mechanism of deferring is `NodeWritablePiper`, which is just a function that can be called with `ServerResponse` (or another `Writable`, as we now do to render to string for static results) to have content written to it. `NodeWritablePiper` takes a `next` argument so that we can chain both synchronous and asynchronous pipers together. Also does some clean up and adds another streaming test for backpressure. --- packages/next/compiled/zen-observable/LICENSE | 18 - packages/next/compiled/zen-observable/esm.js | 1 - .../next/compiled/zen-observable/package.json | 1 - packages/next/package.json | 4 +- packages/next/server/render-result.ts | 34 +- packages/next/server/render.tsx | 738 ++++++++++-------- packages/next/taskfile.js | 11 - packages/next/types/misc.d.ts | 4 - test/integration/react-18/app/pages/_app.js | 4 +- .../app/pages/suspense/backpressure.js | 28 + test/integration/react-18/test/concurrent.js | 9 + yarn.lock | 11 - 12 files changed, 455 insertions(+), 408 deletions(-) delete mode 100644 packages/next/compiled/zen-observable/LICENSE delete mode 100644 packages/next/compiled/zen-observable/esm.js delete mode 100644 packages/next/compiled/zen-observable/package.json create mode 100644 test/integration/react-18/app/pages/suspense/backpressure.js diff --git a/packages/next/compiled/zen-observable/LICENSE b/packages/next/compiled/zen-observable/LICENSE deleted file mode 100644 index d850f52720232..0000000000000 --- a/packages/next/compiled/zen-observable/LICENSE +++ /dev/null @@ -1,18 +0,0 @@ -Copyright (c) 2018 zenparsing (Kevin Smith) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/next/compiled/zen-observable/esm.js b/packages/next/compiled/zen-observable/esm.js deleted file mode 100644 index 141e5f414a3a3..0000000000000 --- a/packages/next/compiled/zen-observable/esm.js +++ /dev/null @@ -1 +0,0 @@ -module.exports=(()=>{"use strict";var e={343:(e,t,r)=>{r.r(t);r.d(t,{Observable:()=>Observable,combineLatest:()=>combineLatest,default:()=>l,merge:()=>merge,zip:()=>zip});const n=()=>typeof Symbol==="function";const o=e=>n()&&Boolean(Symbol[e]);const i=e=>o(e)?Symbol[e]:"@@"+e;if(n()&&!o("observable")){Symbol.observable=Symbol("observable")}const s=i("iterator");const u=i("observable");const c=i("species");function getMethod(e,t){let r=e[t];if(r==null)return undefined;if(typeof r!=="function")throw new TypeError(r+" is not a function");return r}function getSpecies(e){let t=e.constructor;if(t!==undefined){t=t[c];if(t===null){t=undefined}}return t!==undefined?t:Observable}function isObservable(e){return e instanceof Observable}function hostReportError(e){if(hostReportError.log){hostReportError.log(e)}else{setTimeout(()=>{throw e})}}function enqueue(e){Promise.resolve().then(()=>{try{e()}catch(e){hostReportError(e)}})}function cleanupSubscription(e){let t=e._cleanup;if(t===undefined)return;e._cleanup=undefined;if(!t){return}try{if(typeof t==="function"){t()}else{let e=getMethod(t,"unsubscribe");if(e){e.call(t)}}}catch(e){hostReportError(e)}}function closeSubscription(e){e._observer=undefined;e._queue=undefined;e._state="closed"}function flushSubscription(e){let t=e._queue;if(!t){return}e._queue=undefined;e._state="ready";for(let r=0;rflushSubscription(e));return}notifySubscription(e,t,r)}class Subscription{constructor(e,t){this._cleanup=undefined;this._observer=e;this._queue=undefined;this._state="initializing";let r=new SubscriptionObserver(this);try{this._cleanup=t.call(undefined,r)}catch(e){r.error(e)}if(this._state==="initializing")this._state="ready"}get closed(){return this._state==="closed"}unsubscribe(){if(this._state!=="closed"){closeSubscription(this);cleanupSubscription(this)}}}class SubscriptionObserver{constructor(e){this._subscription=e}get closed(){return this._subscription._state==="closed"}next(e){onNotify(this._subscription,"next",e)}error(e){onNotify(this._subscription,"error",e)}complete(){onNotify(this._subscription,"complete")}}class Observable{constructor(e){if(!(this instanceof Observable))throw new TypeError("Observable cannot be called as a function");if(typeof e!=="function")throw new TypeError("Observable initializer must be a function");this._subscriber=e}subscribe(e){if(typeof e!=="object"||e===null){e={next:e,error:arguments[1],complete:arguments[2]}}return new Subscription(e,this._subscriber)}forEach(e){return new Promise((t,r)=>{if(typeof e!=="function"){r(new TypeError(e+" is not a function"));return}function done(){n.unsubscribe();t()}let n=this.subscribe({next(t){try{e(t,done)}catch(e){r(e);n.unsubscribe()}},error:r,complete:t})})}map(e){if(typeof e!=="function")throw new TypeError(e+" is not a function");let t=getSpecies(this);return new t(t=>this.subscribe({next(r){try{r=e(r)}catch(e){return t.error(e)}t.next(r)},error(e){t.error(e)},complete(){t.complete()}}))}filter(e){if(typeof e!=="function")throw new TypeError(e+" is not a function");let t=getSpecies(this);return new t(t=>this.subscribe({next(r){try{if(!e(r))return}catch(e){return t.error(e)}t.next(r)},error(e){t.error(e)},complete(){t.complete()}}))}reduce(e){if(typeof e!=="function")throw new TypeError(e+" is not a function");let t=getSpecies(this);let r=arguments.length>1;let n=false;let o=arguments[1];let i=o;return new t(t=>this.subscribe({next(o){let s=!n;n=true;if(!s||r){try{i=e(i,o)}catch(e){return t.error(e)}}else{i=o}},error(e){t.error(e)},complete(){if(!n&&!r)return t.error(new TypeError("Cannot reduce an empty sequence"));t.next(i);t.complete()}}))}concat(...e){let t=getSpecies(this);return new t(r=>{let n;let o=0;function startNext(i){n=i.subscribe({next(e){r.next(e)},error(e){r.error(e)},complete(){if(o===e.length){n=undefined;r.complete()}else{startNext(t.from(e[o++]))}}})}startNext(this);return()=>{if(n){n.unsubscribe();n=undefined}}})}flatMap(e){if(typeof e!=="function")throw new TypeError(e+" is not a function");let t=getSpecies(this);return new t(r=>{let n=[];let o=this.subscribe({next(o){if(e){try{o=e(o)}catch(e){return r.error(e)}}let i=t.from(o).subscribe({next(e){r.next(e)},error(e){r.error(e)},complete(){let e=n.indexOf(i);if(e>=0)n.splice(e,1);completeIfDone()}});n.push(i)},error(e){r.error(e)},complete(){completeIfDone()}});function completeIfDone(){if(o.closed&&n.length===0)r.complete()}return()=>{n.forEach(e=>e.unsubscribe());o.unsubscribe()}})}[u](){return this}static from(e){let t=typeof this==="function"?this:Observable;if(e==null)throw new TypeError(e+" is not an object");let r=getMethod(e,u);if(r){let n=r.call(e);if(Object(n)!==n)throw new TypeError(n+" is not an object");if(isObservable(n)&&n.constructor===t)return n;return new t(e=>n.subscribe(e))}if(o("iterator")){r=getMethod(e,s);if(r){return new t(t=>{enqueue(()=>{if(t.closed)return;for(let n of r.call(e)){t.next(n);if(t.closed)return}t.complete()})})}}if(Array.isArray(e)){return new t(t=>{enqueue(()=>{if(t.closed)return;for(let r=0;r{enqueue(()=>{if(t.closed)return;for(let r=0;r{if(e.length===0)return Observable.from([]);let r=e.length;let n=e.map(e=>Observable.from(e).subscribe({next(e){t.next(e)},error(e){t.error(e)},complete(){if(--r===0)t.complete()}}));return()=>n.forEach(e=>e.unsubscribe())})}function combineLatest(...e){return new Observable(t=>{if(e.length===0)return Observable.from([]);let r=e.length;let n=new Set;let o=false;let i=e.map(()=>undefined);let s=e.map((s,u)=>Observable.from(s).subscribe({next(r){i[u]=r;if(!o){n.add(u);if(n.size!==e.length)return;n=null;o=true}t.next(Array.from(i))},error(e){t.error(e)},complete(){if(--r===0)t.complete()}}));return()=>s.forEach(e=>e.unsubscribe())})}function zip(...e){return new Observable(t=>{if(e.length===0)return Observable.from([]);let r=e.map(()=>[]);function done(){return r.some((e,t)=>e.length===0&&n[t].closed)}let n=e.map((e,n)=>Observable.from(e).subscribe({next(e){r[n].push(e);if(r.every(e=>e.length>0)){t.next(r.map(e=>e.shift()));if(done())t.complete()}},error(e){t.error(e)},complete(){if(done())t.complete()}}));return()=>n.forEach(e=>e.unsubscribe())})}const l=Observable}};var t={};function __nccwpck_require__(r){if(t[r]){return t[r].exports}var n=t[r]={exports:{}};var o=true;try{e[r](n,n.exports,__nccwpck_require__);o=false}finally{if(o)delete t[r]}return n.exports}(()=>{__nccwpck_require__.d=((e,t)=>{for(var r in t){if(__nccwpck_require__.o(t,r)&&!__nccwpck_require__.o(e,r)){Object.defineProperty(e,r,{enumerable:true,get:t[r]})}}})})();(()=>{__nccwpck_require__.o=((e,t)=>Object.prototype.hasOwnProperty.call(e,t))})();(()=>{__nccwpck_require__.r=(e=>{if(typeof Symbol!=="undefined"&&Symbol.toStringTag){Object.defineProperty(e,Symbol.toStringTag,{value:"Module"})}Object.defineProperty(e,"__esModule",{value:true})})})();__nccwpck_require__.ab=__dirname+"/";return __nccwpck_require__(343)})(); \ No newline at end of file diff --git a/packages/next/compiled/zen-observable/package.json b/packages/next/compiled/zen-observable/package.json deleted file mode 100644 index 13ba577d97d99..0000000000000 --- a/packages/next/compiled/zen-observable/package.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"zen-observable","main":"esm.js","license":"MIT"} diff --git a/packages/next/package.json b/packages/next/package.json index 6960d5922896b..336922f478d9d 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -185,7 +185,6 @@ "@types/text-table": "0.2.1", "@types/webpack": "5.28.0", "@types/webpack-sources": "0.1.5", - "@types/zen-observable": "0.8.3", "@vercel/ncc": "0.27.0", "@vercel/nft": "0.12.2", "amphtml-validator": "1.0.33", @@ -252,8 +251,7 @@ "unistore": "3.4.1", "web-vitals": "2.1.0", "webpack": "4.44.1", - "webpack-sources": "1.4.3", - "zen-observable": "0.8.15" + "webpack-sources": "1.4.3" }, "engines": { "node": ">=12.0.0" diff --git a/packages/next/server/render-result.ts b/packages/next/server/render-result.ts index b7cf3c1f4815d..ab4d0ffc5fb08 100644 --- a/packages/next/server/render-result.ts +++ b/packages/next/server/render-result.ts @@ -1,41 +1,41 @@ import { ServerResponse } from 'http' -import Observable from 'next/dist/compiled/zen-observable' +import { Writable } from 'stream' + +export type NodeWritablePiper = ( + res: Writable, + next: (err?: Error) => void +) => void export default class RenderResult { - _response: string | Observable + _result: string | NodeWritablePiper - constructor(response: string | Observable) { - this._response = response + constructor(response: string | NodeWritablePiper) { + this._result = response } toUnchunkedString(): string { - if (typeof this._response !== 'string') { + if (typeof this._result !== 'string') { throw new Error( 'invariant: dynamic responses cannot be unchunked. This is a bug in Next.js' ) } - return this._response + return this._result } - async pipe(res: ServerResponse): Promise { - if (typeof this._response === 'string') { + pipe(res: ServerResponse): Promise { + if (typeof this._result === 'string') { throw new Error( 'invariant: static responses cannot be piped. This is a bug in Next.js' ) } - const maybeFlush = - typeof (res as any).flush === 'function' - ? () => (res as any).flush() - : () => {} - await this._response.forEach((chunk) => { - res.write(chunk) - maybeFlush() + const response = this._result + return new Promise((resolve, reject) => { + response(res, (err) => (err ? reject(err) : resolve())) }) - res.end() } isDynamic(): boolean { - return typeof this._response !== 'string' + return typeof this._result !== 'string' } static fromStatic(value: string): RenderResult { diff --git a/packages/next/server/render.tsx b/packages/next/server/render.tsx index feeeae9525e36..08d98aad10cba 100644 --- a/packages/next/server/render.tsx +++ b/packages/next/server/render.tsx @@ -1,10 +1,9 @@ import { IncomingMessage, ServerResponse } from 'http' import { ParsedUrlQuery } from 'querystring' -import { PassThrough } from 'stream' +import { Writable } from 'stream' import React from 'react' import * as ReactDOMServer from 'react-dom/server' import { StyleRegistry, createStyleRegistry } from 'styled-jsx' -import Observable from 'next/dist/compiled/zen-observable' import { warn } from '../build/output/log' import { UnwrapPromise } from '../lib/coalesced-function' import { @@ -66,7 +65,7 @@ import { Redirect, } from '../lib/load-custom-routes' import { DomainLocale } from './config' -import RenderResult from './render-result' +import RenderResult, { NodeWritablePiper } from './render-result' function noRouter() { const message = @@ -558,290 +557,278 @@ export async function renderToHTML( ) - try { - props = await loadGetInitialProps(App, { - AppTree: ctx.AppTree, - Component, - router, - ctx, - }) + props = await loadGetInitialProps(App, { + AppTree: ctx.AppTree, + Component, + router, + ctx, + }) + + if ((isSSG || getServerSideProps) && isPreview) { + props.__N_PREVIEW = true + } - if ((isSSG || getServerSideProps) && isPreview) { - props.__N_PREVIEW = true + if (isSSG) { + props[STATIC_PROPS_ID] = true + } + + if (isSSG && !isFallback) { + let data: UnwrapPromise> + + try { + data = await getStaticProps!({ + ...(pageIsDynamic ? { params: query as ParsedUrlQuery } : undefined), + ...(isPreview + ? { preview: true, previewData: previewData } + : undefined), + locales: renderOpts.locales, + locale: renderOpts.locale, + defaultLocale: renderOpts.defaultLocale, + }) + } catch (staticPropsError) { + // remove not found error code to prevent triggering legacy + // 404 rendering + if (staticPropsError.code === 'ENOENT') { + delete staticPropsError.code + } + throw staticPropsError } - if (isSSG) { - props[STATIC_PROPS_ID] = true + if (data == null) { + throw new Error(GSP_NO_RETURNED_VALUE) } - if (isSSG && !isFallback) { - let data: UnwrapPromise> - - try { - data = await getStaticProps!({ - ...(pageIsDynamic ? { params: query as ParsedUrlQuery } : undefined), - ...(isPreview - ? { preview: true, previewData: previewData } - : undefined), - locales: renderOpts.locales, - locale: renderOpts.locale, - defaultLocale: renderOpts.defaultLocale, - }) - } catch (staticPropsError) { - // remove not found error code to prevent triggering legacy - // 404 rendering - if (staticPropsError.code === 'ENOENT') { - delete staticPropsError.code - } - throw staticPropsError - } + const invalidKeys = Object.keys(data).filter( + (key) => + key !== 'revalidate' && + key !== 'props' && + key !== 'redirect' && + key !== 'notFound' + ) - if (data == null) { - throw new Error(GSP_NO_RETURNED_VALUE) - } + if (invalidKeys.includes('unstable_revalidate')) { + throw new Error(UNSTABLE_REVALIDATE_RENAME_ERROR) + } - const invalidKeys = Object.keys(data).filter( - (key) => - key !== 'revalidate' && - key !== 'props' && - key !== 'redirect' && - key !== 'notFound' - ) + if (invalidKeys.length) { + throw new Error(invalidKeysMsg('getStaticProps', invalidKeys)) + } - if (invalidKeys.includes('unstable_revalidate')) { - throw new Error(UNSTABLE_REVALIDATE_RENAME_ERROR) + if (process.env.NODE_ENV !== 'production') { + if ( + typeof (data as any).notFound !== 'undefined' && + typeof (data as any).redirect !== 'undefined' + ) { + throw new Error( + `\`redirect\` and \`notFound\` can not both be returned from ${ + isSSG ? 'getStaticProps' : 'getServerSideProps' + } at the same time. Page: ${pathname}\nSee more info here: https://nextjs.org/docs/messages/gssp-mixed-not-found-redirect` + ) } + } - if (invalidKeys.length) { - throw new Error(invalidKeysMsg('getStaticProps', invalidKeys)) + if ('notFound' in data && data.notFound) { + if (pathname === '/404') { + throw new Error( + `The /404 page can not return notFound in "getStaticProps", please remove it to continue!` + ) } - if (process.env.NODE_ENV !== 'production') { - if ( - typeof (data as any).notFound !== 'undefined' && - typeof (data as any).redirect !== 'undefined' - ) { - throw new Error( - `\`redirect\` and \`notFound\` can not both be returned from ${ - isSSG ? 'getStaticProps' : 'getServerSideProps' - } at the same time. Page: ${pathname}\nSee more info here: https://nextjs.org/docs/messages/gssp-mixed-not-found-redirect` - ) - } - } + ;(renderOpts as any).isNotFound = true + } - if ('notFound' in data && data.notFound) { - if (pathname === '/404') { - throw new Error( - `The /404 page can not return notFound in "getStaticProps", please remove it to continue!` - ) - } + if ( + 'redirect' in data && + data.redirect && + typeof data.redirect === 'object' + ) { + checkRedirectValues(data.redirect as Redirect, req, 'getStaticProps') - ;(renderOpts as any).isNotFound = true + if (isBuildTimeSSG) { + throw new Error( + `\`redirect\` can not be returned from getStaticProps during prerendering (${req.url})\n` + + `See more info here: https://nextjs.org/docs/messages/gsp-redirect-during-prerender` + ) } - if ( - 'redirect' in data && - data.redirect && - typeof data.redirect === 'object' - ) { - checkRedirectValues(data.redirect as Redirect, req, 'getStaticProps') + ;(data as any).props = { + __N_REDIRECT: data.redirect.destination, + __N_REDIRECT_STATUS: getRedirectStatus(data.redirect), + } + if (typeof data.redirect.basePath !== 'undefined') { + ;(data as any).props.__N_REDIRECT_BASE_PATH = data.redirect.basePath + } + ;(renderOpts as any).isRedirect = true + } - if (isBuildTimeSSG) { + if ( + (dev || isBuildTimeSSG) && + !(renderOpts as any).isNotFound && + !isSerializableProps(pathname, 'getStaticProps', (data as any).props) + ) { + // this fn should throw an error instead of ever returning `false` + throw new Error( + 'invariant: getStaticProps did not return valid props. Please report this.' + ) + } + + if ('revalidate' in data) { + if (typeof data.revalidate === 'number') { + if (!Number.isInteger(data.revalidate)) { throw new Error( - `\`redirect\` can not be returned from getStaticProps during prerendering (${req.url})\n` + - `See more info here: https://nextjs.org/docs/messages/gsp-redirect-during-prerender` + `A page's revalidate option must be seconds expressed as a natural number for ${req.url}. Mixed numbers, such as '${data.revalidate}', cannot be used.` + + `\nTry changing the value to '${Math.ceil( + data.revalidate + )}' or using \`Math.ceil()\` if you're computing the value.` + ) + } else if (data.revalidate <= 0) { + throw new Error( + `A page's revalidate option can not be less than or equal to zero for ${req.url}. A revalidate option of zero means to revalidate after _every_ request, and implies stale data cannot be tolerated.` + + `\n\nTo never revalidate, you can set revalidate to \`false\` (only ran once at build-time).` + + `\nTo revalidate as soon as possible, you can set the value to \`1\`.` + ) + } else if (data.revalidate > 31536000) { + // if it's greater than a year for some reason error + console.warn( + `Warning: A page's revalidate option was set to more than a year for ${req.url}. This may have been done in error.` + + `\nTo only run getStaticProps at build-time and not revalidate at runtime, you can set \`revalidate\` to \`false\`!` ) } - - ;(data as any).props = { - __N_REDIRECT: data.redirect.destination, - __N_REDIRECT_STATUS: getRedirectStatus(data.redirect), - } - if (typeof data.redirect.basePath !== 'undefined') { - ;(data as any).props.__N_REDIRECT_BASE_PATH = data.redirect.basePath - } - ;(renderOpts as any).isRedirect = true - } - - if ( - (dev || isBuildTimeSSG) && - !(renderOpts as any).isNotFound && - !isSerializableProps(pathname, 'getStaticProps', (data as any).props) + } else if (data.revalidate === true) { + // When enabled, revalidate after 1 second. This value is optimal for + // the most up-to-date page possible, but without a 1-to-1 + // request-refresh ratio. + data.revalidate = 1 + } else if ( + data.revalidate === false || + typeof data.revalidate === 'undefined' ) { - // this fn should throw an error instead of ever returning `false` + // By default, we never revalidate. + data.revalidate = false + } else { throw new Error( - 'invariant: getStaticProps did not return valid props. Please report this.' + `A page's revalidate option must be seconds expressed as a natural number. Mixed numbers and strings cannot be used. Received '${JSON.stringify( + data.revalidate + )}' for ${req.url}` ) } + } else { + // By default, we never revalidate. + ;(data as any).revalidate = false + } - if ('revalidate' in data) { - if (typeof data.revalidate === 'number') { - if (!Number.isInteger(data.revalidate)) { - throw new Error( - `A page's revalidate option must be seconds expressed as a natural number for ${req.url}. Mixed numbers, such as '${data.revalidate}', cannot be used.` + - `\nTry changing the value to '${Math.ceil( - data.revalidate - )}' or using \`Math.ceil()\` if you're computing the value.` - ) - } else if (data.revalidate <= 0) { - throw new Error( - `A page's revalidate option can not be less than or equal to zero for ${req.url}. A revalidate option of zero means to revalidate after _every_ request, and implies stale data cannot be tolerated.` + - `\n\nTo never revalidate, you can set revalidate to \`false\` (only ran once at build-time).` + - `\nTo revalidate as soon as possible, you can set the value to \`1\`.` - ) - } else if (data.revalidate > 31536000) { - // if it's greater than a year for some reason error - console.warn( - `Warning: A page's revalidate option was set to more than a year for ${req.url}. This may have been done in error.` + - `\nTo only run getStaticProps at build-time and not revalidate at runtime, you can set \`revalidate\` to \`false\`!` - ) - } - } else if (data.revalidate === true) { - // When enabled, revalidate after 1 second. This value is optimal for - // the most up-to-date page possible, but without a 1-to-1 - // request-refresh ratio. - data.revalidate = 1 - } else if ( - data.revalidate === false || - typeof data.revalidate === 'undefined' - ) { - // By default, we never revalidate. - data.revalidate = false - } else { - throw new Error( - `A page's revalidate option must be seconds expressed as a natural number. Mixed numbers and strings cannot be used. Received '${JSON.stringify( - data.revalidate - )}' for ${req.url}` - ) - } - } else { - // By default, we never revalidate. - ;(data as any).revalidate = false - } + props.pageProps = Object.assign( + {}, + props.pageProps, + 'props' in data ? data.props : undefined + ) - props.pageProps = Object.assign( - {}, - props.pageProps, - 'props' in data ? data.props : undefined - ) + // pass up revalidate and props for export + // TODO: change this to a different passing mechanism + ;(renderOpts as any).revalidate = + 'revalidate' in data ? data.revalidate : undefined + ;(renderOpts as any).pageData = props - // pass up revalidate and props for export - // TODO: change this to a different passing mechanism - ;(renderOpts as any).revalidate = - 'revalidate' in data ? data.revalidate : undefined - ;(renderOpts as any).pageData = props + // this must come after revalidate is added to renderOpts + if ((renderOpts as any).isNotFound) { + return null + } + } - // this must come after revalidate is added to renderOpts - if ((renderOpts as any).isNotFound) { - return null + if (getServerSideProps) { + props[SERVER_PROPS_ID] = true + } + + if (getServerSideProps && !isFallback) { + let data: UnwrapPromise> + + try { + data = await getServerSideProps({ + req: req as IncomingMessage & { + cookies: NextApiRequestCookies + }, + res, + query, + resolvedUrl: renderOpts.resolvedUrl as string, + ...(pageIsDynamic ? { params: params as ParsedUrlQuery } : undefined), + ...(previewData !== false + ? { preview: true, previewData: previewData } + : undefined), + locales: renderOpts.locales, + locale: renderOpts.locale, + defaultLocale: renderOpts.defaultLocale, + }) + } catch (serverSidePropsError) { + // remove not found error code to prevent triggering legacy + // 404 rendering + if (serverSidePropsError.code === 'ENOENT') { + delete serverSidePropsError.code } + throw serverSidePropsError } - if (getServerSideProps) { - props[SERVER_PROPS_ID] = true + if (data == null) { + throw new Error(GSSP_NO_RETURNED_VALUE) } - if (getServerSideProps && !isFallback) { - let data: UnwrapPromise> - - try { - data = await getServerSideProps({ - req: req as IncomingMessage & { - cookies: NextApiRequestCookies - }, - res, - query, - resolvedUrl: renderOpts.resolvedUrl as string, - ...(pageIsDynamic ? { params: params as ParsedUrlQuery } : undefined), - ...(previewData !== false - ? { preview: true, previewData: previewData } - : undefined), - locales: renderOpts.locales, - locale: renderOpts.locale, - defaultLocale: renderOpts.defaultLocale, - }) - } catch (serverSidePropsError) { - // remove not found error code to prevent triggering legacy - // 404 rendering - if (serverSidePropsError.code === 'ENOENT') { - delete serverSidePropsError.code - } - throw serverSidePropsError - } - - if (data == null) { - throw new Error(GSSP_NO_RETURNED_VALUE) - } + const invalidKeys = Object.keys(data).filter( + (key) => key !== 'props' && key !== 'redirect' && key !== 'notFound' + ) - const invalidKeys = Object.keys(data).filter( - (key) => key !== 'props' && key !== 'redirect' && key !== 'notFound' + if ((data as any).unstable_notFound) { + throw new Error( + `unstable_notFound has been renamed to notFound, please update the field to continue. Page: ${pathname}` ) + } + if ((data as any).unstable_redirect) { + throw new Error( + `unstable_redirect has been renamed to redirect, please update the field to continue. Page: ${pathname}` + ) + } - if ((data as any).unstable_notFound) { - throw new Error( - `unstable_notFound has been renamed to notFound, please update the field to continue. Page: ${pathname}` - ) - } - if ((data as any).unstable_redirect) { + if (invalidKeys.length) { + throw new Error(invalidKeysMsg('getServerSideProps', invalidKeys)) + } + + if ('notFound' in data && data.notFound) { + if (pathname === '/404') { throw new Error( - `unstable_redirect has been renamed to redirect, please update the field to continue. Page: ${pathname}` + `The /404 page can not return notFound in "getStaticProps", please remove it to continue!` ) } - if (invalidKeys.length) { - throw new Error(invalidKeysMsg('getServerSideProps', invalidKeys)) - } - - if ('notFound' in data && data.notFound) { - if (pathname === '/404') { - throw new Error( - `The /404 page can not return notFound in "getStaticProps", please remove it to continue!` - ) - } + ;(renderOpts as any).isNotFound = true + return null + } - ;(renderOpts as any).isNotFound = true - return null + if ('redirect' in data && typeof data.redirect === 'object') { + checkRedirectValues(data.redirect as Redirect, req, 'getServerSideProps') + ;(data as any).props = { + __N_REDIRECT: data.redirect.destination, + __N_REDIRECT_STATUS: getRedirectStatus(data.redirect), } - - if ('redirect' in data && typeof data.redirect === 'object') { - checkRedirectValues( - data.redirect as Redirect, - req, - 'getServerSideProps' - ) - ;(data as any).props = { - __N_REDIRECT: data.redirect.destination, - __N_REDIRECT_STATUS: getRedirectStatus(data.redirect), - } - if (typeof data.redirect.basePath !== 'undefined') { - ;(data as any).props.__N_REDIRECT_BASE_PATH = data.redirect.basePath - } - ;(renderOpts as any).isRedirect = true - } - - if ((data as any).props instanceof Promise) { - ;(data as any).props = await (data as any).props + if (typeof data.redirect.basePath !== 'undefined') { + ;(data as any).props.__N_REDIRECT_BASE_PATH = data.redirect.basePath } + ;(renderOpts as any).isRedirect = true + } - if ( - (dev || isBuildTimeSSG) && - !isSerializableProps( - pathname, - 'getServerSideProps', - (data as any).props - ) - ) { - // this fn should throw an error instead of ever returning `false` - throw new Error( - 'invariant: getServerSideProps did not return valid props. Please report this.' - ) - } + if ((data as any).props instanceof Promise) { + ;(data as any).props = await (data as any).props + } - props.pageProps = Object.assign({}, props.pageProps, (data as any).props) - ;(renderOpts as any).pageData = props + if ( + (dev || isBuildTimeSSG) && + !isSerializableProps(pathname, 'getServerSideProps', (data as any).props) + ) { + // this fn should throw an error instead of ever returning `false` + throw new Error( + 'invariant: getServerSideProps did not return valid props. Please report this.' + ) } - } catch (dataFetchError) { - throw dataFetchError + + props.pageProps = Object.assign({}, props.pageProps, (data as any).props) + ;(renderOpts as any).pageData = props } if ( @@ -912,53 +899,6 @@ export async function renderToHTML( * coalescing, and ISR continue working as intended. */ const generateStaticHTML = supportsDynamicHTML !== true - const renderToStream = (element: React.ReactElement) => - new Promise>((resolve, reject) => { - const stream = new PassThrough() - let resolved = false - const doResolve = () => { - if (!resolved) { - resolved = true - - resolve( - new Observable((observer) => { - stream.on('data', (chunk) => { - observer.next(chunk.toString('utf-8')) - }) - stream.once('end', () => { - observer.complete() - }) - - startWriting() - return () => { - abort() - } - }) - ) - } - } - - const { abort, startWriting } = ( - ReactDOMServer as any - ).pipeToNodeWritable(element, stream, { - onError(error: Error) { - if (!resolved) { - resolved = true - reject(error) - } - abort() - }, - onReadyToStream() { - if (!generateStaticHTML) { - doResolve() - } - }, - onCompleteAll() { - doResolve() - }, - }) - }) - const renderDocument = async () => { if (Document.getInitialProps) { const renderPage: RenderPage = ( @@ -1007,7 +947,7 @@ export async function renderToHTML( } return { - bodyResult: Observable.of(docProps.html), + bodyResult: piperFromArray([docProps.html]), documentElement: (htmlProps: HtmlProps) => ( ), @@ -1025,8 +965,8 @@ export async function renderToHTML( ) const bodyResult = concurrentFeatures - ? await renderToStream(content) - : Observable.of(ReactDOMServer.renderToString(content)) + ? await renderToStream(content, generateStaticHTML) + : piperFromArray([ReactDOMServer.renderToString(content)]) return { bodyResult, @@ -1156,16 +1096,21 @@ export async function renderToHTML( } } - let results: Array> = [] const renderTargetIdx = documentHTML.indexOf(BODY_RENDER_TARGET) - results.push('' + documentHTML.substring(0, renderTargetIdx)) + const prefix: Array = [] + prefix.push('') + prefix.push(documentHTML.substring(0, renderTargetIdx)) if (inAmpMode) { - results.push('') + prefix.push('') } - results.push(documentResult.bodyResult) - results.push( - documentHTML.substring(renderTargetIdx + BODY_RENDER_TARGET.length) - ) + + let pipers: Array = [ + piperFromArray(prefix), + documentResult.bodyResult, + piperFromArray([ + documentHTML.substring(renderTargetIdx + BODY_RENDER_TARGET.length), + ]), + ] const postProcessors: Array<((html: string) => Promise) | null> = ( generateStaticHTML @@ -1217,54 +1162,17 @@ export async function renderToHTML( : [] ).filter(Boolean) - if (postProcessors.length > 0) { - let html = (await observableToChunks(resultsToObservable(results))).join('') + if (generateStaticHTML || postProcessors.length > 0) { + let html = await piperToString(chainPipers(pipers)) for (const postProcessor of postProcessors) { if (postProcessor) { html = await postProcessor(html) } } - results = [html] + return new RenderResult(html) } - return new RenderResult( - generateStaticHTML - ? (await observableToChunks(resultsToObservable(results))).join('') - : resultsToObservable(results) - ) -} - -function resultsToObservable( - results: Array> -): Observable { - const observables: Array> = [] - let stringBuffer: Array | null = null - - for (const result of results) { - if (typeof result === 'string') { - stringBuffer = stringBuffer ?? [] - stringBuffer.push(result) - } else { - if (stringBuffer) { - observables.push(Observable.from(stringBuffer)) - stringBuffer = null - } - observables.push(result) - } - } - if (stringBuffer) { - observables.push(Observable.from(stringBuffer)) - } - // @ts-ignore - return Observable.prototype.concat.call(...observables) -} - -async function observableToChunks( - observable: Observable -): Promise { - const chunks: string[] = [] - await observable.forEach((chunk) => chunks.push(chunk)) - return chunks + return new RenderResult(chainPipers(pipers)) } function errorToJSON(err: Error): Error { @@ -1286,3 +1194,155 @@ function serializeError( statusCode: 500, } } + +function renderToStream( + element: React.ReactElement, + generateStaticHTML: boolean +): Promise { + return new Promise((resolve, reject) => { + let underlyingStream: { + resolve: (error?: Error) => void + writable: Writable + queuedCallbacks: Array<() => void> + } | null = null + const stream = new Writable({ + // Use the buffer from the underlying stream + highWaterMark: 0, + write(chunk, encoding, callback) { + if (!underlyingStream) { + throw new Error( + 'invariant: write called without an underlying stream. This is a bug in Next.js' + ) + } + if (!underlyingStream.writable.write(chunk, encoding)) { + underlyingStream.queuedCallbacks.push(() => callback()) + } else { + callback() + } + }, + }) + stream.once('finish', () => { + if (!underlyingStream) { + throw new Error( + 'invariant: finish called without an underlying stream. This is a bug in Next.js' + ) + } + underlyingStream.resolve() + }) + stream.once('error', (err) => { + if (!underlyingStream) { + throw new Error( + 'invariant: error called without an underlying stream. This is a bug in Next.js' + ) + } + underlyingStream.resolve(err) + }) + // React uses `flush` to prevent stream middleware like gzip from buffering to the + // point of harming streaming performance, so we make sure to expose it and forward it. + // See: https://github.com/reactwg/react-18/discussions/91 + Object.defineProperty(stream, 'flush', { + value: () => { + if (!underlyingStream) { + throw new Error( + 'invariant: flush called without an underlying stream. This is a bug in Next.js' + ) + } + if (typeof (underlyingStream.writable as any).flush === 'function') { + ;(underlyingStream.writable as any).flush() + } + }, + enumerable: true, + }) + + let resolved = false + const doResolve = () => { + if (!resolved) { + resolved = true + resolve((res, next) => { + const drainHandler = () => { + const prevCallbacks = underlyingStream!.queuedCallbacks + underlyingStream!.queuedCallbacks = [] + prevCallbacks.forEach((callback) => callback()) + } + res.on('drain', drainHandler) + underlyingStream = { + resolve: (err) => { + underlyingStream = null + res.removeListener('drain', drainHandler) + next(err) + }, + writable: res, + queuedCallbacks: [], + } + startWriting() + }) + } + } + + const { abort, startWriting } = (ReactDOMServer as any).pipeToNodeWritable( + element, + stream, + { + onError(error: Error) { + if (!resolved) { + resolved = true + reject(error) + } + abort() + }, + onReadyToStream() { + if (!generateStaticHTML) { + doResolve() + } + }, + onCompleteAll() { + doResolve() + }, + } + ) + }) +} + +function chainPipers(pipers: NodeWritablePiper[]): NodeWritablePiper { + return pipers.reduceRight( + (lhs, rhs) => (res, next) => { + rhs(res, (err) => (err ? next(err) : lhs(res, next))) + }, + (res, next) => { + res.end() + next() + } + ) +} + +function piperFromArray(chunks: string[]): NodeWritablePiper { + return (res, next) => { + if (typeof (res as any).cork === 'function') { + res.cork() + } + chunks.forEach((chunk) => res.write(chunk)) + if (typeof (res as any).uncork === 'function') { + res.uncork() + } + next() + } +} + +function piperToString(input: NodeWritablePiper): Promise { + return new Promise((resolve, reject) => { + const bufferedChunks: Buffer[] = [] + const stream = new Writable({ + writev(chunks, callback) { + chunks.forEach((chunk) => bufferedChunks.push(chunk.chunk)) + callback() + }, + }) + input(stream, (err) => { + if (err) { + reject(err) + } else { + resolve(Buffer.concat(bufferedChunks).toString()) + } + }) + }) +} diff --git a/packages/next/taskfile.js b/packages/next/taskfile.js index 880ebbb8ba689..a6d35b1c1c45f 100644 --- a/packages/next/taskfile.js +++ b/packages/next/taskfile.js @@ -709,16 +709,6 @@ export async function ncc_web_vitals(task, opts) { .target('compiled/web-vitals') } // eslint-disable-next-line camelcase -externals['zen-observable'] = 'next/dist/compiled/zen-observable' -export async function ncc_zen_observable(task, opts) { - await task - .source( - opts.src || relative(__dirname, require.resolve('zen-observable/esm')) - ) - .ncc({ packageName: 'zen-observable', externals }) - .target('compiled/zen-observable') -} -// eslint-disable-next-line camelcase externals['webpack-sources'] = 'next/dist/compiled/webpack-sources' export async function ncc_webpack_sources(task, opts) { await task @@ -910,7 +900,6 @@ export async function ncc(task, opts) { 'ncc_text_table', 'ncc_unistore', 'ncc_web_vitals', - 'ncc_zen_observable', 'ncc_webpack_bundle4', 'ncc_webpack_bundle5', 'ncc_webpack_bundle_packages', diff --git a/packages/next/types/misc.d.ts b/packages/next/types/misc.d.ts index f053560806c3c..bab7e6dd90cd3 100644 --- a/packages/next/types/misc.d.ts +++ b/packages/next/types/misc.d.ts @@ -218,10 +218,6 @@ declare module 'next/dist/compiled/web-vitals' { import m from 'web-vitals' export = m } -declare module 'next/dist/compiled/zen-observable' { - import m from 'zen-observable' - export = m -} declare module 'next/dist/compiled/comment-json' { import m from 'comment-json' diff --git a/test/integration/react-18/app/pages/_app.js b/test/integration/react-18/app/pages/_app.js index db9e691d37e02..1f136e7e8a8dc 100644 --- a/test/integration/react-18/app/pages/_app.js +++ b/test/integration/react-18/app/pages/_app.js @@ -1,10 +1,8 @@ import { PromiseCacheProvider } from '../components/promise-cache' -const cache = new Map() - function MyApp({ Component, pageProps }) { return ( - + ) diff --git a/test/integration/react-18/app/pages/suspense/backpressure.js b/test/integration/react-18/app/pages/suspense/backpressure.js new file mode 100644 index 0000000000000..3c7218950e58a --- /dev/null +++ b/test/integration/react-18/app/pages/suspense/backpressure.js @@ -0,0 +1,28 @@ +import React from 'react' +import { useCachedPromise } from '../../components/promise-cache' + +export default function Backpressure() { + const elements = [] + for (let i = 0; i < 2000; i++) { + elements.push( + loading...}> + + + ) + } + return {elements} +} + +function Item({ idx }) { + const key = idx.toString() + if (typeof window === 'undefined') { + // eslint-disable-next-line react-hooks/rules-of-hooks + useCachedPromise(key, () => Promise.resolve(), true) + } + return {key} +} + +// Disable offline build +export function getServerSideProps() { + return { props: {} } +} diff --git a/test/integration/react-18/test/concurrent.js b/test/integration/react-18/test/concurrent.js index cec3937b75d36..2f11225ba7821 100644 --- a/test/integration/react-18/test/concurrent.js +++ b/test/integration/react-18/test/concurrent.js @@ -61,4 +61,13 @@ export default (context, _render) => { ) }) }) + + it('should drain the entire response', async () => { + await withBrowser('/suspense/backpressure', async (browser) => { + await check( + () => browser.eval('document.querySelectorAll(".item").length'), + /2000/ + ) + }) + }) } diff --git a/yarn.lock b/yarn.lock index 9ec65c68604d9..f7d8aad8621c9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4891,11 +4891,6 @@ dependencies: "@types/yargs-parser" "*" -"@types/zen-observable@0.8.3": - version "0.8.3" - resolved "https://registry.yarnpkg.com/@types/zen-observable/-/zen-observable-0.8.3.tgz#781d360c282436494b32fe7d9f7f8e64b3118aa3" - integrity sha512-fbF6oTd4sGGy0xjHPKAt+eS2CrxJ3+6gQ3FGcBoIJR2TLAyCkCyI8JqZNy+FeON0AhVgNJoUumVoZQjBFUqHkw== - "@typescript-eslint/eslint-plugin@4.29.1": version "4.29.1" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.29.1.tgz#808d206e2278e809292b5de752a91105da85860b" @@ -13722,7 +13717,6 @@ minipass-fetch@^1.3.0, minipass-fetch@^1.3.2: resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-1.3.3.tgz#34c7cea038c817a8658461bf35174551dce17a0a" integrity sha512-akCrLDWfbdAWkMLBxJEeWTdNsjML+dt5YgOI4gJ53vuO0vrmYQkUPxa6j6V65s9CcePIr2SSWqjT2EcrNseryQ== dependencies: - encoding "^0.1.12" minipass "^3.1.0" minipass-sized "^1.0.3" minizlib "^2.0.0" @@ -20495,11 +20489,6 @@ yocto-queue@^0.1.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -zen-observable@0.8.15: - version "0.8.15" - resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15" - integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ== - zwitch@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920"