From 9322ba705ce74b00f57250439161d305d5f4d408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=BA=E6=99=BA=E7=9A=84=E5=B0=8F=E9=B1=BC=E5=90=9B?= <44761872+dragon-fish@users.noreply.github.com> Date: Sat, 2 Mar 2024 07:53:05 +0000 Subject: [PATCH 1/6] fix: release workers after transform --- src/utils/UgoiraPlayer.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils/UgoiraPlayer.ts b/src/utils/UgoiraPlayer.ts index 4fda14b0..7828d927 100644 --- a/src/utils/UgoiraPlayer.ts +++ b/src/utils/UgoiraPlayer.ts @@ -175,6 +175,11 @@ export class UgoiraPlayer { }) encoder.on('finished', async (blob) => { console.info('[ENCODER]', 'render finished', encoder) + // FIXME: 渲染结束时释放内存 + // @ts-ignore + encoder.freeWorkers?.forEach((worker: Worker) => { + worker && worker.terminate() + }) resolve(blob) }) console.info('[ENCODER]', 'render start', encoder) From 4c8b99eccc04c49e88808ac08137bd3543c111c2 Mon Sep 17 00:00:00 2001 From: Dragon-Fish Date: Tue, 5 Mar 2024 15:41:35 +0800 Subject: [PATCH 2/6] fix: close #92 --- api/http.ts | 106 +------------------------------- api/image.ts | 4 +- api/random.ts | 12 ++-- api/user.ts | 4 +- api/utils.ts | 163 ++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 115 deletions(-) create mode 100644 api/utils.ts diff --git a/api/http.ts b/api/http.ts index 99d6c588..f74dd74f 100644 --- a/api/http.ts +++ b/api/http.ts @@ -1,20 +1,6 @@ import type { VercelRequest, VercelResponse } from '@vercel/node' -import axios from 'axios' -import colors from 'picocolors' import escapeRegExp from 'lodash.escaperegexp' - -const PROD = process.env.NODE_ENV === 'production' - -export class CookieUtils { - static toJSON(raw: string) { - return Object.fromEntries(new URLSearchParams(raw.replace(/;\s*/g, '&'))) - } - static toString(obj: any) { - return Object.keys(obj) - .map((i) => `${i}=${obj[i]}`) - .join(';') - } -} +import { ajax } from './utils.js' export default async function (req: VercelRequest, res: VercelResponse) { if (!isAccepted(req)) { @@ -36,96 +22,6 @@ export default async function (req: VercelRequest, res: VercelResponse) { } } -export const ajax = axios.create({ - baseURL: 'https://www.pixiv.net/', - params: {}, - headers: { - 'User-Agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.69', - }, - timeout: 9 * 1000, -}) -ajax.interceptors.request.use((ctx) => { - // 去除内部参数 - ctx.params = ctx.params || {} - delete ctx.params.__PATH - delete ctx.params.__PREFIX - - const cookies = CookieUtils.toJSON(ctx.headers.cookie || '') - const csrfToken = ctx.headers['x-csrf-token'] ?? cookies.CSRFTOKEN ?? '' - // 强制覆写部分 headers - ctx.headers = ctx.headers || {} - ctx.headers.host = 'www.pixiv.net' - ctx.headers.origin = 'https://www.pixiv.net' - ctx.headers.referer = 'https://www.pixiv.net/' - ctx.headers['user-agent'] = - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.69' - ctx.headers['accept-language'] ??= - 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6' - csrfToken && (ctx.headers['x-csrf-token'] = csrfToken) - - if (!PROD) { - console.info( - colors.green(`[${ctx.method?.toUpperCase()}] <`), - colors.cyan(ctx.url || '') - ) - console.info({ - params: ctx.params, - data: ctx.data, - cookies, - }) - } - - return ctx -}) -ajax.interceptors.response.use((ctx) => { - typeof ctx.data === 'object' && - (ctx.data = replaceUrlInObject(ctx.data?.body ?? ctx.data)) - if (!PROD) { - const out: string = - typeof ctx.data === 'object' - ? JSON.stringify(ctx.data, null, 2) - : ctx.data.toString().trim() - console.info( - colors.green('[SEND] >'), - colors.cyan(ctx.request?.path?.replace('https://www.pixiv.net', '')), - `\n${colors.yellow(typeof ctx.data)} ${ - out.length >= 200 ? out.slice(0, 200).trim() + '\n...' : out - }` - ) - } - return ctx -}) - -export function replaceUrlInString(str: string): string { - const { VITE_PXIMG_BASEURL_I: i, VITE_PXIMG_BASEURL_S: s } = process.env - if (i) { - str = str.replace(/https:\/\/i\.pximg\.net\//g, i.replace(/\/$/, '') + '/') - } - if (s) { - str = str.replace(/https:\/\/s\.pximg\.net\//g, s.replace(/\/$/, '') + '/') - } - return str -} - -export function replaceUrlInObject( - obj: Record | string -): Record | string { - if (typeof obj === 'string') return replaceUrlInString(obj) - - for (const key in obj) { - if ( - typeof obj[key] === 'string' && - /^https:\/\/[is]\.pximg\.net\//.test(obj[key]) - ) { - obj[key] = replaceUrlInString(obj[key]) - } else if (typeof obj[key] === 'object') { - obj[key] = replaceUrlInObject(obj[key]) - } - } - return obj -} - function isAccepted(req: VercelRequest) { const { UA_BLACKLIST = '[]' } = process.env try { diff --git a/api/image.ts b/api/image.ts index 530b44b6..8c87b104 100644 --- a/api/image.ts +++ b/api/image.ts @@ -1,5 +1,6 @@ import { VercelRequest, VercelResponse } from '@vercel/node' import axios from 'axios' +import { USER_AGENT } from './utils.js' export default async (req: VercelRequest, res: VercelResponse) => { const { __PREFIX, __PATH } = req.query @@ -14,8 +15,7 @@ export default async (req: VercelRequest, res: VercelResponse) => { responseType: 'arraybuffer', headers: { referer: 'https://www.pixiv.net/', - 'user-agent': - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0', + 'user-agent': USER_AGENT, }, }) .then( diff --git a/api/random.ts b/api/random.ts index 170bdcd9..51bdeed7 100644 --- a/api/random.ts +++ b/api/random.ts @@ -1,6 +1,6 @@ import { VercelRequest, VercelResponse } from '@vercel/node' import { formatInTimeZone } from 'date-fns-tz' -import { ajax } from './http.js' +import { PXIMG_BASEURL_I, ajax } from './utils.js' import { Artwork } from '../src/types/Artworks.js' type ArtworkOrAd = Artwork | { isAdContainer: boolean } @@ -30,11 +30,11 @@ export default async (req: VercelRequest, res: VercelResponse) => { 'yyyy/MM/dd/HH/mm/ss' )}/${value.id}` value.urls = { - mini: `/-/c/48x48/img-master/${middle}_p0_square1200.jpg`, - thumb: `/-/c/250x250_80_a2/img-master/${middle}_p0_square1200.jpg`, - small: `/-/c/540x540_70/img-master/${middle}_p0_master1200.jpg`, - regular: `/-/img-master/${middle}_p0_master1200.jpg`, - original: `/-/img-original/${middle}_p0.jpg`, + mini: `${PXIMG_BASEURL_I}c/48x48/img-master/${middle}_p0_square1200.jpg`, + thumb: `${PXIMG_BASEURL_I}c/250x250_80_a2/img-master/${middle}_p0_square1200.jpg`, + small: `${PXIMG_BASEURL_I}c/540x540_70/img-master/${middle}_p0_master1200.jpg`, + regular: `${PXIMG_BASEURL_I}img-master/${middle}_p0_master1200.jpg`, + original: `${PXIMG_BASEURL_I}img-original/${middle}_p0.jpg`, } }) if (requestImage) { diff --git a/api/user.ts b/api/user.ts index b9f0f141..3ab88501 100644 --- a/api/user.ts +++ b/api/user.ts @@ -1,6 +1,6 @@ import { VercelRequest, VercelResponse } from '@vercel/node' import { load } from 'cheerio' -import { ajax, replaceUrlInObject } from './http.js' +import { ajax, replacePximgUrlsInObject } from './utils.js' export default async (req: VercelRequest, res: VercelResponse) => { const token = req.cookies.PHPSESSID || req.query.token @@ -46,7 +46,7 @@ export default async (req: VercelRequest, res: VercelResponse) => { 'set-cookie', `CSRFTOKEN=${meta.token}; path=/; secure; sameSite=Lax` ) - res.send(replaceUrlInObject(meta)) + res.send(replacePximgUrlsInObject(meta)) }) .catch((err) => { return res diff --git a/api/utils.ts b/api/utils.ts new file mode 100644 index 00000000..5ac7b926 --- /dev/null +++ b/api/utils.ts @@ -0,0 +1,163 @@ +import { VercelRequest, VercelResponse } from '@vercel/node' +import axios from 'axios' +import colors from 'picocolors' + +// HTTP handler +export default async function (req: VercelRequest, res: VercelResponse) { + res.status(404).send({ + error: true, + message: 'Not Found', + body: null, + }) +} + +export const PROD = process.env.NODE_ENV === 'production' +export const DEV = !PROD +export const USER_AGENT = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0' +export const PXIMG_BASEURL_I = (() => { + const i = process.env.VITE_PXIMG_BASEURL_I + return i ? i.replace(/\/$/, '') + '/' : 'https://i.pximg.net/' +})() +export const PXIMG_BASEURL_S = (() => { + const s = process.env.VITE_PXIMG_BASEURL_S + return s ? s.replace(/\/$/, '') + '/' : 'https://s.pximg.net/' +})() + +export class CookieUtils { + static toJSON(raw: string) { + return Object.fromEntries(new URLSearchParams(raw.replace(/;\s*/g, '&'))) + } + static toString(obj: any) { + return Object.keys(obj) + .map((i) => `${i}=${obj[i]}`) + .join(';') + } +} + +export const ajax = axios.create({ + baseURL: 'https://www.pixiv.net/', + params: {}, + headers: { + 'user-agent': USER_AGENT, + }, + timeout: 9 * 1000, +}) +ajax.interceptors.request.use((ctx) => { + // 去除内部参数 + ctx.params = ctx.params || {} + delete ctx.params.__PATH + delete ctx.params.__PREFIX + + const cookies = CookieUtils.toJSON(ctx.headers.cookie || '') + const csrfToken = ctx.headers['x-csrf-token'] ?? cookies.CSRFTOKEN ?? '' + // 强制覆写部分 headers + ctx.headers = ctx.headers || {} + ctx.headers.host = 'www.pixiv.net' + ctx.headers.origin = 'https://www.pixiv.net' + ctx.headers.referer = 'https://www.pixiv.net/' + ctx.headers['user-agent'] = USER_AGENT + ctx.headers['accept-language'] ??= + 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6' + csrfToken && (ctx.headers['x-csrf-token'] = csrfToken) + + if (DEV) { + console.info( + colors.green(`[${ctx.method?.toUpperCase()}] <`), + colors.cyan(ctx.url || '') + ) + console.info({ + params: ctx.params, + data: ctx.data, + cookies, + }) + } + + return ctx +}) +ajax.interceptors.response.use((ctx) => { + typeof ctx.data === 'object' && + (ctx.data = replacePximgUrlsInObject(ctx.data?.body ?? ctx.data)) + if (DEV) { + const out: string = + typeof ctx.data === 'object' + ? JSON.stringify(ctx.data, null, 2) + : ctx.data.toString().trim() + console.info( + colors.green('[SEND] >'), + colors.cyan(ctx.request?.path?.replace('https://www.pixiv.net', '')), + `\n${colors.yellow(typeof ctx.data)} ${ + out.length >= 200 ? out.slice(0, 200).trim() + '\n...' : out + }` + ) + } + return ctx +}) + +export function replacePximgUrlsInString(str: string): string { + if (!str.includes('pximg.net')) return str + return str + .replace(/https:\/\/i\.pximg\.net\//g, PXIMG_BASEURL_I) + .replace(/https:\/\/s\.pximg\.net\//g, PXIMG_BASEURL_S) +} + +export function replacePximgUrlsInObject( + obj: Record | string +): Record | string { + if (typeof obj === 'string') return replacePximgUrlsInString(obj) + if (['arraybuffer', 'blob'].includes(obj.constructor.name.toLowerCase())) { + return obj + } + + return JSON.parse(safelyStringify(obj), (key: string, val: any) => { + if (typeof val === 'string' && val.includes('pximg.net')) { + return replacePximgUrlsInString(val) + } + return val + }) +} + +export function safelyStringify(value: any, space?: number) { + const visited = new WeakSet() + + const replacer = (key: string, val: any) => { + // 处理 BigInt + if (typeof val === 'bigint') { + return val.toString() + } + + // 处理 Set + if (val instanceof Set) { + return Array.from(val) + } + + // 处理 Map + if (val instanceof Map) { + return Array.from(val.entries()) + } + + // 处理 function + if (typeof val === 'function') { + return val.toString() + } + + // 处理自循环引用 + if (typeof val === 'object' && val !== null) { + if (visited.has(val)) { + return '' + } + visited.add(val) + } + + return val + } + + return JSON.stringify(value, replacer, space) +} + +JSON.safelyStringify = safelyStringify +declare global { + interface JSON { + safelyStringify: typeof safelyStringify + } +} From 46e5cf0bd3ee4bd693c3c4da7123c8eac72ab368 Mon Sep 17 00:00:00 2001 From: Dragon-Fish Date: Tue, 5 Mar 2024 15:59:00 +0800 Subject: [PATCH 3/6] fix: close #92 again --- src/view/index.vue | 50 +++++++++++++++++----------------------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/src/view/index.vue b/src/view/index.vue index 7152fb56..70d792af 100644 --- a/src/view/index.vue +++ b/src/view/index.vue @@ -1,7 +1,7 @@