From 5d3768f65e488cc49c34a303231bf08da80092ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Chopin?= Date: Wed, 22 Jan 2025 19:01:38 +0100 Subject: [PATCH 1/2] feat: add support for worker type --- src/utils/data.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utils/data.mjs b/src/utils/data.mjs index 04a5c5e..04de834 100644 --- a/src/utils/data.mjs +++ b/src/utils/data.mjs @@ -131,6 +131,15 @@ export async function selectProject(team) { }) if (isCancel(projectName)) return null projectName = projectName || defaultProjectName + const projectType = await select({ + message: 'Select your project type', + initialValue: 'pages', + options: [ + { label: 'Cloudflare Pages', value: 'pages' }, + { label: 'Cloudflare Workers (beta)', value: 'worker' }, + ] + }) + if (isCancel(projectType)) return null const projectLocation = await select({ message: 'Select a region for the storage', initialValue: 'weur', @@ -159,6 +168,7 @@ export async function selectProject(team) { method: 'POST', body: { name: projectName, + type: projectType, location: projectLocation, productionBranch: productionBranch || defaultProductionBranch } From fd3d0b678589fdb689cf64b3aa064ea71d10278d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Chopin?= Date: Fri, 24 Jan 2025 12:38:26 +0100 Subject: [PATCH 2/2] add support for workers paths --- src/commands/deploy.mjs | 82 ++++++++++++++++++++++++++++------------- src/utils/deploy.mjs | 76 +++++++++++++++++++++++++++++++++++++- 2 files changed, 131 insertions(+), 27 deletions(-) diff --git a/src/commands/deploy.mjs b/src/commands/deploy.mjs index 16c5faf..34644ce 100644 --- a/src/commands/deploy.mjs +++ b/src/commands/deploy.mjs @@ -8,7 +8,7 @@ import { join, resolve, relative } from 'pathe' import { execa } from 'execa' import { setupDotenv } from 'c12' import { $api, fetchUser, selectTeam, selectProject, projectPath, fetchProject, linkProject, gitInfo, getPackageJson } from '../utils/index.mjs' -import { getStorage, getPathsToDeploy, getFile, uploadAssetsToCloudflare, isMetaPath, isServerPath, getPublicFiles } from '../utils/deploy.mjs' +import { getStorage, getPathsToDeploy, getFile, uploadAssetsToCloudflare, uploadWorkersAssetsToCloudflare, isMetaPath, isWorkerMetaPath, isServerPath, isWorkerServerPath, getPublicFiles, getWorkerPublicFiles } from '../utils/deploy.mjs' import { createMigrationsTable, fetchRemoteMigrations, queryDatabase } from '../utils/database.mjs' import login from './login.mjs' @@ -137,6 +137,7 @@ export default defineCommand({ const fileKeys = await storage.getKeys() const pathsToDeploy = getPathsToDeploy(fileKeys) const config = await storage.getItem('hub.config.json') + const isWorkerPreset = ['cloudflare_module', 'cloudflare_durable'].includes(config.nitroPreset) const { format: formatNumber } = new Intl.NumberFormat('en-US') let spinner = ora(`Preparing ${colors.blueBright(linkedProject.slug)} deployment for ${deployEnvColored}...`).start() @@ -147,40 +148,64 @@ export default defineCommand({ spinnerColorIndex = (spinnerColorIndex + 1) % spinnerColors.length }, 2500) - let deploymentKey, serverFiles, metaFiles + let deploymentKey, serverFiles, metaFiles, completionToken try { - const publicFiles = await getPublicFiles(storage, pathsToDeploy) + let url = `/teams/${linkedProject.teamSlug}/projects/${linkedProject.slug}/${deployEnv}/deploy/prepare` + let publicFiles, publicManifest - const deploymentInfo = await $api(`/teams/${linkedProject.teamSlug}/projects/${linkedProject.slug}/${deployEnv}/deploy/prepare`, { + if (isWorkerPreset) { + url = `/teams/${linkedProject.teamSlug}/projects/${linkedProject.slug}/${deployEnv}/deploy/worker/prepare` + publicFiles = await getWorkerPublicFiles(storage, pathsToDeploy) + /** + * { "/index.html": { hash: "hash", size: 30 } + */ + publicManifest = publicFiles.reduce((acc, file) => { + acc[file.path] = { + hash: file.hash, + size: file.size + } + return acc + }, {}) + } else { + publicFiles = await getPublicFiles(storage, pathsToDeploy) + /** + * { "/index.html": "hash" } + */ + publicManifest = publicFiles.reduce((acc, file) => { + acc[file.path] = file.hash + return acc + }, {}) + } + // Get deployment info by preparing the deployment + const deploymentInfo = await $api(url, { method: 'POST', body: { config, - /** - * Public manifest is a map of file paths to their unique hash (SHA256 sliced to 32 characters). - * @example - * { - * "/index.html": "hash", - * "/assets/image.png": "hash" - * } - */ - publicManifest: publicFiles.reduce((acc, file) => { - acc[file.path] = file.hash - return acc - }, {}) + publicManifest } }) spinner.succeed(`${colors.blueBright(linkedProject.slug)} ready to deploy.`) - const { missingPublicHashes, cloudflareUploadJwt } = deploymentInfo deploymentKey = deploymentInfo.deploymentKey + + const { cloudflareUploadJwt, buckets, accountId } = deploymentInfo + // missingPublicHash is sent for pages & buckets for worker + let missingPublicHashes = deploymentInfo.missingPublicHashes || buckets.flat() const publicFilesToUpload = publicFiles.filter(file => missingPublicHashes.includes(file.hash)) if (publicFilesToUpload.length) { const totalSizeToUpload = publicFilesToUpload.reduce((acc, file) => acc + file.size, 0) spinner = ora(`Uploading ${colors.blueBright(formatNumber(publicFilesToUpload.length))} new static assets (${colors.blueBright(prettyBytes(totalSizeToUpload))})...`).start() - await uploadAssetsToCloudflare(publicFilesToUpload, cloudflareUploadJwt, ({ progressSize, totalSize }) => { - const percentage = Math.round((progressSize / totalSize) * 100) - spinner.text = `${percentage}% uploaded (${prettyBytes(progressSize)}/${prettyBytes(totalSize)})...` - }) + if (linkedProject.type === 'pages') { + await uploadAssetsToCloudflare(publicFilesToUpload, cloudflareUploadJwt, ({ progressSize, totalSize }) => { + const percentage = Math.round((progressSize / totalSize) * 100) + spinner.text = `${percentage}% uploaded (${prettyBytes(progressSize)}/${prettyBytes(totalSize)})...` + }) + } else { + completionToken = await uploadWorkersAssetsToCloudflare(accountId, publicFilesToUpload, cloudflareUploadJwt, ({ progressSize, totalSize }) => { + const percentage = Math.round((progressSize / totalSize) * 100) + spinner.text = `${percentage}% uploaded (${prettyBytes(progressSize)}/${prettyBytes(totalSize)})...` + }) + } spinner.succeed(`${colors.blueBright(formatNumber(publicFilesToUpload.length))} new static assets uploaded (${colors.blueBright(prettyBytes(totalSizeToUpload))})`) } @@ -190,8 +215,14 @@ export default defineCommand({ consola.info(`${colors.blueBright(formatNumber(publicFiles.length))} static assets (${colors.blueBright(prettyBytes(totalSize))} / ${colors.blueBright(prettyBytes(totalGzipSize))} gzip)`) } - metaFiles = await Promise.all(pathsToDeploy.filter(isMetaPath).map(p => getFile(storage, p, 'base64'))) - serverFiles = await Promise.all(pathsToDeploy.filter(isServerPath).map(p => getFile(storage, p, 'base64'))) + metaFiles = await Promise.all(pathsToDeploy.filter(isWorkerPreset ? isWorkerMetaPath : isMetaPath).map(p => getFile(storage, p, 'base64'))) + serverFiles = await Promise.all(pathsToDeploy.filter(isWorkerPreset ? isWorkerServerPath : isServerPath).map(p => getFile(storage, p, 'base64'))) + if (isWorkerPreset) { + serverFiles = serverFiles.map(file => ({ + ...file, + path: file.path.replace('/server/', '/') + })) + } const serverFilesSize = serverFiles.reduce((acc, file) => acc + file.size, 0) const serverFilesGzipSize = serverFiles.reduce((acc, file) => acc + file.gzipSize, 0) consola.info(`${colors.blueBright(formatNumber(serverFiles.length))} server files (${colors.blueBright(prettyBytes(serverFilesSize))} / ${colors.blueBright(prettyBytes(serverFilesGzipSize))} gzip)...`) @@ -286,13 +317,14 @@ export default defineCommand({ // #region Complete deployment spinner = ora(`Deploying ${colors.blueBright(linkedProject.slug)} to ${deployEnvColored}...`).start() - const deployment = await $api(`/teams/${linkedProject.teamSlug}/projects/${linkedProject.slug}/${deployEnv}/deploy/complete`, { + const deployment = await $api(`/teams/${linkedProject.teamSlug}/projects/${linkedProject.slug}/${deployEnv}/deploy/${isWorkerPreset ? 'worker/complete' : 'complete'}`, { method: 'POST', body: { deploymentKey, git, serverFiles, - metaFiles + metaFiles, + completionToken }, }).catch((err) => { spinner.fail(`Failed to deploy ${colors.blueBright(linkedProject.slug)} to ${deployEnvColored}.`) diff --git a/src/utils/deploy.mjs b/src/utils/deploy.mjs index bc42a2f..2411251 100644 --- a/src/utils/deploy.mjs +++ b/src/utils/deploy.mjs @@ -126,12 +126,18 @@ export const META_PATHS = [ '/nitro.json', '/hub.config.json', '/wrangler.toml', + '/package-lock.json', + '/package.json' ] export const isMetaPath = (path) => META_PATHS.includes(path) export const isServerPath = (path) => path.startsWith('/_worker.js/') export const isPublicPath = (path) => !isMetaPath(path) && !isServerPath(path) +export const isWorkerMetaPath = (path) => META_PATHS.includes(path) +export const isWorkerPublicPath = (path) => path.startsWith('/public/') +export const isWorkerServerPath = (path) => path.startsWith('/server/') + /** * Get all public files with their metadata * @param {import('unstorage').Storage} storage - Storage instance @@ -143,9 +149,18 @@ export async function getPublicFiles(storage, paths) { paths.filter(isPublicPath).map(p => getFile(storage, p, 'base64')) ) } +export async function getWorkerPublicFiles(storage, paths) { + const files = await Promise.all( + paths.filter(isWorkerPublicPath).map(p => getFile(storage, p, 'base64')) + ) + return files.map((file) => ({ + ...file, + path: file.path.replace('/public/', '/') + })) +} /** - * Upload assets to Cloudflare with concurrent uploads + * Upload assets to Cloudflare Pages with concurrent uploads * @param {Array<{ path: string, data: string, hash: string, contentType: string }>} files - Files to upload * @param {string} cloudflareUploadJwt - Cloudflare upload JWT * @param {Function} onProgress - Callback function to update progress @@ -200,4 +215,61 @@ export async function uploadAssetsToCloudflare(files, cloudflareUploadJwt, onPro } } -// async function uploadToCloudflare(body, cloudflareUploadJwt) { + +/** + * Upload assets to Cloudflare Workers with concurrent uploads + * @param {Array} buckets - Buckets of hashes to upload + * @param {Array<{ path: string, data: string, hash: string, contentType: string }>} files - Files to upload + * @param {string} cloudflareUploadJwt - Cloudflare upload JWT + * @param {Function} onProgress - Callback function to update progress + */ +export async function uploadWorkersAssetsToCloudflare(accountId, files, cloudflareUploadJwt, onProgress) { + const chunks = await createChunks(files) + if (!chunks.length) { + return + } + + let filesUploaded = 0 + let progressSize = 0 + let completionToken + const totalSize = files.reduce((acc, file) => acc + file.size, 0) + for (let i = 0; i < chunks.length; i += CONCURRENT_UPLOADS) { + const chunkGroup = chunks.slice(i, i + CONCURRENT_UPLOADS) + + await Promise.all(chunkGroup.map(async (filesInChunk) => { + const form = new FormData() + for (const file of filesInChunk) { + form.append(file.hash, new File([file.data], file.hash, { type: file.contentType}), file.hash) + } + return ofetch(`/accounts/${accountId}/workers/assets/upload?base64=true`, { + baseURL: 'https://api.cloudflare.com/client/v4/', + method: 'POST', + headers: { + Authorization: `Bearer ${cloudflareUploadJwt}` + }, + retry: MAX_UPLOAD_ATTEMPTS, + retryDelay: UPLOAD_RETRY_DELAY, + body: form + }) + .then((data) => { + if (data && data.result?.jwt) { + completionToken = data.result.jwt + } + if (typeof onProgress === 'function') { + filesUploaded += filesInChunk.length + progressSize += filesInChunk.reduce((acc, file) => acc + file.size, 0) + onProgress({ progress: filesUploaded, progressSize, total: files.length, totalSize }) + } + }) + .catch((err) => { + if (err.data) { + throw new Error(`Error while uploading assets to Cloudflare: ${JSON.stringify(err.data)} - ${err.message}`) + } + else { + throw new Error(`Error while uploading assets to Cloudflare: ${err.message.split(' - ')[1] || err.message}`) + } + }) + })) + } + return completionToken +}