From cb7be6c6c3be870b88805e09a6a9d6c1d6dd6639 Mon Sep 17 00:00:00 2001 From: Vitali Lovich Date: Sat, 29 Oct 2022 12:02:25 -0700 Subject: [PATCH] Add put-object and more config commands Add ability to remove configs / creds / list creds. Add ability to dump outbound request verbosely. --- config.ts | 111 ++++++++++++++++++++++++++++- main.ts | 51 ++++++++++--- s3.ts | 210 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 348 insertions(+), 24 deletions(-) diff --git a/config.ts b/config.ts index 3ca413e..ab64fea 100644 --- a/config.ts +++ b/config.ts @@ -127,7 +127,8 @@ class CandidatePaths { try { await new Promise((resolve, reject) => fs.mkdir(parentDir, (err) => err ? reject(err) : resolve())) } catch (e) { - if (Object.prototype.hasOwnProperty.call(e, 'code') && (e as NodeJS.ErrnoException).code !== 'EEXIST') { + if (Object.prototype.hasOwnProperty.call(e, 'code') && (e as NodeJS.ErrnoException).code === 'EEXIST') { + } else { console.warn('Trouble creating path', parentDir, e) continue } @@ -263,6 +264,85 @@ async function retrieveCreds(config: { account_id: string; access_key_id: string return Ok(secret_access_key) } +async function removeCred(config: { account_id: string; access_key_id: string }): Promise> { + const endpoint = `https://${config.account_id}.r2.cloudflarestorage.com` + + console.log( + `Removing R2 token secret with id ${config.access_key_id} for ${endpoint} from your OS encrypted password storage.`, + ) + + const keytar = await loadKeytar() + if (keytar.err) { + return keytar + } + + if (await keytar.val.deletePassword(endpoint, config.access_key_id)) { + return Ok.EMPTY + } + + return Err(new Error('Unknown problem removing token secret')) +} + +async function listCreds(account: string): Promise> { + const keytar = await loadKeytar() + if (keytar.err) { + return keytar + } + + const endpoint = `https://${account}.r2.cloudflarestorage.com` + + const creds = await keytar.val.findCredentials(endpoint) + return Ok(creds.map(({ account }) => account)) +} + +export async function listCredsCommand(argv: ArgumentsCamelCase<{ account: string }>): Promise { + const keytar = await loadKeytar() + if (keytar.err) { + process.exitCode = 1 + return + } + + const endpoint = `https://${argv.account}.r2.cloudflarestorage.com` + + for (const cred of await keytar.val.findCredentials(endpoint)) { + console.info(`Found token id ${cred.account}`) + } +} + +export async function removeCredCommand( + argv: ArgumentsCamelCase<{ account: string; 'access-key-id'?: string }>, +): Promise { + let access_key_id = argv['access-key-id'] + if (access_key_id === undefined) { + const choices = await listCreds(argv.account) + if (choices.err) { + process.exitCode = 1 + return + } + + if (choices.val.length === 0) { + console.info('No credentials found') + return + } + + const prompt = inquirer.createPromptModule() + const answer = await prompt({ + name: 'id', + message: `Which access key would you like to remove for account ${argv.account}`, + type: 'list', + choices: choices.val, + }) + + access_key_id = answer.id as string + } + + const result = await removeCred({ account_id: argv.account, access_key_id }) + if (result.err) { + process.exitCode = 1 + return + } +} + export async function importConfig(argv: ArgumentsCamelCase): Promise { const r2ConfigPaths = new CandidatePaths('cloudflare', 'r2.toml') const configFilePath = (await r2ConfigPaths.createInitialConfig()).unwrap() @@ -334,7 +414,7 @@ export async function importConfig(argv: ArgumentsCamelCase): Promise { console.info(`Imported ${numConfigurationsImported} ${importSource} configurations into ${configFilePath}`) } -export async function initConfig(argv: ArgumentsCamelCase): Promise { +export async function initConfigCommand(argv: ArgumentsCamelCase): Promise { // TODO: It would be nice to just navigate you through available accounts like wrangler does. // TODO: Use wrangler creds from ~/.wrangler/config/default.toml to communicate with the API. const name = argv['name'] as string @@ -361,7 +441,7 @@ export async function initConfig(argv: ArgumentsCamelCase): Promise { console.info(`Added configuration ${name} to ${configFilePath}`) } -export async function listConfigs(): Promise { +export async function listConfigsCommand(): Promise { const r2ConfigPaths = new CandidatePaths('cloudflare', 'r2.toml') const configFilePath = (await r2ConfigPaths.createInitialConfig()).unwrap() console.log(await readTextFile(configFilePath)) @@ -411,6 +491,31 @@ export async function retrieveOnlyConfig(): Promise> { }) } +export async function removeConfigCommand(argv: ArgumentsCamelCase<{ name: string }>): Promise { + const config = await retrieveConfig(argv.name) + if (config.err) { + process.exitCode = 1 + return + } + + const r2ConfigPaths = new CandidatePaths('cloudflare', 'r2.toml') + const configFilePath = (await r2ConfigPaths.createInitialConfig()).unwrap() + const existingConfig = TOML.parse(await readTextFile(configFilePath)) as Record< + string, + { account: string; access_key_id: string } + > + delete existingConfig[config.val.profile] + + const removal = await removeCred(config.val) + if (removal.err) { + process.exitCode = 1 + console.error(`Failed to remove creds for token ${config.val.access_key_id}`) + return + } + + await writeTextFile(configFilePath, TOML.stringify(existingConfig)) +} + export async function retrieveConfig(accountOrProfile: string): Promise> { const r2ConfigPaths = new CandidatePaths('cloudflare', 'r2.toml') const configFilePath = (await r2ConfigPaths.createInitialConfig()).unwrap() diff --git a/main.ts b/main.ts index 0c2f935..85c8a38 100644 --- a/main.ts +++ b/main.ts @@ -3,7 +3,7 @@ import cliProgress from 'cli-progress' import process from 'node:process' import { ArgumentsCamelCase } from 'yargs' import yargs from 'yargs/yargs' -import { importConfig, initConfig, listConfigs } from './config' +import { importConfig, initConfigCommand, listConfigsCommand, listCredsCommand as listCredsCommand, removeConfigCommand, removeCredCommand } from './config' import { buildS3Commands, GenericCmdArgs, handleS3Cmd } from './s3' interface ProgressBarOptions { @@ -20,18 +20,50 @@ const argv = .command('import', 'Import your configuration from another tool', (yargs) => { yargs.option('r', { alias: 'rclone' }) }, importConfig) - .command(['add', 'init'], 'Add an R2 account profile', (yargs) => { + .command(['add ', 'init'], 'Add an R2 account profile', (yargs) => { yargs - .option('name', { describe: 'The name of the profile', requiresArg: true, type: 'string' }) - .option('account', { - alias: 'a', - describe: 'The Cloudflare account ID with an R2 subscription', + .positional('name', { + describe: 'The name of the profile', requiresArg: true, type: 'string', + demandOption: true, + }) + .positional('account', { + describe: 'The Cloudflare account ID with an R2 subscription', + type: 'string', + demandOption: true, }) .demandOption(['name', 'account']) - }, initConfig) - .command(['list', 'ls'], 'List R2 accounts that are configured', () => {}, listConfigs) + }, initConfigCommand) + .command('rm ', 'Remove by profile name or account', (yargs) => { + yargs.positional('name', { + type: 'string', + description: + 'The name of the profile or the account id. If multiple profiles match the account id you will be prompted which one to remove.', + demandOption: true, + }) + }, removeConfigCommand) + .command(['list', 'ls'], 'List R2 accounts that are configured', () => {}, listConfigsCommand) + .command('list-creds ', 'List all R2 credentials saved', (yargs) => { + yargs.positional('account', { + type: 'string', + description: 'The Cloudflare account ID to list saved R2 tokens for', + demandOption: true, + }) + }, listCredsCommand) + .command('rm-cred [access-key-id]', 'List all R2 credentials saved', (yargs) => { + yargs + .positional('account', { + type: 'string', + description: 'The Cloudflare account ID to list saved R2 tokens for', + demandOption: true, + }) + .positional('access-key-id', { + type: 'string', + description: + 'The token ID to remove. If not specified you will be prompted to confirm which one to remove.', + }) + }, removeCredCommand) .demandCommand(1, 1) .help('h') .alias('h', 'help') @@ -48,7 +80,8 @@ const argv = } | {percentage}% | {value}/{total} | {eta_formatted} | {speed}`, }, cliProgress.Presets.shades_classic) return bar - }, moreHeaders), yargs) + }, moreHeaders) + .then(() => process.exit()), yargs) }) .demandCommand(1, 1) .strict() diff --git a/s3.ts b/s3.ts index d1d35f5..20f9aa5 100644 --- a/s3.ts +++ b/s3.ts @@ -2,14 +2,15 @@ import * as S3 from '@aws-sdk/client-s3' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { Command as AWSCommand } from '@aws-sdk/smithy-client' import * as AWSTypes from '@aws-sdk/types' -import { SingleBar } from 'cli-progress' +import colors from 'ansi-colors' import { sizeFormatter } from 'human-readable' -import { createWriteStream } from 'node:fs' +import inquirer from 'inquirer' +import { accessSync, constants, createReadStream, createWriteStream, statSync } from 'node:fs' import { IncomingMessage } from 'node:http' import { Transform } from 'node:stream' import { pipeline } from 'node:stream/promises' import { inspect } from 'node:util' -import { ArgumentsCamelCase, Argv, string } from 'yargs' +import { ArgumentsCamelCase, Argv } from 'yargs' import { retrieveConfig, retrieveOnlyConfig } from './config' import { ProgressBarCreator } from './main' @@ -45,6 +46,11 @@ function addSaveOption(yargs: Argv): Argv description: 'Save the body of the response to this file (use `-` to request stdout).', }) } + +function addVerboseOption(yargs: Argv): Argv { + return yargs.option('verbose', { alias: 'v', nargs: 0, boolean: true, description: 'Log the outbound request' }) +} + export function buildS3Commands( commandHandler: >( args: ArgumentsCamelCase, @@ -53,7 +59,7 @@ export function buildS3Commands( ) => Promise | void, yargs: Argv, ): Argv { - return addAccountArg(addPresignArg(yargs)) + return addVerboseOption(addAccountArg(addPresignArg(yargs))) .command('list-buckets', 'List the buckets currently created on your account.', (yargs) => addHelp(yargs) .option('prefix', { @@ -275,6 +281,11 @@ export function buildS3Commands( .group('head-object', 'Object') .command('get-object ', 'Retrieve the object from the R2 bucket.', (yargs) => addSaveOption(addObjectArg(addBucketArg(addHelp(yargs)))) + .option('range', { + nargs: 1, + string: true, + description: 'The range of the body to retrieve. Specified in HTTP Range syntax.', + }) .option('is-etag', { nargs: 1, string: true, @@ -292,11 +303,6 @@ export function buildS3Commands( description: 'Only returns a successful response if the specified object was uploaded before this date (If-Unmodified-Since header).', }) - .option('range', { - nargs: 1, - string: true, - description: 'The range of the body to retrieve. Specified in HTTP Range syntax.', - }) .option('uploaded-after', { nargs: 1, string: true, @@ -352,6 +358,116 @@ export function buildS3Commands( }), )) .group('get-object', 'Object') + .command('put-object [file|string]', 'Upload an object to the R2 bucket.', (yargs) => + addSaveOption(addObjectArg(addBucketArg(addHelp(yargs)))) + .positional('file', { + type: 'string', + description: + 'Unless --simple is provided, this is the filename. If not specified, then the object name is interpreted as the name of the file', + }) + .option('simple', { + boolean: true, + nargs: 0, + desccription: 'Treat the input argument as a string instead of a file name.', + }) + .option('is-etag', { + nargs: 1, + string: true, + description: 'R2 only attempts the upload if the provided ETag matches (If-Match header)', + }) + .option('not-etag', { + nargs: 1, + string: true, + description: 'R2 only attempts the upload if the provided ETag does not matches (If-None-Match header)', + }) + .option('uploaded-before', { + nargs: 1, + string: true, + description: + 'R2 only attempts the upload if the specified object was uploaded before this date (If-Unmodified-Since header).', + }) + .option('uploaded-after', { + nargs: 1, + string: true, + description: + 'Only returns a successful response if the specified object was uploaded after this date (If-Modified-Since header).', + }) + .option('cache-control', { + nargs: 1, + string: true, + description: + 'Set the `cache-control` metadata header for this object which is rendered when the file is retrieved (unless overridden).', + }) + .option('content-disposition', { + nargs: 1, + string: true, + description: + 'Set the `content-disposition` metadata header for this object which is rendered when the file is retrieved (unless overridden).', + }) + .option('content-encoding', { + nargs: 1, + string: true, + description: + 'Set the `content-encoding` metadata header for this object which is rendered when the file is retrieved (unless overridden).', + }) + .option('content-language', { + nargs: 1, + string: true, + description: + 'Set the `content-language` metadata header for this object which is rendered when the file is retrieved (unless overridden).', + }) + .option('content-type', { + nargs: 1, + string: true, + description: + 'Set the `content-type` metadata header for this object which is rendered when the file is retrieved (unless overridden).', + }) + .option('expires', { + nargs: 1, + string: true, + description: + 'Set the `expires` metadata header for this object which is rendered when the file is retrieved (unless overridden).', + }), (argv) => + commandHandler( + argv, + new S3.PutObjectCommand({ + Bucket: argv['bucket'], + Key: argv['object'], + Body: (() => { + if (argv['simple']) { + if (argv.file === undefined) { + throw new Error('--simple provided but no value provided') + } + return argv + .file + } + + // Can't interpret `-` because the file size is required. + return createReadStream(argv.file ?? argv.object, 'binary') + })(), + ContentLength: (() => { + if (argv['simple']) { return undefined } + return statSync( + argv.file ?? argv + .object, + ) + .size + })(), + CacheControl: argv['cache-control'], + ContentDisposition: argv['content-disposition'], + ContentEncoding: argv['content-encoding'], + ContentLanguage: argv['content-language'], + ContentType: argv['content-type'], + Expires: argv['expires'] ? new Date(argv['expires']) : undefined, + }), + { + ...(argv['is-etag'] && { 'IfMatch': argv['is-etag'] }), + ...(argv['not-etag'] && { 'IfNoneMatch': argv['not-etag'] }), + ...(argv['uploaded-after'] && { 'IfModifiedSince': argv['uploaded-before'] }), + ...(argv['uploaded-before'] && { 'IfUnmodifiedSince': argv['uploaded-before'] }), + }, + )) + .group('get-object', 'Object') .strict() .help('h') .alias('h', 'help') @@ -376,7 +492,10 @@ function addHeaders>( region: 'auto', endpoint: `https://${config.val.account_id}.r2.cloudflarestorage.com`, credentials: { accessKeyId: config.val.access_key_id, secretAccessKey: config.val.secret_access_key }, + forcePathStyle: true, }) + if (argv.verbose) { + client.middlewareStack.add((next, context) => async (args) => { + const request = args.request as { + method: string + protocol: string + hostname: string + query?: Record + headers?: Record + path: string + } + + const url = new URL(`${request.protocol}//${request.hostname}${request.path}`) + for (const [k, v] of Object.entries(request.query ?? {})) { + url.searchParams.set(k, v) + } + + const curlHeaderArgs = Object.entries(request.headers ?? {}).flatMap(([k, v]) => ['-H', `'${k}: ${v}'`]) + + let serializedHeaderArgs = curlHeaderArgs.length === 0 ? '' : ' ' + colors.blue(curlHeaderArgs.join(' ')) + + let serializedCommandName = colors.green(context['commandName']) + let serializedMethod = colors.red(`-X ${request.method}`) + let serializedUrl = colors.grey(url.toString()) + + console.debug(`${serializedCommandName}: curl ${serializedMethod}${serializedHeaderArgs} '${serializedUrl}'`) + + return await next(args) + }, { name: 'logRequest', step: 'finalizeRequest', priority: 'low' }) + } + if (argv.presign) { const now = new Date() const expiryInXSeconds = argv['expires-in']! @@ -472,7 +622,10 @@ export async function handleS3Cmd>( console.info() console.info(`URL expires ${expiryDate.toUTCString()}`) console.info() - console.info(curlArgs.join(' ')) + process.stdout.write((curlArgs.join(' '))) + if (process.stdout.isTTY) { + process.stdout.write('\n') + } return } @@ -496,10 +649,43 @@ export async function handleS3Cmd>( if (body !== undefined) { if (argv['save-body-to'] === undefined) { - console.warn('The response from the request is not saved - please specify --save/-o') - return + if (argv['object'] !== undefined) { + // Infer the save location + let inferredSavePath = argv['object'].split('/').at(-1)! + argv['save-body-to'] = inferredSavePath + } else { + console.warn('The response from the request is not saved - please specify --save/-o') + return + } } + if (argv['save-body-to'] !== '-') { + try { + accessSync(argv['save-body-to'], constants.W_OK) + + const prompt = inquirer.createPromptModule() + const answer = await prompt({ + name: 'overwrite', + message: `${argv['save-body-to']} already exists and --save-body-to/-o was not provided. Overwrite?`, + type: 'confirm', + default: false, + }) + if (!answer.overwrite) { + return + } + } catch (e) { + if (Object.prototype.hasOwnProperty.call(e, 'code') && (e as NodeJS.ErrnoException).code === 'ENOENT') { + // Creating a new file is fine. + } else { + console.error(`Unrecognized error trying to save to inferred location from object name.`, e) + process.exitCode = 1 + return + } + } + } + + // TODO(later): If dumping to stdout and stdout is connected to a TTY, prompt confirmation when file is > some threshold. + let output = argv['save-body-to'] !== '-' ? createWriteStream(argv['save-body-to'], { encoding: 'binary' }) : process.stdout