From 5e635b934ae3b68c97612e0c761b7964c95387d4 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 23 Sep 2024 16:21:38 -0400 Subject: [PATCH 01/68] basic api surface area --- packages/aws-cdk/lib/cdk-toolkit.ts | 16 ++++++++++++++++ packages/aws-cdk/lib/cli.ts | 13 +++++++++++++ 2 files changed, 29 insertions(+) diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index d6c6000092f6d..afbbc8f2857a7 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -763,6 +763,15 @@ export class CdkToolkit { })); } + /** + * Garbage collects assets from a CDK app's environment + * @param options Options for Garbage Collection + */ + public async garbageCollect(userEnvironmentSpecs: string[], options: GarbageCollectionOptions) { + // eslint-disable-next-line no-console + console.log(userEnvironmentSpecs, options); + } + /** * Migrates a CloudFormation stack/template to a CDK app * @param options Options for CDK app creation @@ -1407,6 +1416,13 @@ export interface DestroyOptions { readonly ci?: boolean; } +export interface GarbageCollectionOptions { + readonly dryRun?: boolean; + readonly tagOnly?: boolean; + readonly type: 'ecr' | 'days' | 'all'; + readonly days: number; +} + export interface MigrateOptions { /** * The name assigned to the generated stack. This is also used to get diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index f63c7cbf21eef..33f08a77539ac 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -271,6 +271,11 @@ async function parseCommandLineArguments(args: string[]) { .command('notices', 'Returns a list of relevant notices', (yargs: Argv) => yargs .option('unacknowledged', { type: 'boolean', alias: 'u', default: false, desc: 'Returns a list of unacknowledged notices' }), ) + .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs + .option('dry-run', { type: 'boolean', desc: 'List assets instead of garbage collecting them', default: false }) + .option('tag-only', { type: 'boolean', desc: 'Tag assets as isolated without deleting them', default: false }) + .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) + .option('days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 })) .command('init [TEMPLATE]', 'Create a new, empty CDK project from a template.', (yargs: Argv) => yargs .option('language', { type: 'string', alias: 'l', desc: 'The language to be used for the new project (default can be configured in ~/.cdk.json)', choices: initTemplateLanguages }) .option('list', { type: 'boolean', desc: 'List the available templates' }) @@ -665,6 +670,14 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Mon, 23 Sep 2024 17:59:17 -0400 Subject: [PATCH 02/68] more scaffolding with sdks --- packages/aws-cdk/lib/api/garbage-collector.ts | 126 ++++++++++++++++++ packages/aws-cdk/lib/api/index.ts | 1 + packages/aws-cdk/lib/cdk-toolkit.ts | 36 ++++- 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 packages/aws-cdk/lib/api/garbage-collector.ts diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts new file mode 100644 index 0000000000000..60d911cd563fe --- /dev/null +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -0,0 +1,126 @@ +import * as cxapi from '@aws-cdk/cx-api'; +import { ISDK, Mode, SdkProvider } from './aws-auth'; +import { ToolkitInfo } from './toolkit-info'; +import { S3 } from 'aws-sdk'; + +/** + * Props for the Garbage Collector + */ +interface GarbageCollectorProps { + /** + * If this property is set, then instead of garbage collecting, we will + * print the isolated asset hashes. + * + * @default false + */ + readonly dryRun: boolean; + + /** + * If this property is set, then we will tag assets as isolated but skip + * the actual asset deletion process. + * + * @default false + */ + readonly tagOnly: boolean; + + /** + * The type of asset to garbage collect. + * + * @default 'all' + */ + readonly type: 's3' | 'ecr' | 'all'; + + /** + * The days an asset must be in isolation before being actually deleted. + * + * @default 0 + */ + readonly isolationDays: number; + + /** + * The environment to deploy this stack in + * + * The environment on the stack artifact may be unresolved, this one + * must be resolved. + */ + readonly resolvedEnvironment: cxapi.Environment; + + /** + * SDK provider (seeded with default credentials) + * + * Will be used to make SDK calls to CloudFormation, S3, and ECR. + */ + readonly sdkProvider: SdkProvider; +} + +export class GarbageCollector { + private garbageCollectS3Assets: boolean; + private garbageCollectEcrAssets: boolean; + private resolvedEnvironment: cxapi.Environment; + private sdkProvider: SdkProvider; + + public constructor(props: GarbageCollectorProps) { + this.garbageCollectS3Assets = props.type === 's3' || props.type === 'all'; + this.garbageCollectEcrAssets = props.type === 'ecr' || props.type === 'all'; + this.resolvedEnvironment = props.resolvedEnvironment; + this.sdkProvider = props.sdkProvider; + + if (this.garbageCollectEcrAssets) { + throw new Error('ECR garbage collection is not yet supported'); + } + } + + public async garbageCollect() { + // set up the sdks used in garbage collection + const sdk = (await this.sdkProvider.forEnvironment(this.resolvedEnvironment, Mode.ForWriting)).sdk; + const { cfn: _cfn, s3 } = this.setUpSDKs(sdk); + + if (this.garbageCollectS3Assets) { + const bucket = await this.getBootstrapBucket(sdk); + console.log(bucket); + const objects = await this.collectObjects(s3, bucket); + console.log(objects); + } + } + + private setUpSDKs(sdk: ISDK) { + const cfn = sdk.cloudFormation(); + const s3 = sdk.s3(); + return { + cfn, + s3, + }; + } + + private async getBootstrapBucket(sdk: ISDK) { + const info = await ToolkitInfo.lookup(this.resolvedEnvironment, sdk, undefined); + return info.bucketName; + } + + private async collectObjects(s3: S3, bucket: string) { + const objects: string[] = []; + await paginateSdkCall(async (nextToken) => { + const response = await s3.listObjectsV2({ + Bucket: bucket, + ContinuationToken: nextToken, + }).promise(); + response.Contents?.forEach((obj) => { + objects.push(obj.Key ?? ''); + }); + return response.NextContinuationToken; + }); + + return objects; + } +} + +async function paginateSdkCall(cb: (nextToken?: string) => Promise) { + let finished = false; + let nextToken: string | undefined; + while (!finished) { + nextToken = await cb(nextToken); + if (nextToken === undefined) { + finished = true; + } + } +} diff --git a/packages/aws-cdk/lib/api/index.ts b/packages/aws-cdk/lib/api/index.ts index 5671f05837205..2bd0a29378d26 100644 --- a/packages/aws-cdk/lib/api/index.ts +++ b/packages/aws-cdk/lib/api/index.ts @@ -1,5 +1,6 @@ export * from './aws-auth/credentials'; export * from './bootstrap'; +export * from './garbage-collector'; export * from './deploy-stack'; export * from './toolkit-info'; export * from './aws-auth'; diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index afbbc8f2857a7..74ce85ca15ab1 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -9,6 +9,7 @@ import * as uuid from 'uuid'; import { DeploymentMethod } from './api'; import { SdkProvider } from './api/aws-auth'; import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap'; +import { GarbageCollector } from './api/garbage-collector'; import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly'; import { CloudExecutable } from './api/cxapp/cloud-executable'; import { Deployments } from './api/deployments'; @@ -770,6 +771,39 @@ export class CdkToolkit { public async garbageCollect(userEnvironmentSpecs: string[], options: GarbageCollectionOptions) { // eslint-disable-next-line no-console console.log(userEnvironmentSpecs, options); + + // TODO: copied from bootstrap, make into function + // By default, glob for everything + const environmentSpecs = userEnvironmentSpecs.length > 0 ? [...userEnvironmentSpecs] : ['**']; + + // Partition into globs and non-globs (this will mutate environmentSpecs). + const globSpecs = partition(environmentSpecs, looksLikeGlob); + if (globSpecs.length > 0 && !this.props.cloudExecutable.hasApp) { + if (userEnvironmentSpecs.length > 0) { + // User did request this glob + throw new Error(`'${globSpecs}' is not an environment name. Specify an environment name like 'aws://123456789012/us-east-1', or run in a directory with 'cdk.json' to use wildcards.`); + } else { + // User did not request anything + throw new Error('Specify an environment name like \'aws://123456789012/us-east-1\', or run in a directory with \'cdk.json\'.'); + } + } + + const environments: cxapi.Environment[] = [ + ...environmentsFromDescriptors(environmentSpecs), + ]; + + await Promise.all(environments.map(async (environment) => { + success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); + const gc = new GarbageCollector({ + sdkProvider: this.props.sdkProvider, + resolvedEnvironment: environment, + isolationDays: options.days, + dryRun: options.dryRun ?? false, + tagOnly: options.tagOnly ?? false, + type: options.type ?? 'all', + }); + await gc.garbageCollect(); + })); } /** @@ -1419,7 +1453,7 @@ export interface DestroyOptions { export interface GarbageCollectionOptions { readonly dryRun?: boolean; readonly tagOnly?: boolean; - readonly type: 'ecr' | 'days' | 'all'; + readonly type: 'ecr' | 's3' | 'all'; readonly days: number; } From b12362a0d373999ff113260d8fc15f0290df359d Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 24 Sep 2024 17:39:24 -0400 Subject: [PATCH 03/68] add cfn api calls --- packages/aws-cdk/lib/api/garbage-collector.ts | 62 ++++++++++++++++--- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 60d911cd563fe..c1555e55f7c52 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -1,7 +1,8 @@ import * as cxapi from '@aws-cdk/cx-api'; import { ISDK, Mode, SdkProvider } from './aws-auth'; import { ToolkitInfo } from './toolkit-info'; -import { S3 } from 'aws-sdk'; +import { CloudFormation, S3 } from 'aws-sdk'; +import path = require('path'); /** * Props for the Garbage Collector @@ -53,7 +54,13 @@ interface GarbageCollectorProps { readonly sdkProvider: SdkProvider; } +/** + * A class to facilitate Garbage Collection of S3 and ECR assets + */ export class GarbageCollector { + private readonly inUseHashes: Set = new Set(); + private readonly objectHashes: Set = new Set(); + private garbageCollectS3Assets: boolean; private garbageCollectEcrAssets: boolean; private resolvedEnvironment: cxapi.Environment; @@ -65,22 +72,34 @@ export class GarbageCollector { this.resolvedEnvironment = props.resolvedEnvironment; this.sdkProvider = props.sdkProvider; + // TODO: ECR garbage collection if (this.garbageCollectEcrAssets) { throw new Error('ECR garbage collection is not yet supported'); } } + /** + * Perform garbage collection on the resolved environment(s) + */ public async garbageCollect() { // set up the sdks used in garbage collection const sdk = (await this.sdkProvider.forEnvironment(this.resolvedEnvironment, Mode.ForWriting)).sdk; - const { cfn: _cfn, s3 } = this.setUpSDKs(sdk); + const { cfn, s3 } = this.setUpSDKs(sdk); if (this.garbageCollectS3Assets) { const bucket = await this.getBootstrapBucket(sdk); console.log(bucket); - const objects = await this.collectObjects(s3, bucket); - console.log(objects); + await this.collectObjects(s3, bucket); + console.log(this.objectHashes); } + + await this.collectHashes(cfn); + console.log(this.inUseHashes); + + console.log(setDifference(this.objectHashes, this.inUseHashes)); + + // TODO: match asset hashes with object keys + // TODO: tag isolated assets } private setUpSDKs(sdk: ISDK) { @@ -92,25 +111,44 @@ export class GarbageCollector { }; } + private async collectHashes(cfn: CloudFormation) { + const stackNames: string[] = []; + await paginateSdkCall(async (nextToken) => { + const response = await cfn.listStacks({ NextToken: nextToken }).promise(); + stackNames.push(...(response.StackSummaries ?? []).map(s => s.StackId ?? s.StackName)); + return response.NextToken; + }); + + console.log(`Parsing through ${stackNames.length} stacks`); + + for (const stack of stackNames) { + const template = await cfn.getTemplate({ + StackName: stack, + }).promise(); + + const templateHashes = template.TemplateBody?.match(/[a-f0-9]{64}/g); + templateHashes?.forEach(this.inUseHashes.add, this.inUseHashes); + } + + console.log(`Found ${this.inUseHashes.size} unique hashes`); + } + private async getBootstrapBucket(sdk: ISDK) { const info = await ToolkitInfo.lookup(this.resolvedEnvironment, sdk, undefined); return info.bucketName; } private async collectObjects(s3: S3, bucket: string) { - const objects: string[] = []; await paginateSdkCall(async (nextToken) => { const response = await s3.listObjectsV2({ Bucket: bucket, ContinuationToken: nextToken, }).promise(); response.Contents?.forEach((obj) => { - objects.push(obj.Key ?? ''); + this.objectHashes.add(path.parse(obj.Key ?? '').name); }); return response.NextContinuationToken; }); - - return objects; } } @@ -124,3 +162,11 @@ async function paginateSdkCall(cb: (nextToken?: string) => Promise(setA: Set, setB: Set): Set { + let difference = new Set(setA); + for (let elem of setB) { + difference.delete(elem); + } + return difference; +} From 796e8d24402fea45b726ca068d2522f4a75a76ee Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 26 Sep 2024 16:15:50 -0400 Subject: [PATCH 04/68] tagging api in use now --- packages/aws-cdk/lib/api/garbage-collector.ts | 172 +++++++++++++++--- 1 file changed, 144 insertions(+), 28 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index c1555e55f7c52..7330a317adbf3 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -1,8 +1,12 @@ import * as cxapi from '@aws-cdk/cx-api'; import { ISDK, Mode, SdkProvider } from './aws-auth'; import { ToolkitInfo } from './toolkit-info'; +import { print } from '../logging'; import { CloudFormation, S3 } from 'aws-sdk'; -import path = require('path'); +import * as chalk from 'chalk'; +import * as path from 'path'; + +const ISOLATED_TAG = 'awscdk.isolated'; /** * Props for the Garbage Collector @@ -58,19 +62,16 @@ interface GarbageCollectorProps { * A class to facilitate Garbage Collection of S3 and ECR assets */ export class GarbageCollector { - private readonly inUseHashes: Set = new Set(); - private readonly objectHashes: Set = new Set(); + private readonly activeHashes = new Set(); + private readonly objects = new Map(); + private bootstrapBucketName: string | undefined = undefined; private garbageCollectS3Assets: boolean; private garbageCollectEcrAssets: boolean; - private resolvedEnvironment: cxapi.Environment; - private sdkProvider: SdkProvider; - public constructor(props: GarbageCollectorProps) { + public constructor(readonly props: GarbageCollectorProps) { this.garbageCollectS3Assets = props.type === 's3' || props.type === 'all'; this.garbageCollectEcrAssets = props.type === 'ecr' || props.type === 'all'; - this.resolvedEnvironment = props.resolvedEnvironment; - this.sdkProvider = props.sdkProvider; // TODO: ECR garbage collection if (this.garbageCollectEcrAssets) { @@ -82,24 +83,40 @@ export class GarbageCollector { * Perform garbage collection on the resolved environment(s) */ public async garbageCollect() { + print(chalk.green(`Garbage collecting for ${this.props.resolvedEnvironment.account} in ${this.props.resolvedEnvironment.region}`)); + // set up the sdks used in garbage collection - const sdk = (await this.sdkProvider.forEnvironment(this.resolvedEnvironment, Mode.ForWriting)).sdk; + const sdk = (await this.props.sdkProvider.forEnvironment(this.props.resolvedEnvironment, Mode.ForWriting)).sdk; const { cfn, s3 } = this.setUpSDKs(sdk); if (this.garbageCollectS3Assets) { + print(chalk.green('Getting bootstrap bucket')); const bucket = await this.getBootstrapBucket(sdk); - console.log(bucket); + print(chalk.green(`Bootstrap Bucket ${bucket}`)); await this.collectObjects(s3, bucket); - console.log(this.objectHashes); + print(chalk.blue(`Object hashes: ${this.objects.size}`)); } await this.collectHashes(cfn); - console.log(this.inUseHashes); + print(chalk.green(`Hashes in use: ${this.activeHashes.size}`)); + + for (const hash of this.activeHashes) { + const obj = this.objects.get(hash); + if (obj) { + obj.isolated = false; + } + } + const isolatedObjects = Array.from(this.objects.values()) + .filter(obj => obj.isolated === true); - console.log(setDifference(this.objectHashes, this.inUseHashes)); + print(chalk.red(`Isolated hashes: ${isolatedObjects.length}`)); - // TODO: match asset hashes with object keys - // TODO: tag isolated assets + if (!this.props.dryRun) { + if (this.garbageCollectS3Assets) { + print(chalk.green('Tagging Isolated Objects')); + this.tagObjects(s3, await this.getBootstrapBucket(sdk)); + } + } } private setUpSDKs(sdk: ISDK) { @@ -119,7 +136,7 @@ export class GarbageCollector { return response.NextToken; }); - console.log(`Parsing through ${stackNames.length} stacks`); + print(chalk.blue(`Parsing through ${stackNames.length} stacks`)); for (const stack of stackNames) { const template = await cfn.getTemplate({ @@ -127,15 +144,18 @@ export class GarbageCollector { }).promise(); const templateHashes = template.TemplateBody?.match(/[a-f0-9]{64}/g); - templateHashes?.forEach(this.inUseHashes.add, this.inUseHashes); + templateHashes?.forEach(this.activeHashes.add, this.activeHashes); } - console.log(`Found ${this.inUseHashes.size} unique hashes`); + print(chalk.blue(`Found ${this.activeHashes.size} unique hashes`)); } - private async getBootstrapBucket(sdk: ISDK) { - const info = await ToolkitInfo.lookup(this.resolvedEnvironment, sdk, undefined); - return info.bucketName; + private async getBootstrapBucket(sdk: ISDK): Promise { + if (!this.bootstrapBucketName) { + const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, undefined); + this.bootstrapBucketName = info.bucketName; + } + return this.bootstrapBucketName; } private async collectObjects(s3: S3, bucket: string) { @@ -145,11 +165,109 @@ export class GarbageCollector { ContinuationToken: nextToken, }).promise(); response.Contents?.forEach((obj) => { - this.objectHashes.add(path.parse(obj.Key ?? '').name); + let key = obj.Key; + if (key) { + let hash = path.parse(key).name; + this.objects.set(hash, { + key, + hash, + isolated: true, // Default is isolated + }); + } }); return response.NextContinuationToken; }); } + + private async tagObjects(s3: S3, bucket: string) { + let newlyTagged = 0; + let alreadyTagged = 0; + let newlyUntagged = 0; + let alreadyUntagged = 0; + for (const [_, obj] of this.objects) { + if (obj.isolated) { + const result = await this.tagIsolatedObject(s3, bucket, obj); + if (result) { + newlyTagged++; + } else { + alreadyTagged++; + } + } else { + const result = await this.untagActiveObject(s3, bucket, obj); + if (result) { + newlyUntagged++; + } else { + alreadyUntagged++; + } + } + } + + print(chalk.white(`Newly Tagged: ${newlyTagged}\nAlready Tagged: ${alreadyTagged}\nNewly Untagged: ${newlyUntagged}\nAlready Untagged: ${alreadyUntagged}`)); + } + + /** + * Returns true if object gets tagged (was previously untagged but is now isolated) + */ + private async tagIsolatedObject(s3: S3, bucket: string, object: S3Object) { + const response = await s3.getObjectTagging({ + Bucket: bucket, + Key: object.key, + }).promise(); + const isolatedTag = await this.getIsolatedTag(response.TagSet); + + // tag new objects with the current date + if (!isolatedTag) { + await s3.putObjectTagging({ + Bucket: bucket, + Key: object.key, + Tagging: { + TagSet: [{ + Key: ISOLATED_TAG, + Value: Date.now().toString(), + }], + }, + }).promise(); + + return true; + } + return false; + } + + /** + * Returns true if object gets untagged (was previously tagged but is now active) + */ + private async untagActiveObject(s3: S3, bucket: string, object: S3Object) { + const response = await s3.getObjectTagging({ + Bucket: bucket, + Key: object.key, + }).promise(); + const isolatedTag = await this.getIsolatedTag(response.TagSet); + + // remove isolated tag and put any other tags back + if (isolatedTag) { + const newTags = response.TagSet?.filter(tag => tag.Key !== ISOLATED_TAG); + // TODO: double check what happens when newTags = [] + await s3.putObjectTagging({ + Bucket: bucket, + Key: object.key, + Tagging: { + TagSet: newTags, + }, + }).promise(); + + return true; + } + return false; + } + + private async getIsolatedTag(tags: S3.TagSet): Promise { + for (const tag of tags) { + if (tag.Key === ISOLATED_TAG) { + return tag.Value; + } + } + return; + } } async function paginateSdkCall(cb: (nextToken?: string) => Promise) { @@ -163,10 +281,8 @@ async function paginateSdkCall(cb: (nextToken?: string) => Promise(setA: Set, setB: Set): Set { - let difference = new Set(setA); - for (let elem of setB) { - difference.delete(elem); - } - return difference; +interface S3Object { + readonly key: string; + readonly hash: string; + isolated: boolean; } From 52b6c102831dfc1cec52e5d044ce3a7d478c4a07 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Fri, 27 Sep 2024 17:01:13 -0400 Subject: [PATCH 05/68] new garbo collector --- packages/aws-cdk/lib/api/garbage-collector.ts | 312 ++++++++---------- 1 file changed, 145 insertions(+), 167 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 7330a317adbf3..d77cba33e0fc2 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -1,13 +1,60 @@ import * as cxapi from '@aws-cdk/cx-api'; -import { ISDK, Mode, SdkProvider } from './aws-auth'; -import { ToolkitInfo } from './toolkit-info'; +import { ISDK, Mode, SdkProvider } from './aws-auth'; import { print } from '../logging'; import { CloudFormation, S3 } from 'aws-sdk'; import * as chalk from 'chalk'; -import * as path from 'path'; +import { ToolkitInfo } from './toolkit-info'; +import path = require('path'); const ISOLATED_TAG = 'awscdk.isolated'; +class ActiveAssets { + private readonly stacks: Set = new Set(); + + public rememberStack(stackTemplate: string) { + this.stacks.add(stackTemplate); + } + + public contains(asset: string): boolean { + for (const stack of this.stacks) { + if (stack.includes(asset)) { + return true; + } + } + return false; + } +} + +class S3Asset { + private tags: S3.TagSet | undefined = undefined; + + public constructor(private readonly bucket: string, private readonly key: string) {} + + public getHash(): string { + return path.parse(this.key).name; + } + + public async getAllTags(s3: S3) { + if (this.tags) { + return this.tags; + } + + const response = await s3.getObjectTagging({ Bucket: this.bucket, Key: this.key }).promise(); + this.tags = response.TagSet; + return response.TagSet; + } + + public async getTag(s3: S3, tag: string) { + const tags = await this.getAllTags(s3); + return tags.find(t => t.Key === tag)?.Value; + } + + public async hasTag(s3: S3, tag: string) { + const tags = await this.getAllTags(s3); + return tags.some(t => t.Key === tag); + } +} + /** * Props for the Garbage Collector */ @@ -62,10 +109,6 @@ interface GarbageCollectorProps { * A class to facilitate Garbage Collection of S3 and ECR assets */ export class GarbageCollector { - private readonly activeHashes = new Set(); - private readonly objects = new Map(); - private bootstrapBucketName: string | undefined = undefined; - private garbageCollectS3Assets: boolean; private garbageCollectEcrAssets: boolean; @@ -83,52 +126,101 @@ export class GarbageCollector { * Perform garbage collection on the resolved environment(s) */ public async garbageCollect() { - print(chalk.green(`Garbage collecting for ${this.props.resolvedEnvironment.account} in ${this.props.resolvedEnvironment.region}`)); - - // set up the sdks used in garbage collection + console.log(this.garbageCollectS3Assets); + // SDKs const sdk = (await this.props.sdkProvider.forEnvironment(this.props.resolvedEnvironment, Mode.ForWriting)).sdk; - const { cfn, s3 } = this.setUpSDKs(sdk); - - if (this.garbageCollectS3Assets) { - print(chalk.green('Getting bootstrap bucket')); - const bucket = await this.getBootstrapBucket(sdk); - print(chalk.green(`Bootstrap Bucket ${bucket}`)); - await this.collectObjects(s3, bucket); - print(chalk.blue(`Object hashes: ${this.objects.size}`)); - } + const cfn = sdk.cloudFormation(); + const s3 = sdk.s3(); - await this.collectHashes(cfn); - print(chalk.green(`Hashes in use: ${this.activeHashes.size}`)); + const activeAssets = new ActiveAssets(); - for (const hash of this.activeHashes) { - const obj = this.objects.get(hash); - if (obj) { - obj.isolated = false; + const refreshStacks = async () => { + const stacks = await this.fetchAllStackTemplates(cfn); + for (const stack of stacks) { + activeAssets.rememberStack(stack); } - } - const isolatedObjects = Array.from(this.objects.values()) - .filter(obj => obj.isolated === true); - - print(chalk.red(`Isolated hashes: ${isolatedObjects.length}`)); + }; - if (!this.props.dryRun) { - if (this.garbageCollectS3Assets) { - print(chalk.green('Tagging Isolated Objects')); - this.tagObjects(s3, await this.getBootstrapBucket(sdk)); + // Refresh stacks every 5 minutes + const timer = setInterval(refreshStacks, 30_000); + + try { + // Grab stack templates first + await refreshStacks(); + + const bucket = await this.getBootstrapBucketName(sdk); + // Process objects in batches of 10,000 + for await (const batch of this.readBucketInBatches(s3, bucket)) { + print(chalk.red(batch.length)); + const currentTime = Date.now(); + const graceDays = this.props.isolationDays; + const isolated = batch.filter((obj) => { + return !activeAssets.contains(obj.getHash()); + }); + + const deletables = graceDays > 0 + ? await Promise.all(isolated.map(async (obj) => { + const tagTime = await obj.getTag(s3, ISOLATED_TAG); + if (tagTime) { + return olderThan(Number(tagTime), currentTime, graceDays) ? obj : null; + } else { + return null; + } + })).then(results => results.filter(Boolean)) + : isolated; + + const taggables = graceDays > 0 + ? await Promise.all(isolated.map(async (obj) => { + const hasTag = await obj.hasTag(s3, ISOLATED_TAG); + return !hasTag ? obj : null; + })).then(results => results.filter(Boolean)) + : []; + + + print(chalk.blue(deletables.length)); + print(chalk.white(taggables.length)); } + } finally { + clearInterval(timer); } } - private setUpSDKs(sdk: ISDK) { - const cfn = sdk.cloudFormation(); - const s3 = sdk.s3(); - return { - cfn, - s3, - }; + private async getBootstrapBucketName(sdk: ISDK): Promise { + const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, undefined); + return info.bucketName; + } + + private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 10000): AsyncGenerator { + let continuationToken: string | undefined; + + do { + const batch: S3Asset[] = []; + + while (batch.length < batchSize) { + const response = await s3.listObjectsV2({ + Bucket: bucket, + ContinuationToken: continuationToken, + MaxKeys: batchSize - batch.length + }).promise(); + + response.Contents?.forEach((obj) => { + if (obj.Key) { + batch.push(new S3Asset(bucket, obj.Key)); + } + }); + + continuationToken = response.NextContinuationToken; + + if (!continuationToken) break; // No more objects to fetch + } + + if (batch.length > 0) { + yield batch; + } + } while (continuationToken); } - private async collectHashes(cfn: CloudFormation) { + private async fetchAllStackTemplates(cfn: CloudFormation) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { const response = await cfn.listStacks({ NextToken: nextToken }).promise(); @@ -138,135 +230,21 @@ export class GarbageCollector { print(chalk.blue(`Parsing through ${stackNames.length} stacks`)); + const templates: string[] = []; for (const stack of stackNames) { - const template = await cfn.getTemplate({ + const summary = await cfn.getTemplateSummary({ StackName: stack, }).promise(); - - const templateHashes = template.TemplateBody?.match(/[a-f0-9]{64}/g); - templateHashes?.forEach(this.activeHashes.add, this.activeHashes); - } - - print(chalk.blue(`Found ${this.activeHashes.size} unique hashes`)); - } - - private async getBootstrapBucket(sdk: ISDK): Promise { - if (!this.bootstrapBucketName) { - const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, undefined); - this.bootstrapBucketName = info.bucketName; - } - return this.bootstrapBucketName; - } - - private async collectObjects(s3: S3, bucket: string) { - await paginateSdkCall(async (nextToken) => { - const response = await s3.listObjectsV2({ - Bucket: bucket, - ContinuationToken: nextToken, - }).promise(); - response.Contents?.forEach((obj) => { - let key = obj.Key; - if (key) { - let hash = path.parse(key).name; - this.objects.set(hash, { - key, - hash, - isolated: true, // Default is isolated - }); - } - }); - return response.NextContinuationToken; - }); - } - - private async tagObjects(s3: S3, bucket: string) { - let newlyTagged = 0; - let alreadyTagged = 0; - let newlyUntagged = 0; - let alreadyUntagged = 0; - for (const [_, obj] of this.objects) { - if (obj.isolated) { - const result = await this.tagIsolatedObject(s3, bucket, obj); - if (result) { - newlyTagged++; - } else { - alreadyTagged++; - } - } else { - const result = await this.untagActiveObject(s3, bucket, obj); - if (result) { - newlyUntagged++; - } else { - alreadyUntagged++; - } - } - } - - print(chalk.white(`Newly Tagged: ${newlyTagged}\nAlready Tagged: ${alreadyTagged}\nNewly Untagged: ${newlyUntagged}\nAlready Untagged: ${alreadyUntagged}`)); - } - - /** - * Returns true if object gets tagged (was previously untagged but is now isolated) - */ - private async tagIsolatedObject(s3: S3, bucket: string, object: S3Object) { - const response = await s3.getObjectTagging({ - Bucket: bucket, - Key: object.key, - }).promise(); - const isolatedTag = await this.getIsolatedTag(response.TagSet); - - // tag new objects with the current date - if (!isolatedTag) { - await s3.putObjectTagging({ - Bucket: bucket, - Key: object.key, - Tagging: { - TagSet: [{ - Key: ISOLATED_TAG, - Value: Date.now().toString(), - }], - }, + const template = await cfn.getTemplate({ + StackName: stack, }).promise(); - return true; + templates.push(template.TemplateBody ?? '' + summary.Parameters); } - return false; - } - /** - * Returns true if object gets untagged (was previously tagged but is now active) - */ - private async untagActiveObject(s3: S3, bucket: string, object: S3Object) { - const response = await s3.getObjectTagging({ - Bucket: bucket, - Key: object.key, - }).promise(); - const isolatedTag = await this.getIsolatedTag(response.TagSet); - - // remove isolated tag and put any other tags back - if (isolatedTag) { - const newTags = response.TagSet?.filter(tag => tag.Key !== ISOLATED_TAG); - // TODO: double check what happens when newTags = [] - await s3.putObjectTagging({ - Bucket: bucket, - Key: object.key, - Tagging: { - TagSet: newTags, - }, - }).promise(); + print(chalk.red('Done parsing through stacks')); - return true; - } - return false; - } - - private async getIsolatedTag(tags: S3.TagSet): Promise { - for (const tag of tags) { - if (tag.Key === ISOLATED_TAG) { - return tag.Value; - } - } - return; + return templates; } } @@ -281,8 +259,8 @@ async function paginateSdkCall(cb: (nextToken?: string) => Promise= graceDays; } From 71062e00b60850f24f9b547200bceea6b69f2898 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Fri, 27 Sep 2024 17:10:43 -0400 Subject: [PATCH 06/68] parallel delete --- packages/aws-cdk/lib/api/garbage-collector.ts | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index d77cba33e0fc2..4ff7140cd9905 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -28,7 +28,7 @@ class ActiveAssets { class S3Asset { private tags: S3.TagSet | undefined = undefined; - public constructor(private readonly bucket: string, private readonly key: string) {} + public constructor(private readonly bucket: string, public readonly key: string) {} public getHash(): string { return path.parse(this.key).name; @@ -149,7 +149,7 @@ export class GarbageCollector { await refreshStacks(); const bucket = await this.getBootstrapBucketName(sdk); - // Process objects in batches of 10,000 + // Process objects in batches of 1000 for await (const batch of this.readBucketInBatches(s3, bucket)) { print(chalk.red(batch.length)); const currentTime = Date.now(); @@ -158,15 +158,16 @@ export class GarbageCollector { return !activeAssets.contains(obj.getHash()); }); - const deletables = graceDays > 0 - ? await Promise.all(isolated.map(async (obj) => { - const tagTime = await obj.getTag(s3, ISOLATED_TAG); - if (tagTime) { - return olderThan(Number(tagTime), currentTime, graceDays) ? obj : null; - } else { + const deletables: S3Asset[] = graceDays > 0 + ? await Promise.all( + isolated.map(async (obj) => { + const tagTime = await obj.getTag(s3, ISOLATED_TAG); + if (tagTime && olderThan(Number(tagTime), currentTime, graceDays)) { + return obj; + } return null; - } - })).then(results => results.filter(Boolean)) + }) + ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) : isolated; const taggables = graceDays > 0 @@ -176,21 +177,45 @@ export class GarbageCollector { })).then(results => results.filter(Boolean)) : []; - print(chalk.blue(deletables.length)); print(chalk.white(taggables.length)); + + if (!this.props.dryRun) { + this.parallelDelete(s3, bucket, deletables); + // parallelTag(taggables, ISOLATED_TAG) + } + + // TODO: maybe undelete } } finally { clearInterval(timer); } } + private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[]) { + const objectsToDelete: S3.ObjectIdentifierList = deletables.map(asset => ({ + Key: asset.key, + })); + + try { + await s3.deleteObjects({ + Bucket: bucket, + Delete: { + Objects: objectsToDelete, + Quiet: true, + }, + }).promise(); + } catch (err) { + console.error(`Error deleting objects: ${err}`); + } + } + private async getBootstrapBucketName(sdk: ISDK): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, undefined); return info.bucketName; } - private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 10000): AsyncGenerator { + private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000): AsyncGenerator { let continuationToken: string | undefined; do { From 6fbcce4a318217169c48594df70a1dd5b3be052e Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 30 Sep 2024 12:54:57 -0400 Subject: [PATCH 07/68] integ test --- .../cli-integ/lib/with-cdk-app.ts | 25 +++++++++++++ .../garbage-collection.integtest.ts | 35 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 934531f45ffcb..924f28678c2b2 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -311,6 +311,18 @@ export interface CdkModernBootstrapCommandOptions extends CommonCdkBootstrapComm readonly usePreviousParameters?: boolean; } +export interface CdkGarbageCollectionCommandOptions { + /** + * @default 0 + */ + readonly days?: number; + + /** + * @default 'all' + */ + readonly type?: 'ecr' | 's3' | 'all'; +} + export class TestFixture extends ShellHelper { public readonly qualifier = this.randomString.slice(0, 10); private readonly bucketsToDelete = new Array(); @@ -444,6 +456,18 @@ export class TestFixture extends ShellHelper { }); } + public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { + const args = ['gc']; + if (options.days) { + args.push('--days', String(options.days)); + } + if (options.type) { + args.push('--type', options.type); + } + + return this.cdk(args); + } + public async cdkMigrate(language: string, stackName: string, inputPath?: string, options?: CdkCliOptions) { return this.cdk([ 'migrate', @@ -610,6 +634,7 @@ async function ensureBootstrapped(fixture: TestFixture) { CDK_NEW_BOOTSTRAP: '1', }, }); + ALREADY_BOOTSTRAPPED_IN_THIS_RUN.add(envSpecifier); } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts new file mode 100644 index 0000000000000..279b55c1e9f1c --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -0,0 +1,35 @@ +import { ListObjectsV2Command } from '@aws-sdk/client-s3'; +import { integTest, randomString, withoutBootstrap } from '../../lib'; + +jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime + +// this is to ensure that asset bundling for apps under a stage does not break +integTest( + 'Basic Garbage Collection', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda'); + fixture.log('Setup complete!'); + await fixture.cdkDestroy('lambda'); + fixture.log('Teardown complete!'); + await fixture.cdkGarbageCollect({ + days: 0, + type: 's3', + }); + fixture.log('Garbage collection complete!'); + + // assert that the bootstrap bucket is empty + await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) + .then((result) => { + expect(result.Contents).toHaveLength(0); + }); + }), +); From 574e8ef25e02e58b4d2ddcee776041e3a11c6e90 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 30 Sep 2024 17:55:03 -0400 Subject: [PATCH 08/68] integ tests mostly work --- .../cli-integ/lib/with-cdk-app.ts | 8 ++ .../garbage-collection.integtest.ts | 76 +++++++++++++++++-- packages/aws-cdk/lib/api/garbage-collector.ts | 72 ++++++++++-------- packages/aws-cdk/lib/cdk-toolkit.ts | 15 +++- packages/aws-cdk/lib/cli.ts | 14 ++-- 5 files changed, 142 insertions(+), 43 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 924f28678c2b2..46dc2829552ae 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -321,6 +321,11 @@ export interface CdkGarbageCollectionCommandOptions { * @default 'all' */ readonly type?: 'ecr' | 's3' | 'all'; + + /** + * @default 'CdkToolkit' + */ + readonly bootstrapStackName?: string; } export class TestFixture extends ShellHelper { @@ -464,6 +469,9 @@ export class TestFixture extends ShellHelper { if (options.type) { args.push('--type', options.type); } + if (options.bootstrapStackName) { + args.push('--bootstrapStackName', options.bootstrapStackName); + } return this.cdk(args); } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index 279b55c1e9f1c..e7f8da87fdbd5 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -3,9 +3,8 @@ import { integTest, randomString, withoutBootstrap } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime -// this is to ensure that asset bundling for apps under a stage does not break integTest( - 'Basic Garbage Collection', + 'Garbage Collection deletes unused assets', withoutBootstrap(async (fixture) => { const toolkitStackName = fixture.bootstrapStackName; const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; @@ -16,20 +15,85 @@ integTest( bootstrapBucketName, }); - await fixture.cdkDeploy('lambda'); + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); fixture.log('Setup complete!'); - await fixture.cdkDestroy('lambda'); - fixture.log('Teardown complete!'); + + await fixture.cdkDestroy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + + await fixture.cdkGarbageCollect({ + days: 0, + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the bootstrap bucket is empty + await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) + .then((result) => { + fixture.log(JSON.stringify(result)); + expect(result.Contents).toBeUndefined(); + }); + }), +); + +integTest( + 'Garbage Collection keeps in use assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + // fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + await fixture.cdkGarbageCollect({ days: 0, type: 's3', + bootstrapStackName: toolkitStackName, }); fixture.log('Garbage collection complete!'); // assert that the bootstrap bucket is empty await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) .then((result) => { - expect(result.Contents).toHaveLength(0); + fixture.log(JSON.stringify(result)); + expect(result.Contents).toHaveLength(2); }); + + await fixture.cdkDestroy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Teardown complete!'); }), ); diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 4ff7140cd9905..54590fb1f8428 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -1,10 +1,10 @@ +import * as path from 'path'; import * as cxapi from '@aws-cdk/cx-api'; -import { ISDK, Mode, SdkProvider } from './aws-auth'; -import { print } from '../logging'; import { CloudFormation, S3 } from 'aws-sdk'; import * as chalk from 'chalk'; +import { ISDK, Mode, SdkProvider } from './aws-auth'; +import { print } from '../logging'; import { ToolkitInfo } from './toolkit-info'; -import path = require('path'); const ISOLATED_TAG = 'awscdk.isolated'; @@ -38,7 +38,7 @@ class S3Asset { if (this.tags) { return this.tags; } - + const response = await s3.getObjectTagging({ Bucket: this.bucket, Key: this.key }).promise(); this.tags = response.TagSet; return response.TagSet; @@ -103,6 +103,8 @@ interface GarbageCollectorProps { * Will be used to make SDK calls to CloudFormation, S3, and ECR. */ readonly sdkProvider: SdkProvider; + + readonly bootstrapStackName?: string; } /** @@ -111,10 +113,12 @@ interface GarbageCollectorProps { export class GarbageCollector { private garbageCollectS3Assets: boolean; private garbageCollectEcrAssets: boolean; + private bootstrapStackName: string; public constructor(readonly props: GarbageCollectorProps) { this.garbageCollectS3Assets = props.type === 's3' || props.type === 'all'; this.garbageCollectEcrAssets = props.type === 'ecr' || props.type === 'all'; + this.bootstrapStackName = props.bootstrapStackName ?? 'CdkToolkit'; // TODO: ECR garbage collection if (this.garbageCollectEcrAssets) { @@ -126,7 +130,7 @@ export class GarbageCollector { * Perform garbage collection on the resolved environment(s) */ public async garbageCollect() { - console.log(this.garbageCollectS3Assets); + print(chalk.black(this.garbageCollectS3Assets)); // SDKs const sdk = (await this.props.sdkProvider.forEnvironment(this.props.resolvedEnvironment, Mode.ForWriting)).sdk; const cfn = sdk.cloudFormation(); @@ -148,7 +152,7 @@ export class GarbageCollector { // Grab stack templates first await refreshStacks(); - const bucket = await this.getBootstrapBucketName(sdk); + const bucket = await this.getBootstrapBucketName(sdk, this.bootstrapStackName); // Process objects in batches of 1000 for await (const batch of this.readBucketInBatches(s3, bucket)) { print(chalk.red(batch.length)); @@ -158,30 +162,30 @@ export class GarbageCollector { return !activeAssets.contains(obj.getHash()); }); - const deletables: S3Asset[] = graceDays > 0 + const deletables: S3Asset[] = graceDays > 0 ? await Promise.all( - isolated.map(async (obj) => { - const tagTime = await obj.getTag(s3, ISOLATED_TAG); - if (tagTime && olderThan(Number(tagTime), currentTime, graceDays)) { - return obj; - } - return null; - }) - ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) + isolated.map(async (obj) => { + const tagTime = await obj.getTag(s3, ISOLATED_TAG); + if (tagTime && olderThan(Number(tagTime), currentTime, graceDays)) { + return obj; + } + return null; + }), + ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) : isolated; - const taggables = graceDays > 0 + const taggables = graceDays > 0 ? await Promise.all(isolated.map(async (obj) => { - const hasTag = await obj.hasTag(s3, ISOLATED_TAG); - return !hasTag ? obj : null; - })).then(results => results.filter(Boolean)) + const hasTag = await obj.hasTag(s3, ISOLATED_TAG); + return !hasTag ? obj : null; + })).then(results => results.filter(Boolean)) : []; print(chalk.blue(deletables.length)); print(chalk.white(taggables.length)); if (!this.props.dryRun) { - this.parallelDelete(s3, bucket, deletables); + await this.parallelDelete(s3, bucket, deletables); // parallelTag(taggables, ISOLATED_TAG) } @@ -197,7 +201,10 @@ export class GarbageCollector { Key: asset.key, })); - try { + // eslint-disable-next-line no-console + console.log('O', objectsToDelete); + + try { await s3.deleteObjects({ Bucket: bucket, Delete: { @@ -206,26 +213,26 @@ export class GarbageCollector { }, }).promise(); } catch (err) { - console.error(`Error deleting objects: ${err}`); + print(chalk.red(`Error deleting objects: ${err}`)); } } - private async getBootstrapBucketName(sdk: ISDK): Promise { - const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, undefined); + private async getBootstrapBucketName(sdk: ISDK, bootstrapStackName: string): Promise { + const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); return info.bucketName; } private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000): AsyncGenerator { let continuationToken: string | undefined; - + do { const batch: S3Asset[] = []; - + while (batch.length < batchSize) { const response = await s3.listObjectsV2({ Bucket: bucket, ContinuationToken: continuationToken, - MaxKeys: batchSize - batch.length + MaxKeys: batchSize - batch.length, }).promise(); response.Contents?.forEach((obj) => { @@ -235,7 +242,7 @@ export class GarbageCollector { }); continuationToken = response.NextContinuationToken; - + if (!continuationToken) break; // No more objects to fetch } @@ -249,7 +256,9 @@ export class GarbageCollector { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { const response = await cfn.listStacks({ NextToken: nextToken }).promise(); - stackNames.push(...(response.StackSummaries ?? []).map(s => s.StackId ?? s.StackName)); + // Deleted stacks are ignored + // TODO: need to filter for bootstrap version!!! + stackNames.push(...(response.StackSummaries ?? []).filter(s => !['DELETE_COMPLETE', 'DELETE_IN_PROGRESS'].includes(s.StackStatus)).map(s => s.StackId ?? s.StackName)); return response.NextToken; }); @@ -257,14 +266,15 @@ export class GarbageCollector { const templates: string[] = []; for (const stack of stackNames) { - const summary = await cfn.getTemplateSummary({ + let summary; + summary = await cfn.getTemplateSummary({ StackName: stack, }).promise(); const template = await cfn.getTemplate({ StackName: stack, }).promise(); - templates.push(template.TemplateBody ?? '' + summary.Parameters); + templates.push(template.TemplateBody ?? '' + summary?.Parameters); } print(chalk.red('Done parsing through stacks')); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 74ce85ca15ab1..8a4ec480213c6 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -9,10 +9,10 @@ import * as uuid from 'uuid'; import { DeploymentMethod } from './api'; import { SdkProvider } from './api/aws-auth'; import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap'; -import { GarbageCollector } from './api/garbage-collector'; import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly'; import { CloudExecutable } from './api/cxapp/cloud-executable'; import { Deployments } from './api/deployments'; +import { GarbageCollector } from './api/garbage-collector'; import { HotswapMode } from './api/hotswap/common'; import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs'; import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor'; @@ -776,6 +776,8 @@ export class CdkToolkit { // By default, glob for everything const environmentSpecs = userEnvironmentSpecs.length > 0 ? [...userEnvironmentSpecs] : ['**']; + // eslint-disable-next-line no-console + console.log(environmentSpecs); // Partition into globs and non-globs (this will mutate environmentSpecs). const globSpecs = partition(environmentSpecs, looksLikeGlob); if (globSpecs.length > 0 && !this.props.cloudExecutable.hasApp) { @@ -792,11 +794,21 @@ export class CdkToolkit { ...environmentsFromDescriptors(environmentSpecs), ]; + if (this.props.cloudExecutable.hasApp) { + environments.push(...await globEnvironmentsFromStacks(await this.selectStacksForList([]), globSpecs, this.props.sdkProvider)); + } + + // eslint-disable-next-line no-console + console.log('E', environments); + await Promise.all(environments.map(async (environment) => { + // eslint-disable-next-line no-console + console.log('EE', environment); success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); const gc = new GarbageCollector({ sdkProvider: this.props.sdkProvider, resolvedEnvironment: environment, + bootstrapStackName: options.bootstrapStackName, isolationDays: options.days, dryRun: options.dryRun ?? false, tagOnly: options.tagOnly ?? false, @@ -1455,6 +1467,7 @@ export interface GarbageCollectionOptions { readonly tagOnly?: boolean; readonly type: 'ecr' | 's3' | 'all'; readonly days: number; + readonly bootstrapStackName?: string; } export interface MigrateOptions { diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 33f08a77539ac..6fa1faa28a1a0 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -114,6 +114,14 @@ async function parseCommandLineArguments(args: string[]) { .option('template', { type: 'string', requiresArg: true, desc: 'Use the template from the given file instead of the built-in one (use --show-template to obtain an example)' }) .option('previous-parameters', { type: 'boolean', default: true, desc: 'Use previous values for existing parameters (you must specify all parameters on every deployment if this is disabled)' }), ) + .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs + .option('dry-run', { type: 'boolean', desc: 'List assets instead of garbage collecting them', default: false }) + .option('tag-only', { type: 'boolean', desc: 'Tag assets as isolated without deleting them', default: false }) + .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) + .option('days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) + .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) + .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }), + ) .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', (yargs: Argv) => yargs .option('all', { type: 'boolean', default: false, desc: 'Deploy all available stacks' }) .option('build-exclude', { type: 'array', alias: 'E', nargs: 1, desc: 'Do not rebuild asset with the given ID. Can be specified multiple times', default: [] }) @@ -271,11 +279,6 @@ async function parseCommandLineArguments(args: string[]) { .command('notices', 'Returns a list of relevant notices', (yargs: Argv) => yargs .option('unacknowledged', { type: 'boolean', alias: 'u', default: false, desc: 'Returns a list of unacknowledged notices' }), ) - .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs - .option('dry-run', { type: 'boolean', desc: 'List assets instead of garbage collecting them', default: false }) - .option('tag-only', { type: 'boolean', desc: 'Tag assets as isolated without deleting them', default: false }) - .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) - .option('days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 })) .command('init [TEMPLATE]', 'Create a new, empty CDK project from a template.', (yargs: Argv) => yargs .option('language', { type: 'string', alias: 'l', desc: 'The language to be used for the new project (default can be configured in ~/.cdk.json)', choices: initTemplateLanguages }) .option('list', { type: 'boolean', desc: 'List the available templates' }) @@ -676,6 +679,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Tue, 1 Oct 2024 14:10:48 -0400 Subject: [PATCH 09/68] working integ test --- .../garbage-collection.integtest.ts | 4 +- packages/aws-cdk/lib/api/garbage-collector.ts | 49 ++++++++++++++----- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index e7f8da87fdbd5..e1b65e30885c9 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -44,7 +44,6 @@ integTest( // assert that the bootstrap bucket is empty await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) .then((result) => { - fixture.log(JSON.stringify(result)); expect(result.Contents).toBeUndefined(); }); }), @@ -55,7 +54,7 @@ integTest( withoutBootstrap(async (fixture) => { const toolkitStackName = fixture.bootstrapStackName; const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; - // fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case await fixture.cdkBootstrapModern({ toolkitStackName, @@ -82,7 +81,6 @@ integTest( // assert that the bootstrap bucket is empty await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) .then((result) => { - fixture.log(JSON.stringify(result)); expect(result.Contents).toHaveLength(2); }); diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 54590fb1f8428..4703d9e25943f 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -4,7 +4,7 @@ import { CloudFormation, S3 } from 'aws-sdk'; import * as chalk from 'chalk'; import { ISDK, Mode, SdkProvider } from './aws-auth'; import { print } from '../logging'; -import { ToolkitInfo } from './toolkit-info'; +import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from './toolkit-info'; const ISOLATED_TAG = 'awscdk.isolated'; @@ -118,7 +118,7 @@ export class GarbageCollector { public constructor(readonly props: GarbageCollectorProps) { this.garbageCollectS3Assets = props.type === 's3' || props.type === 'all'; this.garbageCollectEcrAssets = props.type === 'ecr' || props.type === 'all'; - this.bootstrapStackName = props.bootstrapStackName ?? 'CdkToolkit'; + this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; // TODO: ECR garbage collection if (this.garbageCollectEcrAssets) { @@ -136,10 +136,12 @@ export class GarbageCollector { const cfn = sdk.cloudFormation(); const s3 = sdk.s3(); + const qualifier = await this.getBootstrapQualifier(sdk, this.bootstrapStackName); + const activeAssets = new ActiveAssets(); const refreshStacks = async () => { - const stacks = await this.fetchAllStackTemplates(cfn); + const stacks = await this.fetchAllStackTemplates(cfn, qualifier); for (const stack of stacks) { activeAssets.rememberStack(stack); } @@ -184,7 +186,7 @@ export class GarbageCollector { print(chalk.blue(deletables.length)); print(chalk.white(taggables.length)); - if (!this.props.dryRun) { + if (!this.props.dryRun && deletables.length > 0) { await this.parallelDelete(s3, bucket, deletables); // parallelTag(taggables, ISOLATED_TAG) } @@ -219,9 +221,16 @@ export class GarbageCollector { private async getBootstrapBucketName(sdk: ISDK, bootstrapStackName: string): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); + print(chalk.blue(JSON.stringify(info.bootstrapStack.parameters))); return info.bucketName; } + private async getBootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { + const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); + print(chalk.blue(JSON.stringify(info.bootstrapStack.parameters))); + return info.bootstrapStack.parameters.Qualifier; + } + private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000): AsyncGenerator { let continuationToken: string | undefined; @@ -252,13 +261,24 @@ export class GarbageCollector { } while (continuationToken); } - private async fetchAllStackTemplates(cfn: CloudFormation) { + private async fetchAllStackTemplates(cfn: CloudFormation, qualifier: string) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { const response = await cfn.listStacks({ NextToken: nextToken }).promise(); + + // We cannot operate on REVIEW_IN_PROGRESS stacks because we do not know what the template looks like in this case + const reviewInProgressStacks = response.StackSummaries?.filter(s => s.StackStatus === 'REVIEW_IN_PROGRESS') ?? []; + if (reviewInProgressStacks.length > 0) { + throw new Error(`Stacks in REVIEW_IN_PROGRESS state are not allowed: ${reviewInProgressStacks.map(s => s.StackName).join(', ')}`); + } + // Deleted stacks are ignored - // TODO: need to filter for bootstrap version!!! - stackNames.push(...(response.StackSummaries ?? []).filter(s => !['DELETE_COMPLETE', 'DELETE_IN_PROGRESS'].includes(s.StackStatus)).map(s => s.StackId ?? s.StackName)); + stackNames.push( + ...(response.StackSummaries ?? []) + .filter(s => s.StackStatus !== 'DELETE_COMPLETE' && s.StackStatus !== 'DELETE_IN_PROGRESS') + .map(s => s.StackId ?? s.StackName) + ); + return response.NextToken; }); @@ -270,11 +290,18 @@ export class GarbageCollector { summary = await cfn.getTemplateSummary({ StackName: stack, }).promise(); - const template = await cfn.getTemplate({ - StackName: stack, - }).promise(); - templates.push(template.TemplateBody ?? '' + summary?.Parameters); + // TODO: triple check that this all stacks have this parameter + const bootstrapVersion = summary?.Parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); + const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); + if (splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] == qualifier) { + // This stack is bootstrapped to the right version we are searching for + const template = await cfn.getTemplate({ + StackName: stack, + }).promise(); + + templates.push(template.TemplateBody ?? '' + summary?.Parameters); + } } print(chalk.red('Done parsing through stacks')); From 689b283ed2ce974fbd46b00d52d3d8ff02eb20b5 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 1 Oct 2024 14:17:37 -0400 Subject: [PATCH 10/68] documentation --- packages/aws-cdk/lib/api/garbage-collector.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 4703d9e25943f..8f38f031f0663 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -156,6 +156,8 @@ export class GarbageCollector { const bucket = await this.getBootstrapBucketName(sdk, this.bootstrapStackName); // Process objects in batches of 1000 + // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario + // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. for await (const batch of this.readBucketInBatches(s3, bucket)) { print(chalk.red(batch.length)); const currentTime = Date.now(); @@ -291,11 +293,15 @@ export class GarbageCollector { StackName: stack, }).promise(); - // TODO: triple check that this all stacks have this parameter + // Filter out stacks that we KNOW are using a different bootstrap qualifier + // This is necessary because a stack under a different bootstrap could coincidentally reference the same hash + // and cause a false negative (cause an asset to be preserved when its isolated) const bootstrapVersion = summary?.Parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); - if (splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] == qualifier) { - // This stack is bootstrapped to the right version we are searching for + if (splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier) { + // This stack is definitely bootstrapped to a different qualifier so we can safely ignore it + continue; + } else { const template = await cfn.getTemplate({ StackName: stack, }).promise(); From 96fbb5a76b8e0319a060cfcb0f97de96703c55f3 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 1 Oct 2024 14:41:27 -0400 Subject: [PATCH 11/68] delete parallel --- packages/aws-cdk/lib/api/garbage-collector.ts | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 8f38f031f0663..ee5a7bf9e9397 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -6,6 +6,10 @@ import { ISDK, Mode, SdkProvider } from './aws-auth'; import { print } from '../logging'; import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from './toolkit-info'; +// Must use a require() otherwise esbuild complains +// eslint-disable-next-line @typescript-eslint/no-require-imports +const pLimit: typeof import('p-limit') = require('p-limit'); + const ISOLATED_TAG = 'awscdk.isolated'; class ActiveAssets { @@ -178,19 +182,26 @@ export class GarbageCollector { ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) : isolated; - const taggables = graceDays > 0 - ? await Promise.all(isolated.map(async (obj) => { - const hasTag = await obj.hasTag(s3, ISOLATED_TAG); - return !hasTag ? obj : null; - })).then(results => results.filter(Boolean)) + const taggables: S3Asset[] = graceDays > 0 + ? await Promise.all( + isolated.map(async (obj) => { + const hasTag = await obj.hasTag(s3, ISOLATED_TAG); + return hasTag ? null : obj; + }), + ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) : []; + print(chalk.blue(deletables.length)); print(chalk.white(taggables.length)); - if (!this.props.dryRun && deletables.length > 0) { - await this.parallelDelete(s3, bucket, deletables); - // parallelTag(taggables, ISOLATED_TAG) + if (!this.props.dryRun) { + if (deletables.length > 0) { + await this.parallelDelete(s3, bucket, deletables); + } + if (taggables.length > 0) { + await this.parallelTag(s3, bucket, taggables); + } } // TODO: maybe undelete @@ -200,14 +211,37 @@ export class GarbageCollector { } } + private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[]) { + const limit = pLimit(5); + + const input = []; + + for (const obj of taggables) { + input.push(limit(() => { + s3.putObjectTagging({ + Bucket: bucket, + Key: obj.key, + Tagging: { + TagSet: [ + { + Key: ISOLATED_TAG, + Value: String(Date.now()), + }, + ], + }, + }) + })); + } + + await Promise.all(input); + print(chalk.green(`Tagged ${taggables.length} assets`)); + } + private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[]) { const objectsToDelete: S3.ObjectIdentifierList = deletables.map(asset => ({ Key: asset.key, })); - // eslint-disable-next-line no-console - console.log('O', objectsToDelete); - try { await s3.deleteObjects({ Bucket: bucket, @@ -216,6 +250,8 @@ export class GarbageCollector { Quiet: true, }, }).promise(); + + print(chalk.green(`Deleted ${deletables.length} assets`)); } catch (err) { print(chalk.red(`Error deleting objects: ${err}`)); } @@ -223,13 +259,11 @@ export class GarbageCollector { private async getBootstrapBucketName(sdk: ISDK, bootstrapStackName: string): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); - print(chalk.blue(JSON.stringify(info.bootstrapStack.parameters))); return info.bucketName; } private async getBootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); - print(chalk.blue(JSON.stringify(info.bootstrapStack.parameters))); return info.bootstrapStack.parameters.Qualifier; } From 64fe5db82b31ee33a4b4196e6071c2708371d00f Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 1 Oct 2024 14:47:51 -0400 Subject: [PATCH 12/68] more docs --- packages/aws-cdk/lib/api/garbage-collector.ts | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index ee5a7bf9e9397..f914a08c2ba6e 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -108,6 +108,11 @@ interface GarbageCollectorProps { */ readonly sdkProvider: SdkProvider; + /** + * The name of the bootstrap stack to look for. + * + * @default DEFAULT_TOOLKIT_STACK_NAME + */ readonly bootstrapStackName?: string; } @@ -131,7 +136,7 @@ export class GarbageCollector { } /** - * Perform garbage collection on the resolved environment(s) + * Perform garbage collection on the resolved environment. */ public async garbageCollect() { print(chalk.black(this.garbageCollectS3Assets)); @@ -192,8 +197,8 @@ export class GarbageCollector { : []; - print(chalk.blue(deletables.length)); - print(chalk.white(taggables.length)); + print(chalk.blue(`${deletables.length} deletable assets`)); + print(chalk.white(`${taggables.length} taggable assets`)); if (!this.props.dryRun) { if (deletables.length > 0) { @@ -211,6 +216,10 @@ export class GarbageCollector { } } + /** + * Tag objects in parallel using p-limit. The putObjectTagging API does not + * support batch tagging so we must handle the parallelism client-side. + */ private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[]) { const limit = pLimit(5); @@ -237,6 +246,9 @@ export class GarbageCollector { print(chalk.green(`Tagged ${taggables.length} assets`)); } + /** + * Delete objects in parallel. The deleteObjects API supports batches of 1000. + */ private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[]) { const objectsToDelete: S3.ObjectIdentifierList = deletables.map(asset => ({ Key: asset.key, @@ -267,6 +279,9 @@ export class GarbageCollector { return info.bootstrapStack.parameters.Qualifier; } + /** + * Generator function that reads objects from the S3 Bucket in batches. + */ private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000): AsyncGenerator { let continuationToken: string | undefined; @@ -297,6 +312,15 @@ export class GarbageCollector { } while (continuationToken); } + /** + * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: + * - stacks in DELETE_COMPLETE or DELET_IN_PROGRES stage + * - stacks that are using a different bootstrap qualifier + * + * It fails on the following stacks because we cannot get the template and therefore have an imcomplete + * understanding of what assets are being used. + * - stacks in REVIEW_IN_PROGRESS stage + */ private async fetchAllStackTemplates(cfn: CloudFormation, qualifier: string) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { From 6f2381d971a167ebeed177902579c879ceedec89 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 1 Oct 2024 15:43:59 -0400 Subject: [PATCH 13/68] tagging works now --- .../garbage-collection.integtest.ts | 53 ++++++++++++++++++- packages/aws-cdk/lib/api/garbage-collector.ts | 13 ++--- packages/aws-cdk/lib/cdk-toolkit.ts | 5 -- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index e1b65e30885c9..d0dfa4e4163d0 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -1,4 +1,4 @@ -import { ListObjectsV2Command } from '@aws-sdk/client-s3'; +import { GetObjectTaggingCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'; import { integTest, randomString, withoutBootstrap } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -95,3 +95,54 @@ integTest( fixture.log('Teardown complete!'); }), ); + +integTest( + 'Garbage Collection tags unused assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + + await fixture.cdkDestroy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + + await fixture.cdkGarbageCollect({ + days: 100, // this will ensure that we do not delete assets immediately (and just tag them) + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the bootstrap bucket is empty + await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) + .then(async (result) => { + // eslint-disable-next-line no-console + console.log(JSON.stringify(result.Contents)); + expect(result.Contents).toHaveLength(2); + const key = result.Contents![0].Key; + const tags = await fixture.aws.s3.send(new GetObjectTaggingCommand({ Bucket: bootstrapBucketName, Key: key })); + expect(tags.TagSet).toHaveLength(1); + }); + }), +); diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index f914a08c2ba6e..ef09724177f61 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -222,11 +222,9 @@ export class GarbageCollector { */ private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[]) { const limit = pLimit(5); - - const input = []; - + for (const obj of taggables) { - input.push(limit(() => { + await limit(() => s3.putObjectTagging({ Bucket: bucket, Key: obj.key, @@ -238,11 +236,10 @@ export class GarbageCollector { }, ], }, - }) - })); + }).promise(), + ); } - - await Promise.all(input); + print(chalk.green(`Tagged ${taggables.length} assets`)); } diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 8a4ec480213c6..b0ba028c048ee 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -798,12 +798,7 @@ export class CdkToolkit { environments.push(...await globEnvironmentsFromStacks(await this.selectStacksForList([]), globSpecs, this.props.sdkProvider)); } - // eslint-disable-next-line no-console - console.log('E', environments); - await Promise.all(environments.map(async (environment) => { - // eslint-disable-next-line no-console - console.log('EE', environment); success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); const gc = new GarbageCollector({ sdkProvider: this.props.sdkProvider, From 890daaf88833b384ecf306b8656a7cf39dcc8e9b Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 1 Oct 2024 15:55:18 -0400 Subject: [PATCH 14/68] delete logs and add more docs --- .../garbage-collection.integtest.ts | 2 - packages/aws-cdk/lib/api/garbage-collector.ts | 10 +- packages/aws-cdk/lib/cdk-toolkit.ts | 95 +++++++++++-------- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index d0dfa4e4163d0..16bbeabb20fd1 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -137,8 +137,6 @@ integTest( // assert that the bootstrap bucket is empty await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) .then(async (result) => { - // eslint-disable-next-line no-console - console.log(JSON.stringify(result.Contents)); expect(result.Contents).toHaveLength(2); const key = result.Contents![0].Key; const tags = await fixture.aws.s3.send(new GetObjectTaggingCommand({ Bucket: bootstrapBucketName, Key: key })); diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index ef09724177f61..2029c44caad21 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -102,10 +102,10 @@ interface GarbageCollectorProps { readonly resolvedEnvironment: cxapi.Environment; /** - * SDK provider (seeded with default credentials) - * - * Will be used to make SDK calls to CloudFormation, S3, and ECR. - */ + * SDK provider (seeded with default credentials) + * + * Will be used to make SDK calls to CloudFormation, S3, and ECR. + */ readonly sdkProvider: SdkProvider; /** @@ -201,7 +201,7 @@ export class GarbageCollector { print(chalk.white(`${taggables.length} taggable assets`)); if (!this.props.dryRun) { - if (deletables.length > 0) { + if (!this.props.tagOnly && deletables.length > 0) { await this.parallelDelete(s3, bucket, deletables); } if (taggables.length > 0) { diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index b0ba028c048ee..81fc5fecb1c3c 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -725,29 +725,7 @@ export class CdkToolkit { // If there is an '--app' argument and an environment looks like a glob, we // select the environments from the app. Otherwise, use what the user said. - // By default, glob for everything - const environmentSpecs = userEnvironmentSpecs.length > 0 ? [...userEnvironmentSpecs] : ['**']; - - // Partition into globs and non-globs (this will mutate environmentSpecs). - const globSpecs = partition(environmentSpecs, looksLikeGlob); - if (globSpecs.length > 0 && !this.props.cloudExecutable.hasApp) { - if (userEnvironmentSpecs.length > 0) { - // User did request this glob - throw new Error(`'${globSpecs}' is not an environment name. Specify an environment name like 'aws://123456789012/us-east-1', or run in a directory with 'cdk.json' to use wildcards.`); - } else { - // User did not request anything - throw new Error('Specify an environment name like \'aws://123456789012/us-east-1\', or run in a directory with \'cdk.json\'.'); - } - } - - const environments: cxapi.Environment[] = [ - ...environmentsFromDescriptors(environmentSpecs), - ]; - - // If there is an '--app' argument, select the environments from the app. - if (this.props.cloudExecutable.hasApp) { - environments.push(...await globEnvironmentsFromStacks(await this.selectStacksForList([]), globSpecs, this.props.sdkProvider)); - } + const environments = await this.defineEnvironments(userEnvironmentSpecs); await Promise.all(environments.map(async (environment) => { success(' ⏳ Bootstrapping environment %s...', chalk.blue(environment.name)); @@ -769,15 +747,27 @@ export class CdkToolkit { * @param options Options for Garbage Collection */ public async garbageCollect(userEnvironmentSpecs: string[], options: GarbageCollectionOptions) { - // eslint-disable-next-line no-console - console.log(userEnvironmentSpecs, options); + const environments = await this.defineEnvironments(userEnvironmentSpecs); + + await Promise.all(environments.map(async (environment) => { + success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); + const gc = new GarbageCollector({ + sdkProvider: this.props.sdkProvider, + resolvedEnvironment: environment, + bootstrapStackName: options.bootstrapStackName, + isolationDays: options.days, + dryRun: options.dryRun ?? false, + tagOnly: options.tagOnly ?? false, + type: options.type ?? 'all', + }); + await gc.garbageCollect(); + })); + } - // TODO: copied from bootstrap, make into function + private async defineEnvironments(userEnvironmentSpecs: string[]): Promise { // By default, glob for everything const environmentSpecs = userEnvironmentSpecs.length > 0 ? [...userEnvironmentSpecs] : ['**']; - // eslint-disable-next-line no-console - console.log(environmentSpecs); // Partition into globs and non-globs (this will mutate environmentSpecs). const globSpecs = partition(environmentSpecs, looksLikeGlob); if (globSpecs.length > 0 && !this.props.cloudExecutable.hasApp) { @@ -794,23 +784,12 @@ export class CdkToolkit { ...environmentsFromDescriptors(environmentSpecs), ]; + // If there is an '--app' argument, select the environments from the app. if (this.props.cloudExecutable.hasApp) { environments.push(...await globEnvironmentsFromStacks(await this.selectStacksForList([]), globSpecs, this.props.sdkProvider)); } - await Promise.all(environments.map(async (environment) => { - success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); - const gc = new GarbageCollector({ - sdkProvider: this.props.sdkProvider, - resolvedEnvironment: environment, - bootstrapStackName: options.bootstrapStackName, - isolationDays: options.days, - dryRun: options.dryRun ?? false, - tagOnly: options.tagOnly ?? false, - type: options.type ?? 'all', - }); - await gc.garbageCollect(); - })); + return environments; } /** @@ -1457,11 +1436,45 @@ export interface DestroyOptions { readonly ci?: boolean; } +/** + * Options for the garbage collection + */ export interface GarbageCollectionOptions { + /** + * Whether to perform a dry run or not. + * A dry run means that isolated objects are printed to stdout + * but not tagged or deleted + * + * @default false + */ readonly dryRun?: boolean; + + /** + * Whether to only perform tagging. If this is set, no deletion happens + * + * @default false + */ readonly tagOnly?: boolean; + + /** + * The type of the assets to be garbage collected. + * + * @default 'all' + */ readonly type: 'ecr' | 's3' | 'all'; + + /** + * Elapsed time between an asset being marked as isolated and actually deleted. + * + * @default 0 + */ readonly days: number; + + /** + * The stack name of the bootstrap stack. + * + * @default DEFAULT_TOOLKIT_STACK_NAME + */ readonly bootstrapStackName?: string; } From e85c622d3700827fcd57ecb30a26ca2932b3e4e6 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 1 Oct 2024 16:17:08 -0400 Subject: [PATCH 15/68] unstable --- .../cli-integ/lib/with-cdk-app.ts | 2 +- packages/aws-cdk/lib/api/garbage-collector.ts | 32 +++++++++++-------- packages/aws-cdk/lib/cli.ts | 4 +++ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 46dc2829552ae..da28e07d2a6ae 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -462,7 +462,7 @@ export class TestFixture extends ShellHelper { } public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { - const args = ['gc']; + const args = ['gc', '--unstable=gc']; // TODO: remove when stabilizing if (options.days) { args.push('--days', String(options.days)); } diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 2029c44caad21..eedd989cf23d5 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -175,19 +175,24 @@ export class GarbageCollector { return !activeAssets.contains(obj.getHash()); }); - const deletables: S3Asset[] = graceDays > 0 - ? await Promise.all( - isolated.map(async (obj) => { - const tagTime = await obj.getTag(s3, ISOLATED_TAG); - if (tagTime && olderThan(Number(tagTime), currentTime, graceDays)) { - return obj; - } - return null; - }), - ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) - : isolated; - - const taggables: S3Asset[] = graceDays > 0 + let deletables: S3Asset[] = []; + + // no items are deletable if tagOnly is set + if (this.props.tagOnly) { + deletables = graceDays > 0 + ? await Promise.all( + isolated.map(async (obj) => { + const tagTime = await obj.getTag(s3, ISOLATED_TAG); + if (tagTime && olderThan(Number(tagTime), currentTime, graceDays)) { + return obj; + } + return null; + }), + ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) + : isolated; + } + + const taggables: S3Asset[] = graceDays > 0 ? await Promise.all( isolated.map(async (obj) => { const hasTag = await obj.hasTag(s3, ISOLATED_TAG); @@ -195,7 +200,6 @@ export class GarbageCollector { }), ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) : []; - print(chalk.blue(`${deletables.length} deletable assets`)); print(chalk.white(`${taggables.length} taggable assets`)); diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 6fa1faa28a1a0..5bbfa450ddc9e 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -115,6 +115,7 @@ async function parseCommandLineArguments(args: string[]) { .option('previous-parameters', { type: 'boolean', default: true, desc: 'Use previous values for existing parameters (you must specify all parameters on every deployment if this is disabled)' }), ) .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs + .option('unstable', { type: 'array', desc: 'Opt in to specific unstable features. Can be specified multiple times.', default: [] }) .option('dry-run', { type: 'boolean', desc: 'List assets instead of garbage collecting them', default: false }) .option('tag-only', { type: 'boolean', desc: 'Tag assets as isolated without deleting them', default: false }) .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) @@ -674,6 +675,9 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Tue, 1 Oct 2024 16:41:09 -0400 Subject: [PATCH 16/68] linter --- packages/aws-cdk/THIRD_PARTY_LICENSES | 28 +++++++++++++++++++ packages/aws-cdk/lib/api/garbage-collector.ts | 24 ++++++++-------- packages/aws-cdk/package.json | 1 + 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/packages/aws-cdk/THIRD_PARTY_LICENSES b/packages/aws-cdk/THIRD_PARTY_LICENSES index a6c6ecf886ca5..95954ae901fea 100644 --- a/packages/aws-cdk/THIRD_PARTY_LICENSES +++ b/packages/aws-cdk/THIRD_PARTY_LICENSES @@ -2268,6 +2268,20 @@ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +---------------- + +** p-limit@3.1.0 - https://www.npmjs.com/package/p-limit/v/3.1.0 | MIT +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + ---------------- ** pac-proxy-agent@7.0.2 - https://www.npmjs.com/package/pac-proxy-agent/v/7.0.2 | MIT @@ -3582,6 +3596,20 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +---------------- + +** yocto-queue@0.1.0 - https://www.npmjs.com/package/yocto-queue/v/0.1.0 | MIT +MIT License + +Copyright (c) Sindre Sorhus (https://sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + ---------------- ** zip-stream@4.1.1 - https://www.npmjs.com/package/zip-stream/v/4.1.1 | MIT diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index eedd989cf23d5..498be84652840 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -110,7 +110,7 @@ interface GarbageCollectorProps { /** * The name of the bootstrap stack to look for. - * + * * @default DEFAULT_TOOLKIT_STACK_NAME */ readonly bootstrapStackName?: string; @@ -176,7 +176,7 @@ export class GarbageCollector { }); let deletables: S3Asset[] = []; - + // no items are deletable if tagOnly is set if (this.props.tagOnly) { deletables = graceDays > 0 @@ -194,11 +194,11 @@ export class GarbageCollector { const taggables: S3Asset[] = graceDays > 0 ? await Promise.all( - isolated.map(async (obj) => { - const hasTag = await obj.hasTag(s3, ISOLATED_TAG); - return hasTag ? null : obj; - }), - ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) + isolated.map(async (obj) => { + const hasTag = await obj.hasTag(s3, ISOLATED_TAG); + return hasTag ? null : obj; + }), + ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) : []; print(chalk.blue(`${deletables.length} deletable assets`)); @@ -226,7 +226,7 @@ export class GarbageCollector { */ private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[]) { const limit = pLimit(5); - + for (const obj of taggables) { await limit(() => s3.putObjectTagging({ @@ -243,7 +243,7 @@ export class GarbageCollector { }).promise(), ); } - + print(chalk.green(`Tagged ${taggables.length} assets`)); } @@ -317,7 +317,7 @@ export class GarbageCollector { * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: * - stacks in DELETE_COMPLETE or DELET_IN_PROGRES stage * - stacks that are using a different bootstrap qualifier - * + * * It fails on the following stacks because we cannot get the template and therefore have an imcomplete * understanding of what assets are being used. * - stacks in REVIEW_IN_PROGRESS stage @@ -337,7 +337,7 @@ export class GarbageCollector { stackNames.push( ...(response.StackSummaries ?? []) .filter(s => s.StackStatus !== 'DELETE_COMPLETE' && s.StackStatus !== 'DELETE_IN_PROGRESS') - .map(s => s.StackId ?? s.StackName) + .map(s => s.StackId ?? s.StackName), ); return response.NextToken; @@ -364,7 +364,7 @@ export class GarbageCollector { const template = await cfn.getTemplate({ StackName: stack, }).promise(); - + templates.push(template.TemplateBody ?? '' + summary?.Parameters); } } diff --git a/packages/aws-cdk/package.json b/packages/aws-cdk/package.json index 7946d4256d6e5..ad1e4113a5503 100644 --- a/packages/aws-cdk/package.json +++ b/packages/aws-cdk/package.json @@ -115,6 +115,7 @@ "minimatch": "^9.0.5", "promptly": "^3.2.0", "proxy-agent": "^6.4.0", + "p-limit": "3.1.0", "semver": "^7.6.3", "source-map-support": "^0.5.21", "strip-ansi": "^6.0.1", From 4a5fd5033b7095a35b1311eb204a9c57404c6145 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 2 Oct 2024 13:28:45 -0400 Subject: [PATCH 17/68] yarnlock --- yarn.lock | 45 ++++++++++----------------------------------- 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/yarn.lock b/yarn.lock index 86eda7859163b..905067cff2fd9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14838,6 +14838,13 @@ p-finally@^1.0.0: resolved "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== +p-limit@3.1.0, p-limit@^3.0.2, p-limit@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + p-limit@^1.1.0: version "1.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" @@ -14852,13 +14859,6 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.0.2, p-limit@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" - integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== - dependencies: - yocto-queue "^0.1.0" - p-locate@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" @@ -16556,16 +16556,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@*, string-width@^1.0.1, "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2: +"string-width-cjs@npm:string-width@^4.2.0", string-width@*, string-width@^1.0.1, "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.0.1, string-width@^5.1.2: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -16630,7 +16621,7 @@ stringify-package@^1.0.1: resolved "https://registry.npmjs.org/stringify-package/-/stringify-package-1.0.1.tgz#e5aa3643e7f74d0f28628b72f3dad5cecfc3ba85" integrity sha512-sa4DUQsYciMP1xhKWGuFM04fB0LG/9DlluZoSVywUMRNvzid6XucHK0/90xGxRoHrAaROrcHK1aPKaijCtSrhg== -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -16644,13 +16635,6 @@ strip-ansi@^3.0.1: dependencies: ansi-regex "^2.0.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -17648,7 +17632,7 @@ workerpool@^6.5.1: resolved "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz#060f73b39d0caf97c6db64da004cd01b4c099544" integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -17666,15 +17650,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 3141414e826f0ec2d8618ea048fa24dac7d8ed26 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 2 Oct 2024 13:38:39 -0400 Subject: [PATCH 18/68] change name to rollbackBufferDays --- .../@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts | 13 ++++++++++--- .../cli-integ-tests/garbage-collection.integtest.ts | 6 +++--- packages/aws-cdk/lib/cdk-toolkit.ts | 4 ++-- packages/aws-cdk/lib/cli.ts | 4 ++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 1b4eea2680334..0ed4aafd38cb8 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -313,16 +313,23 @@ export interface CdkModernBootstrapCommandOptions extends CommonCdkBootstrapComm export interface CdkGarbageCollectionCommandOptions { /** + * The amount of days an asset should stay isolated before deletion, to + * guard against some pipeline rollback scenarios + * * @default 0 */ - readonly days?: number; + readonly rollbackBufferDays?: number; /** + * The type of asset that is getting garbage collected. + * * @default 'all' */ readonly type?: 'ecr' | 's3' | 'all'; /** + * The name of the bootstrap stack + * * @default 'CdkToolkit' */ readonly bootstrapStackName?: string; @@ -463,8 +470,8 @@ export class TestFixture extends ShellHelper { public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { const args = ['gc', '--unstable=gc']; // TODO: remove when stabilizing - if (options.days) { - args.push('--days', String(options.days)); + if (options.rollbackBufferDays) { + args.push('--days', String(options.rollbackBufferDays)); } if (options.type) { args.push('--type', options.type); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index 16bbeabb20fd1..753511692b6ae 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -35,7 +35,7 @@ integTest( }); await fixture.cdkGarbageCollect({ - days: 0, + rollbackBufferDays: 0, type: 's3', bootstrapStackName: toolkitStackName, }); @@ -72,7 +72,7 @@ integTest( fixture.log('Setup complete!'); await fixture.cdkGarbageCollect({ - days: 0, + rollbackBufferDays: 0, type: 's3', bootstrapStackName: toolkitStackName, }); @@ -128,7 +128,7 @@ integTest( }); await fixture.cdkGarbageCollect({ - days: 100, // this will ensure that we do not delete assets immediately (and just tag them) + rollbackBufferDays: 100, // this will ensure that we do not delete assets immediately (and just tag them) type: 's3', bootstrapStackName: toolkitStackName, }); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 1273883178d7f..f8adf6950a6f7 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -751,7 +751,7 @@ export class CdkToolkit { sdkProvider: this.props.sdkProvider, resolvedEnvironment: environment, bootstrapStackName: options.bootstrapStackName, - isolationDays: options.days, + isolationDays: options.rollbackBufferDays, dryRun: options.dryRun ?? false, tagOnly: options.tagOnly ?? false, type: options.type ?? 'all', @@ -1464,7 +1464,7 @@ export interface GarbageCollectionOptions { * * @default 0 */ - readonly days: number; + readonly rollbackBufferDays: number; /** * The stack name of the bootstrap stack. diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 8ca87c3cea495..d066f05ae85af 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -119,7 +119,7 @@ async function parseCommandLineArguments(args: string[]) { .option('dry-run', { type: 'boolean', desc: 'List assets instead of garbage collecting them', default: false }) .option('tag-only', { type: 'boolean', desc: 'Tag assets as isolated without deleting them', default: false }) .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) - .option('days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) + .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }), ) @@ -663,7 +663,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Wed, 2 Oct 2024 13:44:47 -0400 Subject: [PATCH 19/68] buncha renames --- packages/aws-cdk/lib/api/garbage-collector.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 498be84652840..b1260ddd10cb7 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -10,9 +10,9 @@ import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from './toolkit-info'; // eslint-disable-next-line @typescript-eslint/no-require-imports const pLimit: typeof import('p-limit') = require('p-limit'); -const ISOLATED_TAG = 'awscdk.isolated'; +const ISOLATED_TAG = 'aws-cdk:isolated'; -class ActiveAssets { +class ActiveAssetCache { private readonly stacks: Set = new Set(); public rememberStack(stackTemplate: string) { @@ -30,31 +30,31 @@ class ActiveAssets { } class S3Asset { - private tags: S3.TagSet | undefined = undefined; + private cached_tags: S3.TagSet | undefined = undefined; public constructor(private readonly bucket: string, public readonly key: string) {} - public getHash(): string { + public hash(): string { return path.parse(this.key).name; } - public async getAllTags(s3: S3) { - if (this.tags) { - return this.tags; + public async allTags(s3: S3) { + if (this.cached_tags) { + return this.cached_tags; } const response = await s3.getObjectTagging({ Bucket: this.bucket, Key: this.key }).promise(); - this.tags = response.TagSet; + this.cached_tags = response.TagSet; return response.TagSet; } public async getTag(s3: S3, tag: string) { - const tags = await this.getAllTags(s3); + const tags = await this.allTags(s3); return tags.find(t => t.Key === tag)?.Value; } public async hasTag(s3: S3, tag: string) { - const tags = await this.getAllTags(s3); + const tags = await this.allTags(s3); return tags.some(t => t.Key === tag); } } @@ -145,9 +145,9 @@ export class GarbageCollector { const cfn = sdk.cloudFormation(); const s3 = sdk.s3(); - const qualifier = await this.getBootstrapQualifier(sdk, this.bootstrapStackName); + const qualifier = await this.bootstrapQualifier(sdk, this.bootstrapStackName); - const activeAssets = new ActiveAssets(); + const activeAssets = new ActiveAssetCache(); const refreshStacks = async () => { const stacks = await this.fetchAllStackTemplates(cfn, qualifier); @@ -163,7 +163,7 @@ export class GarbageCollector { // Grab stack templates first await refreshStacks(); - const bucket = await this.getBootstrapBucketName(sdk, this.bootstrapStackName); + const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); // Process objects in batches of 1000 // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. @@ -172,7 +172,7 @@ export class GarbageCollector { const currentTime = Date.now(); const graceDays = this.props.isolationDays; const isolated = batch.filter((obj) => { - return !activeAssets.contains(obj.getHash()); + return !activeAssets.contains(obj.hash()); }); let deletables: S3Asset[] = []; @@ -270,12 +270,12 @@ export class GarbageCollector { } } - private async getBootstrapBucketName(sdk: ISDK, bootstrapStackName: string): Promise { + private async bootstrapBucketName(sdk: ISDK, bootstrapStackName: string): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); return info.bucketName; } - private async getBootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { + private async bootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); return info.bootstrapStack.parameters.Qualifier; } From 778fff07e5eceb51a9bc2383dbc572ae673a35f6 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 2 Oct 2024 14:48:26 -0400 Subject: [PATCH 20/68] consolidate api options --- .../cli-integ/lib/with-cdk-app.ts | 2 +- packages/aws-cdk/lib/api/garbage-collector.ts | 35 +++++++------------ packages/aws-cdk/lib/cdk-toolkit.ts | 20 +++-------- packages/aws-cdk/lib/cli.ts | 6 ++-- 4 files changed, 21 insertions(+), 42 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 3cc7c1b353e0f..63e181410c9c3 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -486,7 +486,7 @@ export class TestFixture extends ShellHelper { public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { const args = ['gc', '--unstable=gc']; // TODO: remove when stabilizing if (options.rollbackBufferDays) { - args.push('--days', String(options.rollbackBufferDays)); + args.push('--rollback-buffer-days', String(options.rollbackBufferDays)); } if (options.type) { args.push('--type', options.type); diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index b1260ddd10cb7..0206fdc271d0d 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -1,4 +1,3 @@ -import * as path from 'path'; import * as cxapi from '@aws-cdk/cx-api'; import { CloudFormation, S3 } from 'aws-sdk'; import * as chalk from 'chalk'; @@ -34,8 +33,8 @@ class S3Asset { public constructor(private readonly bucket: string, public readonly key: string) {} - public hash(): string { - return path.parse(this.key).name; + public fileName(): string { + return this.key.split('.')[0]; } public async allTags(s3: S3) { @@ -64,20 +63,12 @@ class S3Asset { */ interface GarbageCollectorProps { /** - * If this property is set, then instead of garbage collecting, we will - * print the isolated asset hashes. + * The action to perform. Specify this if you want to perform a truncated set + * of actions available. * - * @default false + * @default 'full' */ - readonly dryRun: boolean; - - /** - * If this property is set, then we will tag assets as isolated but skip - * the actual asset deletion process. - * - * @default false - */ - readonly tagOnly: boolean; + readonly action: 'print' | 'tag' | 'delete-tagged' | 'full'; /** * The type of asset to garbage collect. @@ -156,13 +147,13 @@ export class GarbageCollector { } }; + // Grab stack templates first + await refreshStacks(); + // Refresh stacks every 5 minutes const timer = setInterval(refreshStacks, 30_000); try { - // Grab stack templates first - await refreshStacks(); - const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); // Process objects in batches of 1000 // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario @@ -172,13 +163,13 @@ export class GarbageCollector { const currentTime = Date.now(); const graceDays = this.props.isolationDays; const isolated = batch.filter((obj) => { - return !activeAssets.contains(obj.hash()); + return !activeAssets.contains(obj.fileName()); }); let deletables: S3Asset[] = []; // no items are deletable if tagOnly is set - if (this.props.tagOnly) { + if (this.props.action == 'tag') { deletables = graceDays > 0 ? await Promise.all( isolated.map(async (obj) => { @@ -204,8 +195,8 @@ export class GarbageCollector { print(chalk.blue(`${deletables.length} deletable assets`)); print(chalk.white(`${taggables.length} taggable assets`)); - if (!this.props.dryRun) { - if (!this.props.tagOnly && deletables.length > 0) { + if (this.props.action != 'print') { + if (deletables.length > 0) { await this.parallelDelete(s3, bucket, deletables); } if (taggables.length > 0) { diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 578a3fd9dba0a..c416ee0c64709 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -796,8 +796,7 @@ export class CdkToolkit { resolvedEnvironment: environment, bootstrapStackName: options.bootstrapStackName, isolationDays: options.rollbackBufferDays, - dryRun: options.dryRun ?? false, - tagOnly: options.tagOnly ?? false, + action: options.action ?? 'full', type: options.type ?? 'all', }); await gc.garbageCollect(); @@ -1523,27 +1522,18 @@ export interface DestroyOptions { */ export interface GarbageCollectionOptions { /** - * Whether to perform a dry run or not. - * A dry run means that isolated objects are printed to stdout - * but not tagged or deleted + * The action to perform. * - * @default false - */ - readonly dryRun?: boolean; - - /** - * Whether to only perform tagging. If this is set, no deletion happens - * - * @default false + * @default 'full' */ - readonly tagOnly?: boolean; + readonly action: 'print' | 'tag' | 'delete-tagged' | 'full'; /** * The type of the assets to be garbage collected. * * @default 'all' */ - readonly type: 'ecr' | 's3' | 'all'; + readonly type: 's3' | 'ecr' | 'all'; /** * Elapsed time between an asset being marked as isolated and actually deleted. diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 6d84490eaa0e0..bfc7f0ee84202 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -116,8 +116,7 @@ async function parseCommandLineArguments(args: string[]) { ) .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs .option('unstable', { type: 'array', desc: 'Opt in to specific unstable features. Can be specified multiple times.', default: [] }) - .option('dry-run', { type: 'boolean', desc: 'List assets instead of garbage collecting them', default: false }) - .option('tag-only', { type: 'boolean', desc: 'Tag assets as isolated without deleting them', default: false }) + .option('action', { type: 'string', desc: 'The action (or sub-action) you want to perform. Valid entires are "print", "tag", "delete-tagged", "full".', default: 'full' }) .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) @@ -691,8 +690,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Wed, 2 Oct 2024 14:56:28 -0400 Subject: [PATCH 21/68] faciliate actions better --- packages/aws-cdk/lib/api/garbage-collector.ts | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 0206fdc271d0d..a9ea91592f6f5 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -10,6 +10,7 @@ import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from './toolkit-info'; const pLimit: typeof import('p-limit') = require('p-limit'); const ISOLATED_TAG = 'aws-cdk:isolated'; +const P_LIMIT = 50; class ActiveAssetCache { private readonly stacks: Set = new Set(); @@ -113,11 +114,17 @@ interface GarbageCollectorProps { export class GarbageCollector { private garbageCollectS3Assets: boolean; private garbageCollectEcrAssets: boolean; + private permissionToDelete: boolean; + private permissionToTag: boolean; private bootstrapStackName: string; public constructor(readonly props: GarbageCollectorProps) { - this.garbageCollectS3Assets = props.type === 's3' || props.type === 'all'; - this.garbageCollectEcrAssets = props.type === 'ecr' || props.type === 'all'; + this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); + this.garbageCollectEcrAssets = ['ecr', 'all'].includes(props.type); + + this.permissionToDelete = ['delete-tagged', 'full'].includes(props.type); + this.permissionToTag = ['tag', 'full'].includes(props.type); + this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; // TODO: ECR garbage collection @@ -168,8 +175,7 @@ export class GarbageCollector { let deletables: S3Asset[] = []; - // no items are deletable if tagOnly is set - if (this.props.action == 'tag') { + if (this.permissionToDelete) { deletables = graceDays > 0 ? await Promise.all( isolated.map(async (obj) => { @@ -183,25 +189,28 @@ export class GarbageCollector { : isolated; } - const taggables: S3Asset[] = graceDays > 0 - ? await Promise.all( - isolated.map(async (obj) => { - const hasTag = await obj.hasTag(s3, ISOLATED_TAG); - return hasTag ? null : obj; - }), - ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) - : []; + let taggables: S3Asset[] = []; + + if (this.permissionToTag) { + taggables = graceDays > 0 + ? await Promise.all( + isolated.map(async (obj) => { + const hasTag = await obj.hasTag(s3, ISOLATED_TAG); + return hasTag ? null : obj; + }), + ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) + : []; + } print(chalk.blue(`${deletables.length} deletable assets`)); print(chalk.white(`${taggables.length} taggable assets`)); - if (this.props.action != 'print') { - if (deletables.length > 0) { - await this.parallelDelete(s3, bucket, deletables); - } - if (taggables.length > 0) { - await this.parallelTag(s3, bucket, taggables); - } + if (deletables.length > 0) { + await this.parallelDelete(s3, bucket, deletables); + } + + if (taggables.length > 0) { + await this.parallelTag(s3, bucket, taggables, currentTime); } // TODO: maybe undelete @@ -215,8 +224,8 @@ export class GarbageCollector { * Tag objects in parallel using p-limit. The putObjectTagging API does not * support batch tagging so we must handle the parallelism client-side. */ - private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[]) { - const limit = pLimit(5); + private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[], date: number) { + const limit = pLimit(P_LIMIT); for (const obj of taggables) { await limit(() => @@ -227,7 +236,7 @@ export class GarbageCollector { TagSet: [ { Key: ISOLATED_TAG, - Value: String(Date.now()), + Value: String(date), }, ], }, From 73d8eda290d70715235c8f9dcd2e4001a736534c Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 2 Oct 2024 17:15:50 -0400 Subject: [PATCH 22/68] fix refresh stacks --- packages/aws-cdk/lib/api/garbage-collector.ts | 72 ++++++++++++++----- packages/aws-cdk/lib/cdk-toolkit.ts | 2 +- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index a9ea91592f6f5..a97bb330d5390 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -32,7 +32,7 @@ class ActiveAssetCache { class S3Asset { private cached_tags: S3.TagSet | undefined = undefined; - public constructor(private readonly bucket: string, public readonly key: string) {} + public constructor(private readonly bucket: string, public readonly key: string, public readonly size: number) {} public fileName(): string { return this.key.split('.')[0]; @@ -83,7 +83,7 @@ interface GarbageCollectorProps { * * @default 0 */ - readonly isolationDays: number; + readonly rollbackBufferDays: number; /** * The environment to deploy this stack in @@ -122,8 +122,10 @@ export class GarbageCollector { this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); this.garbageCollectEcrAssets = ['ecr', 'all'].includes(props.type); - this.permissionToDelete = ['delete-tagged', 'full'].includes(props.type); - this.permissionToTag = ['tag', 'full'].includes(props.type); + this.permissionToDelete = ['delete-tagged', 'full'].includes(props.action); + this.permissionToTag = ['tag', 'full'].includes(props.action); + + print(chalk.white(this.permissionToDelete, this.permissionToTag, props.action)); this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; @@ -147,32 +149,57 @@ export class GarbageCollector { const activeAssets = new ActiveAssetCache(); - const refreshStacks = async () => { - const stacks = await this.fetchAllStackTemplates(cfn, qualifier); - for (const stack of stacks) { - activeAssets.rememberStack(stack); + let refreshStacksRunning = false; + const refreshStacks = async (isInitial?: boolean) => { + if (refreshStacksRunning) { + return; + } + + refreshStacksRunning = true; + + try { + const stacks = await this.fetchAllStackTemplates(cfn, qualifier); + for (const stack of stacks) { + activeAssets.rememberStack(stack); + } + } catch (err) { + throw new Error(`Error refreshing stacks: ${err}`); + } finally { + refreshStacksRunning = false; + + if (!isInitial) { + setTimeout(refreshStacks, 300_000); + } } }; // Grab stack templates first - await refreshStacks(); - - // Refresh stacks every 5 minutes - const timer = setInterval(refreshStacks, 30_000); + await refreshStacks(true); + // Refresh stacks in the background + const timeout = setTimeout(refreshStacks, 300_000); try { const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); + const numObjects = await this.numObjectsInBucket(s3, bucket); + const batches = 1; + const batchSize = 1000; + const currentTime = Date.now(); + const graceDays = this.props.rollbackBufferDays; + + print(chalk.white(`Parsing through ${numObjects} in batches`)); + // Process objects in batches of 1000 // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. - for await (const batch of this.readBucketInBatches(s3, bucket)) { - print(chalk.red(batch.length)); - const currentTime = Date.now(); - const graceDays = this.props.isolationDays; + for await (const batch of this.readBucketInBatches(s3, bucket, batchSize)) { + print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); + const isolated = batch.filter((obj) => { return !activeAssets.contains(obj.fileName()); }); + print(chalk.blue(`${isolated.length} isolated assets`)); + let deletables: S3Asset[] = []; if (this.permissionToDelete) { @@ -215,8 +242,10 @@ export class GarbageCollector { // TODO: maybe undelete } + } catch (err: any) { + throw new Error(err); } finally { - clearInterval(timer); + clearTimeout(timeout); } } @@ -280,6 +309,11 @@ export class GarbageCollector { return info.bootstrapStack.parameters.Qualifier; } + private async numObjectsInBucket(s3: S3, bucket: string): Promise { + const response = await s3.listObjectsV2({ Bucket: bucket }).promise(); + return response.KeyCount ?? 0; + } + /** * Generator function that reads objects from the S3 Bucket in batches. */ @@ -297,8 +331,10 @@ export class GarbageCollector { }).promise(); response.Contents?.forEach((obj) => { + const key = obj.Key ?? ''; + const size = obj.Size ?? 0; if (obj.Key) { - batch.push(new S3Asset(bucket, obj.Key)); + batch.push(new S3Asset(bucket, key, size)); } }); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index c416ee0c64709..b92307113a818 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -795,7 +795,7 @@ export class CdkToolkit { sdkProvider: this.props.sdkProvider, resolvedEnvironment: environment, bootstrapStackName: options.bootstrapStackName, - isolationDays: options.rollbackBufferDays, + rollbackBufferDays: options.rollbackBufferDays, action: options.action ?? 'full', type: options.type ?? 'all', }); From 8e936a0b61d33737d4cd51ad56b4c12977743c70 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 3 Oct 2024 10:43:03 -0400 Subject: [PATCH 23/68] parallel get tags --- packages/aws-cdk/lib/api/garbage-collector.ts | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index a97bb330d5390..f1d298c00330f 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -11,6 +11,7 @@ const pLimit: typeof import('p-limit') = require('p-limit'); const ISOLATED_TAG = 'aws-cdk:isolated'; const P_LIMIT = 50; +const DAY = 24 * 60 * 60 * 1000; // Number of milliseconds in a day class ActiveAssetCache { private readonly stacks: Set = new Set(); @@ -48,15 +49,27 @@ class S3Asset { return response.TagSet; } - public async getTag(s3: S3, tag: string) { + private async getTag(s3: S3, tag: string) { const tags = await this.allTags(s3); return tags.find(t => t.Key === tag)?.Value; } - public async hasTag(s3: S3, tag: string) { + private async hasTag(s3: S3, tag: string) { const tags = await this.allTags(s3); return tags.some(t => t.Key === tag); } + + public async noIsolatedTag(s3: S3) { + return !(await this.hasTag(s3, ISOLATED_TAG)); + } + + public async isolatedTagBefore(s3: S3, date: Date) { + const tagValue = await this.getTag(s3, ISOLATED_TAG); + if (!tagValue) { + return false; + } + return new Date(tagValue) < date; + } } /** @@ -201,42 +214,22 @@ export class GarbageCollector { print(chalk.blue(`${isolated.length} isolated assets`)); let deletables: S3Asset[] = []; - - if (this.permissionToDelete) { - deletables = graceDays > 0 - ? await Promise.all( - isolated.map(async (obj) => { - const tagTime = await obj.getTag(s3, ISOLATED_TAG); - if (tagTime && olderThan(Number(tagTime), currentTime, graceDays)) { - return obj; - } - return null; - }), - ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) - : isolated; - } - let taggables: S3Asset[] = []; - if (this.permissionToTag) { - taggables = graceDays > 0 - ? await Promise.all( - isolated.map(async (obj) => { - const hasTag = await obj.hasTag(s3, ISOLATED_TAG); - return hasTag ? null : obj; - }), - ).then(results => results.filter((obj): obj is S3Asset => obj !== null)) - : []; + if (graceDays > 0) { + await this.parallelReadAllTags(s3, isolated); + deletables = isolated.filter((obj) => obj.isolatedTagBefore(s3, new Date(currentTime - (graceDays * DAY)))); + taggables = isolated.filter((obj) => obj.noIsolatedTag(s3)); } print(chalk.blue(`${deletables.length} deletable assets`)); print(chalk.white(`${taggables.length} taggable assets`)); - if (deletables.length > 0) { + if (this.permissionToDelete && deletables.length > 0) { await this.parallelDelete(s3, bucket, deletables); } - if (taggables.length > 0) { + if (this.permissionToTag && taggables.length > 0) { await this.parallelTag(s3, bucket, taggables, currentTime); } @@ -249,6 +242,14 @@ export class GarbageCollector { } } + private async parallelReadAllTags(s3: S3, objects: S3Asset[]) { + const limit = pLimit(P_LIMIT); + + for (const obj of objects) { + await limit(() => obj.allTags(s3)); + } + } + /** * Tag objects in parallel using p-limit. The putObjectTagging API does not * support batch tagging so we must handle the parallelism client-side. @@ -421,9 +422,3 @@ async function paginateSdkCall(cb: (nextToken?: string) => Promise= graceDays; -} From 232f1869e07cd67361436e1c0a068bda36aa1c4d Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 3 Oct 2024 10:51:06 -0400 Subject: [PATCH 24/68] remove parallelism --- packages/aws-cdk/lib/cdk-toolkit.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index b92307113a818..b96b9c91270ba 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -789,7 +789,7 @@ export class CdkToolkit { public async garbageCollect(userEnvironmentSpecs: string[], options: GarbageCollectionOptions) { const environments = await this.defineEnvironments(userEnvironmentSpecs); - await Promise.all(environments.map(async (environment) => { + for (const environment of environments) { success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); const gc = new GarbageCollector({ sdkProvider: this.props.sdkProvider, @@ -800,7 +800,7 @@ export class CdkToolkit { type: options.type ?? 'all', }); await gc.garbageCollect(); - })); + }; } private async defineEnvironments(userEnvironmentSpecs: string[]): Promise { From 7ff4273ab89c1579898bb9d0ca44df0b1f36b224 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 3 Oct 2024 14:02:36 -0400 Subject: [PATCH 25/68] small change --- packages/aws-cdk/lib/api/garbage-collector.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index f1d298c00330f..a2ad2751f6dbc 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -328,7 +328,6 @@ export class GarbageCollector { const response = await s3.listObjectsV2({ Bucket: bucket, ContinuationToken: continuationToken, - MaxKeys: batchSize - batch.length, }).promise(); response.Contents?.forEach((obj) => { From 9f7341ec998f70e9b4d3c625932940a55ac7c012 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 3 Oct 2024 15:02:19 -0400 Subject: [PATCH 26/68] readme --- packages/aws-cdk/README.md | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 227a040cd5fb2..4554e2c8e12fc 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -25,6 +25,7 @@ The AWS CDK Toolkit provides the `cdk` command-line interface that can be used t | [`cdk watch`](#cdk-watch) | Watches a CDK app for deployable and hotswappable changes | | [`cdk destroy`](#cdk-destroy) | Deletes a stack from an AWS account | | [`cdk bootstrap`](#cdk-bootstrap) | Deploy a toolkit stack to support deploying large stacks & artifacts | +| [`cdk gc`](#cdk-gc) | Garbage collect assets associated with the bootstrapped stack | | [`cdk doctor`](#cdk-doctor) | Inspect the environment and produce information useful for troubleshooting | | [`cdk acknowledge`](#cdk-acknowledge) | Acknowledge (and hide) a notice by issue number | | [`cdk notices`](#cdk-notices) | List all relevant notices for the application | @@ -878,6 +879,50 @@ In order to remove that permissions boundary you have to specify the cdk bootstrap --no-previous-parameters ``` +### `cdk gc` + +CDK Garbage Collection. + +> [!CAUTION] +> CDK Garbage Collection is under development and therefore must be opted in via the `--unstable` flag: `cdk gc --unstable=gc`. + +> [!WARNING] +> `cdk gc` currently only supports garbage collecting S3 Assets. You must specify `cdk gc --unstable=gc --type=s3` as ECR asset garbage collection has not yet been implemented. + +`cdk gc` garbage collects isolated S3 assets from your bootstrap bucket via the following mechanism: +- for each object in the bootstrap S3 Bucket, check to see if it is referenced in any existing CloudFormation templates +- if not, it is treated as isolated and either tagged or deleted, depending on your configuration. + +The most basic usage looks like this: + +```console +cdk gc --unstable=gc --type=s3 +``` + +This will garbage collect S3 assets from the current bootstrapped environment(s) and immediately delete them. Note that, since the default bootstrap S3 Bucket is versioned, object deletion will be handled by the lifecycle +policy on the bucket. + +If you are concerned about deleting assets too aggressively, you can configure a buffer amount of days to keep +an isolated asset before deletion. In this scenario, instead of deleting isolated objects, `cdk gc` will tag +them with today's date instead. It will also check if any objects have been tagged by previous runs of `cdk gc` +and delete them if they have been tagged for longer than the buffer days. + +```console +cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 +``` + +You can also configure the scope that `cdk gc` performs via the `--action` option. By default, all actions +are performed, but you can specify `print`, `tag`, or `delete-tagged`. +- `print` performs no changes to your AWS account, but finds and prints the number of isolated assets. +- `tag` tags any newly isolated assets, but does not delete any isolated assets. +- `delete-tagged` deletes assets that have been tagged for longer than the buffer days, but does not tag newly isolated assets. + +```console +cdk gc --unstable=gc --type=s3 --action=delete-tagged --rollback-buffer-days=30 +``` + +This will delete assets that have been isolated for >30 days, but will not tag additional assets. + ### `cdk doctor` Inspect the current command-line environment and configurations, and collect information that can be useful for From eb204b3f6973c8979b2bd452320be90ded8e8c95 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 3 Oct 2024 18:08:03 -0400 Subject: [PATCH 27/68] stupid stupid stupid --- packages/aws-cdk/lib/api/garbage-collector.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index a2ad2751f6dbc..b525b64ab1302 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -65,9 +65,12 @@ class S3Asset { public async isolatedTagBefore(s3: S3, date: Date) { const tagValue = await this.getTag(s3, ISOLATED_TAG); - if (!tagValue) { + print(chalk.red('a', tagValue, 'b', date)); + if (!tagValue || tagValue == '') { + print(chalk.red('no tag')); return false; } + print(chalk.red('tag', typeof(tagValue))); return new Date(tagValue) < date; } } @@ -199,7 +202,7 @@ export class GarbageCollector { const currentTime = Date.now(); const graceDays = this.props.rollbackBufferDays; - print(chalk.white(`Parsing through ${numObjects} in batches`)); + print(chalk.white(`Parsing through ${numObjects} objects in batches`)); // Process objects in batches of 1000 // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario @@ -213,13 +216,21 @@ export class GarbageCollector { print(chalk.blue(`${isolated.length} isolated assets`)); - let deletables: S3Asset[] = []; + let deletables: S3Asset[] = isolated; let taggables: S3Asset[] = []; if (graceDays > 0) { + print(chalk.white(`Filtering out assets that are not old enough to delete`)); await this.parallelReadAllTags(s3, isolated); - deletables = isolated.filter((obj) => obj.isolatedTagBefore(s3, new Date(currentTime - (graceDays * DAY)))); - taggables = isolated.filter((obj) => obj.noIsolatedTag(s3)); + deletables = await Promise.all(isolated.map(async (obj) => { + const shouldDelete = await obj.isolatedTagBefore(s3, new Date(currentTime - (graceDays * DAY))); + return shouldDelete ? obj : null; + })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); + + taggables = await Promise.all(isolated.map(async (obj) => { + const shouldTag = await obj.noIsolatedTag(s3); + return shouldTag ? obj : null; + })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); } print(chalk.blue(`${deletables.length} deletable assets`)); From 22507681d3f75bb8f9120a6b11749c62fe023604 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 7 Oct 2024 17:56:01 -0400 Subject: [PATCH 28/68] review in progress means waiting --- packages/aws-cdk/lib/api/garbage-collector.ts | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index b525b64ab1302..484b766e96fd4 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -372,18 +372,21 @@ export class GarbageCollector { private async fetchAllStackTemplates(cfn: CloudFormation, qualifier: string) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { - const response = await cfn.listStacks({ NextToken: nextToken }).promise(); + let response = await cfn.listStacks({ NextToken: nextToken }).promise(); // We cannot operate on REVIEW_IN_PROGRESS stacks because we do not know what the template looks like in this case + // If we encounter this status, we will wait up to a minute for it to land before erroring out. const reviewInProgressStacks = response.StackSummaries?.filter(s => s.StackStatus === 'REVIEW_IN_PROGRESS') ?? []; if (reviewInProgressStacks.length > 0) { - throw new Error(`Stacks in REVIEW_IN_PROGRESS state are not allowed: ${reviewInProgressStacks.map(s => s.StackName).join(', ')}`); + const updatedRequest = await this.waitForStacksAndRefetch(cfn, response, reviewInProgressStacks); + response = await updatedRequest.promise(); } // Deleted stacks are ignored + const ignoredStatues = ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_IN_PROGRESS', 'DELETE_FAILED']; stackNames.push( ...(response.StackSummaries ?? []) - .filter(s => s.StackStatus !== 'DELETE_COMPLETE' && s.StackStatus !== 'DELETE_IN_PROGRESS') + .filter(s => !ignoredStatues.includes(s.StackStatus)) .map(s => s.StackId ?? s.StackName), ); @@ -420,6 +423,41 @@ export class GarbageCollector { return templates; } + + private async waitForStacksAndRefetch( + cfn: CloudFormation, + originalResponse: AWS.CloudFormation.ListStacksOutput, + reviewInProgressStacks: AWS.CloudFormation.StackSummary[] + ): Promise> { + const maxWaitTime = 60000; + const startTime = Date.now(); + + while (Date.now() - startTime < maxWaitTime) { + let allStacksUpdated = true; + + for (const stack of reviewInProgressStacks) { + const response = await cfn.describeStacks({ StackName: stack.StackId ?? stack.StackName }).promise(); + const currentStatus = response.Stacks?.[0]?.StackStatus; + + if (currentStatus === 'REVIEW_IN_PROGRESS') { + allStacksUpdated = false; + break; + } + } + + if (allStacksUpdated) { + // All stacks have left REVIEW_IN_PROGRESS state, refetch the list + return cfn.listStacks({ NextToken: originalResponse.NextToken }); + } + + // Wait for 15 seconds before checking again + await new Promise(resolve => setTimeout(resolve, 15000)); + } + + // If we've reached this point, some stacks are still in REVIEW_IN_PROGRESS after waiting + const remainingStacks = reviewInProgressStacks.map(s => s.StackName).join(', '); + throw new Error(`Stacks still in REVIEW_IN_PROGRESS state after waiting for 1 minute: ${remainingStacks}`); + } } async function paginateSdkCall(cb: (nextToken?: string) => Promise) { From f9428708e187e5761bdbe392ed96dbe119dbd8e0 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 7 Oct 2024 17:56:10 -0400 Subject: [PATCH 29/68] unit tests --- .../test/api/garbage-collection.test.ts | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 packages/aws-cdk/test/api/garbage-collection.test.ts diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts new file mode 100644 index 0000000000000..7867e6975dcfd --- /dev/null +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -0,0 +1,186 @@ +/* eslint-disable import/order */ + +const mockGarbageCollect = jest.fn(); + +import { GarbageCollector, ToolkitInfo } from '../../lib/api'; +import { mockBootstrapStack, MockSdk, MockSdkProvider } from '../util/mock-sdk'; + +let garbageCollector: GarbageCollector; +let mockListStacks: (params: AWS.CloudFormation.Types.ListStacksInput) => AWS.CloudFormation.Types.ListStacksOutput; +let mockGetTemplateSummary: (params: AWS.CloudFormation.Types.GetTemplateSummaryInput) => AWS.CloudFormation.Types.GetTemplateSummaryOutput; +let mockGetTemplate: (params: AWS.CloudFormation.Types.GetTemplateInput) => AWS.CloudFormation.Types.GetTemplateOutput; +let mockListObjectsV2: (params: AWS.S3.Types.ListObjectsV2Request) => AWS.S3.Types.ListObjectsV2Output; +let mockGetObjectTagging: (params: AWS.S3.Types.GetObjectTaggingRequest) => AWS.S3.Types.GetObjectTaggingOutput; +let mockDeleteObjects: (params: AWS.S3.Types.DeleteObjectsRequest) => AWS.S3.Types.DeleteObjectsOutput; +let mockPutObjectTagging: (params: AWS.S3.Types.PutObjectTaggingRequest) => AWS.S3.Types.PutObjectTaggingOutput; + +let stderrMock: jest.SpyInstance; +let sdk: MockSdkProvider; + +function mockTheToolkitInfo(stackProps: Partial) { + const sdk = new MockSdk(); + (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.fromStack(mockBootstrapStack(sdk, stackProps))); +} + +beforeEach(() => { + sdk = new MockSdkProvider({ realSdk: false }); + // By default, we'll return a non-found toolkit info + (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.bootstrapStackNotFoundInfo('GarbageStack')); + + stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); +}); + +afterEach(() => { + stderrMock.mockRestore(); +}); + +describe('Garbage Collection', () => { + beforeEach(() => { + mockListStacks = jest.fn().mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE'}, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE'}, + ] + }); + mockGetTemplateSummary = jest.fn().mockReturnValue({}); + mockGetTemplate = jest.fn().mockReturnValue({ + TemplateBody: 'abcde', + }); + mockListObjectsV2 = jest.fn().mockImplementation(() => { + return Promise.resolve({ + Contents: [ + { Key: 'asset1' }, + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + KeyCount: 3, + }); + }); + mockGetObjectTagging = jest.fn().mockImplementation((params) => { + return Promise.resolve({ + TagSet: params.Key === 'asset2' ? [{ Key: "ISOLATED_TAG", Value: new Date().toISOString() }] : [] + }); + }); + mockPutObjectTagging = jest.fn(); + mockDeleteObjects = jest.fn(); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + }); + sdk.stubS3({ + listObjectsV2: mockListObjectsV2, + getObjectTagging: mockGetObjectTagging, + deleteObjects: mockDeleteObjects, + putObjectTagging: mockPutObjectTagging, + }); + }); + + afterEach(() => { + mockGarbageCollect.mockClear(); + }); + + test('rollbackBufferDays = 0 -- assets to be deleted', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 0, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + // no tagging + expect(mockGetObjectTagging).toHaveBeenCalledTimes(0); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + + // assets are to be deleted + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: "BUCKET_NAME", + Delete: { + Objects: [ + { Key: 'asset1' }, + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + Quiet: true, + }, + }); + }); + + test('rollbackBufferDays > 0 -- assets to be tagged', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // assets tagged + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(3); + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }); + + test('type = ecr -- throws error', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + expect(() => new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 'ecr', + })).toThrow(/ECR garbage collection is not yet supported/); + }); +}); From 18376d81a03feb197290139485c2fa96ca789ec3 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 7 Oct 2024 18:02:49 -0400 Subject: [PATCH 30/68] more unit tests --- .../test/api/garbage-collection.test.ts | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index 7867e6975dcfd..26d4e4f6a84f1 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -183,4 +183,106 @@ describe('Garbage Collection', () => { type: 'ecr', })).toThrow(/ECR garbage collection is not yet supported/); }); + + test('action = print -- does not tag or delete', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'print', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // get tags, but dont put tags + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }); + + test('action = tag -- does not delete', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'tag', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // tags objects + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(3); + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }); + + test('action = delete-tagged -- does not tag', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'delete-tagged', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // get tags, but dont put tags + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + }); }); From 80440df07fbd992176b1eecbe2a200da3a5d340a Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 7 Oct 2024 18:06:52 -0400 Subject: [PATCH 31/68] robust qualifier --- packages/aws-cdk/lib/api/garbage-collector.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 484b766e96fd4..c6a1037c6aa61 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -316,7 +316,7 @@ export class GarbageCollector { return info.bucketName; } - private async bootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { + private async bootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); return info.bootstrapStack.parameters.Qualifier; } @@ -369,7 +369,7 @@ export class GarbageCollector { * understanding of what assets are being used. * - stacks in REVIEW_IN_PROGRESS stage */ - private async fetchAllStackTemplates(cfn: CloudFormation, qualifier: string) { + private async fetchAllStackTemplates(cfn: CloudFormation, qualifier?: string) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { let response = await cfn.listStacks({ NextToken: nextToken }).promise(); @@ -405,9 +405,11 @@ export class GarbageCollector { // Filter out stacks that we KNOW are using a different bootstrap qualifier // This is necessary because a stack under a different bootstrap could coincidentally reference the same hash // and cause a false negative (cause an asset to be preserved when its isolated) + // This is intentionally done in a way where we ONLY filter out stacks that are meant for a different qualifier + // because we are okay with false positives. const bootstrapVersion = summary?.Parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); - if (splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier) { + if (qualifier && splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier) { // This stack is definitely bootstrapped to a different qualifier so we can safely ignore it continue; } else { From 033fed4ccf49028b4b752a877dbf80b672b88c7f Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 7 Oct 2024 18:50:07 -0400 Subject: [PATCH 32/68] update integs --- .../tests/cli-integ-tests/garbage-collection.integtest.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index 753511692b6ae..8860eb66c177d 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -78,10 +78,10 @@ integTest( }); fixture.log('Garbage collection complete!'); - // assert that the bootstrap bucket is empty + // assert that the bootstrap bucket has the object await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) .then((result) => { - expect(result.Contents).toHaveLength(2); + expect(result.Contents).toHaveLength(1); }); await fixture.cdkDestroy('lambda', { @@ -134,10 +134,10 @@ integTest( }); fixture.log('Garbage collection complete!'); - // assert that the bootstrap bucket is empty + // assert that the bootstrap bucket has the object and is tagged await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) .then(async (result) => { - expect(result.Contents).toHaveLength(2); + expect(result.Contents).toHaveLength(2); // also the CFN template const key = result.Contents![0].Key; const tags = await fixture.aws.s3.send(new GetObjectTaggingCommand({ Bucket: bootstrapBucketName, Key: key })); expect(tags.TagSet).toHaveLength(1); From f65f09c6b1022afd2f2b1c364b3f99c0c5dfbbf7 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 8 Oct 2024 11:56:01 -0400 Subject: [PATCH 33/68] more unit tests --- packages/aws-cdk/lib/api/garbage-collector.ts | 20 ++- .../test/api/garbage-collection.test.ts | 136 ++++++++++++++++-- 2 files changed, 142 insertions(+), 14 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index c6a1037c6aa61..bd4bfea794688 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -122,6 +122,13 @@ interface GarbageCollectorProps { * @default DEFAULT_TOOLKIT_STACK_NAME */ readonly bootstrapStackName?: string; + + /** + * Max wait time for retries in milliseconds (for testing purposes). + * + * @default 60000 + */ + readonly maxWaitTime?: number; } /** @@ -133,6 +140,7 @@ export class GarbageCollector { private permissionToDelete: boolean; private permissionToTag: boolean; private bootstrapStackName: string; + private maxWaitTime: number; public constructor(readonly props: GarbageCollectorProps) { this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); @@ -140,6 +148,7 @@ export class GarbageCollector { this.permissionToDelete = ['delete-tagged', 'full'].includes(props.action); this.permissionToTag = ['tag', 'full'].includes(props.action); + this.maxWaitTime = props.maxWaitTime ?? 60000; print(chalk.white(this.permissionToDelete, this.permissionToTag, props.action)); @@ -220,13 +229,13 @@ export class GarbageCollector { let taggables: S3Asset[] = []; if (graceDays > 0) { - print(chalk.white(`Filtering out assets that are not old enough to delete`)); + print(chalk.white('Filtering out assets that are not old enough to delete')); await this.parallelReadAllTags(s3, isolated); deletables = await Promise.all(isolated.map(async (obj) => { const shouldDelete = await obj.isolatedTagBefore(s3, new Date(currentTime - (graceDays * DAY))); return shouldDelete ? obj : null; })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); - + taggables = await Promise.all(isolated.map(async (obj) => { const shouldTag = await obj.noIsolatedTag(s3); return shouldTag ? obj : null; @@ -379,7 +388,7 @@ export class GarbageCollector { const reviewInProgressStacks = response.StackSummaries?.filter(s => s.StackStatus === 'REVIEW_IN_PROGRESS') ?? []; if (reviewInProgressStacks.length > 0) { const updatedRequest = await this.waitForStacksAndRefetch(cfn, response, reviewInProgressStacks); - response = await updatedRequest.promise(); + response = await updatedRequest.promise(); } // Deleted stacks are ignored @@ -425,16 +434,15 @@ export class GarbageCollector { return templates; } - + private async waitForStacksAndRefetch( cfn: CloudFormation, originalResponse: AWS.CloudFormation.ListStacksOutput, reviewInProgressStacks: AWS.CloudFormation.StackSummary[] ): Promise> { - const maxWaitTime = 60000; const startTime = Date.now(); - while (Date.now() - startTime < maxWaitTime) { + while (Date.now() - startTime < this.maxWaitTime) { let allStacksUpdated = true; for (const stack of reviewInProgressStacks) { diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index 26d4e4f6a84f1..ff4e1894b11e2 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -7,6 +7,7 @@ import { mockBootstrapStack, MockSdk, MockSdkProvider } from '../util/mock-sdk'; let garbageCollector: GarbageCollector; let mockListStacks: (params: AWS.CloudFormation.Types.ListStacksInput) => AWS.CloudFormation.Types.ListStacksOutput; +let mockDescribeStacks: (params: AWS.CloudFormation.Types.DescribeStacksInput) => AWS.CloudFormation.Types.DescribeStacksOutput; let mockGetTemplateSummary: (params: AWS.CloudFormation.Types.GetTemplateSummaryInput) => AWS.CloudFormation.Types.GetTemplateSummaryOutput; let mockGetTemplate: (params: AWS.CloudFormation.Types.GetTemplateInput) => AWS.CloudFormation.Types.GetTemplateOutput; let mockListObjectsV2: (params: AWS.S3.Types.ListObjectsV2Request) => AWS.S3.Types.ListObjectsV2Output; @@ -18,15 +19,14 @@ let stderrMock: jest.SpyInstance; let sdk: MockSdkProvider; function mockTheToolkitInfo(stackProps: Partial) { - const sdk = new MockSdk(); - (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.fromStack(mockBootstrapStack(sdk, stackProps))); + const mockSdk = new MockSdk(); + (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.fromStack(mockBootstrapStack(mockSdk, stackProps))); } beforeEach(() => { sdk = new MockSdkProvider({ realSdk: false }); // By default, we'll return a non-found toolkit info (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.bootstrapStackNotFoundInfo('GarbageStack')); - stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); }); @@ -38,9 +38,9 @@ describe('Garbage Collection', () => { beforeEach(() => { mockListStacks = jest.fn().mockResolvedValue({ StackSummaries: [ - { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE'}, - { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE'}, - ] + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], }); mockGetTemplateSummary = jest.fn().mockReturnValue({}); mockGetTemplate = jest.fn().mockReturnValue({ @@ -58,16 +58,18 @@ describe('Garbage Collection', () => { }); mockGetObjectTagging = jest.fn().mockImplementation((params) => { return Promise.resolve({ - TagSet: params.Key === 'asset2' ? [{ Key: "ISOLATED_TAG", Value: new Date().toISOString() }] : [] + TagSet: params.Key === 'asset2' ? [{ Key: 'ISOLATED_TAG', Value: new Date().toISOString() }] : [], }); }); mockPutObjectTagging = jest.fn(); mockDeleteObjects = jest.fn(); + mockDescribeStacks = jest.fn(); sdk.stubCloudFormation({ listStacks: mockListStacks, getTemplateSummary: mockGetTemplateSummary, getTemplate: mockGetTemplate, + describeStacks: mockDescribeStacks, }); sdk.stubS3({ listObjectsV2: mockListObjectsV2, @@ -113,7 +115,7 @@ describe('Garbage Collection', () => { // assets are to be deleted expect(mockDeleteObjects).toHaveBeenCalledWith({ - Bucket: "BUCKET_NAME", + Bucket: 'BUCKET_NAME', Delete: { Objects: [ { Key: 'asset1' }, @@ -285,4 +287,122 @@ describe('Garbage Collection', () => { expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); }); + + test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + // Mock the listStacks call + const mockListStacksStatus = jest.fn() + .mockResolvedValueOnce({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }) + .mockResolvedValueOnce({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'UPDATE_COMPLETE' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + + // Mock the describeStacks call + const mockDescribeStacksStatus = jest.fn() + .mockResolvedValueOnce({ + Stacks: [{ StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }], + }) + .mockResolvedValueOnce({ + Stacks: [{ StackName: 'Stack1', StackStatus: 'UPDATE_COMPLETE' }], + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacksStatus, + describeStacks: mockDescribeStacksStatus, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + }); + + await garbageCollector.garbageCollect(); + + // describe and list are called as expected + expect(mockDescribeStacksStatus).toHaveBeenCalledTimes(2); + expect(mockListStacksStatus).toHaveBeenCalledTimes(2); + + // everything else runs as expected: + // assets tagged + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(3); + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }, 60000); + + test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + // Mock the listStacks call + const mockListStacksStatus = jest.fn() + .mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + + // Mock the describeStacks call + const mockDescribeStacksStatus = jest.fn() + .mockResolvedValue({ + Stacks: [{ StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }], + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacksStatus, + describeStacks: mockDescribeStacksStatus, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + maxWaitTime: 100, // Wait for only 100ms in tests + }); + + await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting for 1 minute/); + }, 60000); }); From 0b5884052e9e9932d0c22c452979fffe7e4d7546 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 8 Oct 2024 12:38:10 -0400 Subject: [PATCH 34/68] finish merge --- packages/aws-cdk/lib/cdk-toolkit.ts | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index e8a7c030fc4c7..eaf884ec66da3 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -772,7 +772,10 @@ export class CdkToolkit { const environments = await this.defineEnvironments(userEnvironmentSpecs); - await Promise.all(environments.map(async (environment) => { + const limit = pLimit(20); + + // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism + await Promise.all(environments.map((environment) => limit(async () => { success(' ⏳ Bootstrapping environment %s...', chalk.blue(environment.name)); try { const result = await bootstrapper.bootstrapEnvironment(environment, this.props.sdkProvider, options); @@ -784,7 +787,7 @@ export class CdkToolkit { error(' ❌ Environment %s failed bootstrapping: %s', chalk.blue(environment.name), e); throw e; } - })); + }))); } /** @@ -833,22 +836,7 @@ export class CdkToolkit { environments.push(...await globEnvironmentsFromStacks(await this.selectStacksForList([]), globSpecs, this.props.sdkProvider)); } - const limit = pLimit(20); - - // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism - await Promise.all(environments.map((environment) => limit(async () => { - success(' ⏳ Bootstrapping environment %s...', chalk.blue(environment.name)); - try { - const result = await bootstrapper.bootstrapEnvironment(environment, this.props.sdkProvider, options); - const message = result.noOp - ? ' ✅ Environment %s bootstrapped (no changes).' - : ' ✅ Environment %s bootstrapped.'; - success(message, chalk.blue(environment.name)); - } catch (e) { - error(' ❌ Environment %s failed bootstrapping: %s', chalk.blue(environment.name), e); - throw e; - } - }))); + return environments; } /** From 607fa3330d2a028a3f6655cc12c734c390143228 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 8 Oct 2024 12:42:24 -0400 Subject: [PATCH 35/68] remove dup dep --- packages/aws-cdk/package.json | 1 - yarn.lock | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/aws-cdk/package.json b/packages/aws-cdk/package.json index 0a966f6100158..d51101027610e 100644 --- a/packages/aws-cdk/package.json +++ b/packages/aws-cdk/package.json @@ -116,7 +116,6 @@ "p-limit": "^3.1.0", "promptly": "^3.2.0", "proxy-agent": "^6.4.0", - "p-limit": "3.1.0", "semver": "^7.6.3", "source-map-support": "^0.5.21", "strip-ansi": "^6.0.1", diff --git a/yarn.lock b/yarn.lock index 2abd133646be3..1cb8b218839dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12871,13 +12871,6 @@ p-finally@^1.0.0: resolved "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== -p-limit@3.1.0, p-limit@^3.0.2, p-limit@^3.1.0: - version "3.1.0" - resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" - integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== - dependencies: - yocto-queue "^0.1.0" - p-limit@^1.1.0: version "1.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" @@ -12892,6 +12885,13 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" +p-limit@^3.0.2, p-limit@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + p-locate@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" From e898d3f34ee0f0f972f430eaa4913a82bf4165e1 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 8 Oct 2024 13:12:02 -0400 Subject: [PATCH 36/68] argh linter --- packages/aws-cdk/README.md | 4 ++- packages/aws-cdk/lib/api/garbage-collector.ts | 20 +++++++------- .../test/api/garbage-collection.test.ts | 26 +++++++++---------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 7afdf3a1b3ae9..022391c2a7a44 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -883,11 +883,12 @@ CDK Garbage Collection. > [!CAUTION] > CDK Garbage Collection is under development and therefore must be opted in via the `--unstable` flag: `cdk gc --unstable=gc`. - +> > [!WARNING] > `cdk gc` currently only supports garbage collecting S3 Assets. You must specify `cdk gc --unstable=gc --type=s3` as ECR asset garbage collection has not yet been implemented. `cdk gc` garbage collects isolated S3 assets from your bootstrap bucket via the following mechanism: + - for each object in the bootstrap S3 Bucket, check to see if it is referenced in any existing CloudFormation templates - if not, it is treated as isolated and either tagged or deleted, depending on your configuration. @@ -911,6 +912,7 @@ cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 You can also configure the scope that `cdk gc` performs via the `--action` option. By default, all actions are performed, but you can specify `print`, `tag`, or `delete-tagged`. + - `print` performs no changes to your AWS account, but finds and prints the number of isolated assets. - `tag` tags any newly isolated assets, but does not delete any isolated assets. - `delete-tagged` deletes assets that have been tagged for longer than the buffer days, but does not tag newly isolated assets. diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index bd4bfea794688..9d072801994f7 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -231,11 +231,13 @@ export class GarbageCollector { if (graceDays > 0) { print(chalk.white('Filtering out assets that are not old enough to delete')); await this.parallelReadAllTags(s3, isolated); + // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism deletables = await Promise.all(isolated.map(async (obj) => { const shouldDelete = await obj.isolatedTagBefore(s3, new Date(currentTime - (graceDays * DAY))); return shouldDelete ? obj : null; })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); + // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism taggables = await Promise.all(isolated.map(async (obj) => { const shouldTag = await obj.noIsolatedTag(s3); return shouldTag ? obj : null; @@ -436,34 +438,34 @@ export class GarbageCollector { } private async waitForStacksAndRefetch( - cfn: CloudFormation, - originalResponse: AWS.CloudFormation.ListStacksOutput, - reviewInProgressStacks: AWS.CloudFormation.StackSummary[] + cfn: CloudFormation, + originalResponse: AWS.CloudFormation.ListStacksOutput, + reviewInProgressStacks: AWS.CloudFormation.StackSummary[], ): Promise> { const startTime = Date.now(); - + while (Date.now() - startTime < this.maxWaitTime) { let allStacksUpdated = true; - + for (const stack of reviewInProgressStacks) { const response = await cfn.describeStacks({ StackName: stack.StackId ?? stack.StackName }).promise(); const currentStatus = response.Stacks?.[0]?.StackStatus; - + if (currentStatus === 'REVIEW_IN_PROGRESS') { allStacksUpdated = false; break; } } - + if (allStacksUpdated) { // All stacks have left REVIEW_IN_PROGRESS state, refetch the list return cfn.listStacks({ NextToken: originalResponse.NextToken }); } - + // Wait for 15 seconds before checking again await new Promise(resolve => setTimeout(resolve, 15000)); } - + // If we've reached this point, some stacks are still in REVIEW_IN_PROGRESS after waiting const remainingStacks = reviewInProgressStacks.map(s => s.StackName).join(', '); throw new Error(`Stacks still in REVIEW_IN_PROGRESS state after waiting for 1 minute: ${remainingStacks}`); diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index ff4e1894b11e2..a9feb43c646da 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -288,7 +288,7 @@ describe('Garbage Collection', () => { expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); }); - test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { + test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { mockTheToolkitInfo({ Outputs: [ { @@ -297,7 +297,7 @@ describe('Garbage Collection', () => { }, ], }); - + // Mock the listStacks call const mockListStacksStatus = jest.fn() .mockResolvedValueOnce({ @@ -312,7 +312,7 @@ describe('Garbage Collection', () => { { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, ], }); - + // Mock the describeStacks call const mockDescribeStacksStatus = jest.fn() .mockResolvedValueOnce({ @@ -321,14 +321,14 @@ describe('Garbage Collection', () => { .mockResolvedValueOnce({ Stacks: [{ StackName: 'Stack1', StackStatus: 'UPDATE_COMPLETE' }], }); - + sdk.stubCloudFormation({ listStacks: mockListStacksStatus, describeStacks: mockDescribeStacksStatus, getTemplateSummary: mockGetTemplateSummary, getTemplate: mockGetTemplate, }); - + garbageCollector = new GarbageCollector({ sdkProvider: sdk, action: 'full', @@ -341,9 +341,9 @@ describe('Garbage Collection', () => { rollbackBufferDays: 3, type: 's3', }); - + await garbageCollector.garbageCollect(); - + // describe and list are called as expected expect(mockDescribeStacksStatus).toHaveBeenCalledTimes(2); expect(mockListStacksStatus).toHaveBeenCalledTimes(2); @@ -357,7 +357,7 @@ describe('Garbage Collection', () => { expect(mockDeleteObjects).toHaveBeenCalledTimes(0); }, 60000); - test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { + test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { mockTheToolkitInfo({ Outputs: [ { @@ -366,7 +366,7 @@ describe('Garbage Collection', () => { }, ], }); - + // Mock the listStacks call const mockListStacksStatus = jest.fn() .mockResolvedValue({ @@ -375,20 +375,20 @@ describe('Garbage Collection', () => { { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, ], }); - + // Mock the describeStacks call const mockDescribeStacksStatus = jest.fn() .mockResolvedValue({ Stacks: [{ StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }], }); - + sdk.stubCloudFormation({ listStacks: mockListStacksStatus, describeStacks: mockDescribeStacksStatus, getTemplateSummary: mockGetTemplateSummary, getTemplate: mockGetTemplate, }); - + garbageCollector = new GarbageCollector({ sdkProvider: sdk, action: 'full', @@ -402,7 +402,7 @@ describe('Garbage Collection', () => { type: 's3', maxWaitTime: 100, // Wait for only 100ms in tests }); - + await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting for 1 minute/); }, 60000); }); From 2e1b780fb4d488146b76160f7e819b46ab523bf1 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Sun, 13 Oct 2024 09:58:38 -0400 Subject: [PATCH 37/68] minor updates --- packages/aws-cdk/README.md | 14 +++++++------- packages/aws-cdk/lib/api/garbage-collector.ts | 6 ------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 022391c2a7a44..fdccd26afb6bb 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -887,10 +887,10 @@ CDK Garbage Collection. > [!WARNING] > `cdk gc` currently only supports garbage collecting S3 Assets. You must specify `cdk gc --unstable=gc --type=s3` as ECR asset garbage collection has not yet been implemented. -`cdk gc` garbage collects isolated S3 assets from your bootstrap bucket via the following mechanism: +`cdk gc` garbage collects unused S3 assets from your bootstrap bucket via the following mechanism: - for each object in the bootstrap S3 Bucket, check to see if it is referenced in any existing CloudFormation templates -- if not, it is treated as isolated and either tagged or deleted, depending on your configuration. +- if not, it is treated as unused and gc will either tag it or delete it, depending on your configuration. The most basic usage looks like this: @@ -902,7 +902,7 @@ This will garbage collect S3 assets from the current bootstrapped environment(s) policy on the bucket. If you are concerned about deleting assets too aggressively, you can configure a buffer amount of days to keep -an isolated asset before deletion. In this scenario, instead of deleting isolated objects, `cdk gc` will tag +an unused asset before deletion. In this scenario, instead of deleting unused objects, `cdk gc` will tag them with today's date instead. It will also check if any objects have been tagged by previous runs of `cdk gc` and delete them if they have been tagged for longer than the buffer days. @@ -913,15 +913,15 @@ cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 You can also configure the scope that `cdk gc` performs via the `--action` option. By default, all actions are performed, but you can specify `print`, `tag`, or `delete-tagged`. -- `print` performs no changes to your AWS account, but finds and prints the number of isolated assets. -- `tag` tags any newly isolated assets, but does not delete any isolated assets. -- `delete-tagged` deletes assets that have been tagged for longer than the buffer days, but does not tag newly isolated assets. +- `print` performs no changes to your AWS account, but finds and prints the number of unused assets. +- `tag` tags any newly unused assets, but does not delete any unused assets. +- `delete-tagged` deletes assets that have been tagged for longer than the buffer days, but does not tag newly unused assets. ```console cdk gc --unstable=gc --type=s3 --action=delete-tagged --rollback-buffer-days=30 ``` -This will delete assets that have been isolated for >30 days, but will not tag additional assets. +This will delete assets that have been unused for >30 days, but will not tag additional assets. ### `cdk doctor` diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collector.ts index 9d072801994f7..b90d4c1b96e98 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collector.ts @@ -82,22 +82,16 @@ interface GarbageCollectorProps { /** * The action to perform. Specify this if you want to perform a truncated set * of actions available. - * - * @default 'full' */ readonly action: 'print' | 'tag' | 'delete-tagged' | 'full'; /** * The type of asset to garbage collect. - * - * @default 'all' */ readonly type: 's3' | 'ecr' | 'all'; /** * The days an asset must be in isolation before being actually deleted. - * - * @default 0 */ readonly rollbackBufferDays: number; From fd08da584fe18c3d9337ca9b60cb88a35c9dc825 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Sun, 13 Oct 2024 11:40:01 -0400 Subject: [PATCH 38/68] refactor how stack refresh is done --- .../garbage-collector.ts | 178 ++--------------- .../api/garbage-collection/stack-refresh.ts | 180 ++++++++++++++++++ packages/aws-cdk/lib/api/index.ts | 2 +- .../test/api/garbage-collection.test.ts | 26 +-- 4 files changed, 202 insertions(+), 184 deletions(-) rename packages/aws-cdk/lib/api/{ => garbage-collection}/garbage-collector.ts (62%) create mode 100644 packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts diff --git a/packages/aws-cdk/lib/api/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts similarity index 62% rename from packages/aws-cdk/lib/api/garbage-collector.ts rename to packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index b90d4c1b96e98..5e291596aedb3 100644 --- a/packages/aws-cdk/lib/api/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -1,9 +1,10 @@ import * as cxapi from '@aws-cdk/cx-api'; -import { CloudFormation, S3 } from 'aws-sdk'; +import { S3 } from 'aws-sdk'; import * as chalk from 'chalk'; -import { ISDK, Mode, SdkProvider } from './aws-auth'; -import { print } from '../logging'; -import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from './toolkit-info'; +import { print } from '../../logging'; +import { ISDK, Mode, SdkProvider } from '../aws-auth'; +import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; +import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; // Must use a require() otherwise esbuild complains // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -13,23 +14,6 @@ const ISOLATED_TAG = 'aws-cdk:isolated'; const P_LIMIT = 50; const DAY = 24 * 60 * 60 * 1000; // Number of milliseconds in a day -class ActiveAssetCache { - private readonly stacks: Set = new Set(); - - public rememberStack(stackTemplate: string) { - this.stacks.add(stackTemplate); - } - - public contains(asset: string): boolean { - for (const stack of this.stacks) { - if (stack.includes(asset)) { - return true; - } - } - return false; - } -} - class S3Asset { private cached_tags: S3.TagSet | undefined = undefined; @@ -165,37 +149,19 @@ export class GarbageCollector { const s3 = sdk.s3(); const qualifier = await this.bootstrapQualifier(sdk, this.bootstrapStackName); - const activeAssets = new ActiveAssetCache(); - let refreshStacksRunning = false; - const refreshStacks = async (isInitial?: boolean) => { - if (refreshStacksRunning) { - return; - } - - refreshStacksRunning = true; - - try { - const stacks = await this.fetchAllStackTemplates(cfn, qualifier); - for (const stack of stacks) { - activeAssets.rememberStack(stack); - } - } catch (err) { - throw new Error(`Error refreshing stacks: ${err}`); - } finally { - refreshStacksRunning = false; - - if (!isInitial) { - setTimeout(refreshStacks, 300_000); - } - } - }; - // Grab stack templates first - await refreshStacks(true); - // Refresh stacks in the background - const timeout = setTimeout(refreshStacks, 300_000); + const startTime = Date.now(); + await refreshStacks(cfn, activeAssets, this.maxWaitTime, qualifier); + // Start the background refresh + const backgroundStackRefresh = new BackgroundStackRefresh({ + cfn, + activeAssets, + qualifier, + maxWaitTime: this.maxWaitTime, + }); + const timeout = setTimeout(backgroundStackRefresh.start, Math.max(startTime + 300_000 - Date.now(), 0)) try { const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); @@ -249,11 +215,12 @@ export class GarbageCollector { await this.parallelTag(s3, bucket, taggables, currentTime); } - // TODO: maybe undelete + // TODO: untag } } catch (err: any) { throw new Error(err); } finally { + backgroundStackRefresh.stop(); clearTimeout(timeout); } } @@ -364,115 +331,4 @@ export class GarbageCollector { } } while (continuationToken); } - - /** - * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: - * - stacks in DELETE_COMPLETE or DELET_IN_PROGRES stage - * - stacks that are using a different bootstrap qualifier - * - * It fails on the following stacks because we cannot get the template and therefore have an imcomplete - * understanding of what assets are being used. - * - stacks in REVIEW_IN_PROGRESS stage - */ - private async fetchAllStackTemplates(cfn: CloudFormation, qualifier?: string) { - const stackNames: string[] = []; - await paginateSdkCall(async (nextToken) => { - let response = await cfn.listStacks({ NextToken: nextToken }).promise(); - - // We cannot operate on REVIEW_IN_PROGRESS stacks because we do not know what the template looks like in this case - // If we encounter this status, we will wait up to a minute for it to land before erroring out. - const reviewInProgressStacks = response.StackSummaries?.filter(s => s.StackStatus === 'REVIEW_IN_PROGRESS') ?? []; - if (reviewInProgressStacks.length > 0) { - const updatedRequest = await this.waitForStacksAndRefetch(cfn, response, reviewInProgressStacks); - response = await updatedRequest.promise(); - } - - // Deleted stacks are ignored - const ignoredStatues = ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_IN_PROGRESS', 'DELETE_FAILED']; - stackNames.push( - ...(response.StackSummaries ?? []) - .filter(s => !ignoredStatues.includes(s.StackStatus)) - .map(s => s.StackId ?? s.StackName), - ); - - return response.NextToken; - }); - - print(chalk.blue(`Parsing through ${stackNames.length} stacks`)); - - const templates: string[] = []; - for (const stack of stackNames) { - let summary; - summary = await cfn.getTemplateSummary({ - StackName: stack, - }).promise(); - - // Filter out stacks that we KNOW are using a different bootstrap qualifier - // This is necessary because a stack under a different bootstrap could coincidentally reference the same hash - // and cause a false negative (cause an asset to be preserved when its isolated) - // This is intentionally done in a way where we ONLY filter out stacks that are meant for a different qualifier - // because we are okay with false positives. - const bootstrapVersion = summary?.Parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); - const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); - if (qualifier && splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier) { - // This stack is definitely bootstrapped to a different qualifier so we can safely ignore it - continue; - } else { - const template = await cfn.getTemplate({ - StackName: stack, - }).promise(); - - templates.push(template.TemplateBody ?? '' + summary?.Parameters); - } - } - - print(chalk.red('Done parsing through stacks')); - - return templates; - } - - private async waitForStacksAndRefetch( - cfn: CloudFormation, - originalResponse: AWS.CloudFormation.ListStacksOutput, - reviewInProgressStacks: AWS.CloudFormation.StackSummary[], - ): Promise> { - const startTime = Date.now(); - - while (Date.now() - startTime < this.maxWaitTime) { - let allStacksUpdated = true; - - for (const stack of reviewInProgressStacks) { - const response = await cfn.describeStacks({ StackName: stack.StackId ?? stack.StackName }).promise(); - const currentStatus = response.Stacks?.[0]?.StackStatus; - - if (currentStatus === 'REVIEW_IN_PROGRESS') { - allStacksUpdated = false; - break; - } - } - - if (allStacksUpdated) { - // All stacks have left REVIEW_IN_PROGRESS state, refetch the list - return cfn.listStacks({ NextToken: originalResponse.NextToken }); - } - - // Wait for 15 seconds before checking again - await new Promise(resolve => setTimeout(resolve, 15000)); - } - - // If we've reached this point, some stacks are still in REVIEW_IN_PROGRESS after waiting - const remainingStacks = reviewInProgressStacks.map(s => s.StackName).join(', '); - throw new Error(`Stacks still in REVIEW_IN_PROGRESS state after waiting for 1 minute: ${remainingStacks}`); - } -} - -async function paginateSdkCall(cb: (nextToken?: string) => Promise) { - let finished = false; - let nextToken: string | undefined; - while (!finished) { - nextToken = await cb(nextToken); - if (nextToken === undefined) { - finished = true; - } - } } diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts new file mode 100644 index 0000000000000..f1f9cc9630f26 --- /dev/null +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -0,0 +1,180 @@ +import { CloudFormation } from 'aws-sdk'; +import * as chalk from 'chalk'; +import { print } from '../../logging'; +import { sleep } from '../../../test/util'; + +export class ActiveAssetCache { + private readonly stacks: Set = new Set(); + + public rememberStack(stackTemplate: string) { + this.stacks.add(stackTemplate); + } + + public contains(asset: string): boolean { + for (const stack of this.stacks) { + if (stack.includes(asset)) { + return true; + } + } + return false; + } +} + +async function paginateSdkCall(cb: (nextToken?: string) => Promise) { + let finished = false; + let nextToken: string | undefined; + while (!finished) { + nextToken = await cb(nextToken); + if (nextToken === undefined) { + finished = true; + } + } +} + +/** We cannot operate on REVIEW_IN_PROGRESS stacks because we do not know what the template looks like in this case + * If we encounter this status, we will wait up to the maxWaitTime before erroring out + */ +async function listStacksNotBeingReviewed(cfn: CloudFormation, maxWaitTime: number, nextToken: string | undefined) { + let sleepMs = 500; + console.log("1"); + + while(sleepMs < maxWaitTime) { + console.log("2"); + let stacks = await cfn.listStacks({ NextToken: nextToken }).promise(); + if (!stacks.StackSummaries?.some(s => s.StackStatus == 'REVIEW_IN_PROGRESS')) { + return stacks; + } + await sleep(Math.floor(Math.random() * sleepMs)); + sleepMs = Math.min(sleepMs * 2, maxWaitTime); + } + + throw new Error(`Stacks still in REVIEW_IN_PROGRESS state after waiting for ${maxWaitTime} ms.`); +} + +/** + * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: + * - stacks in DELETE_COMPLETE or DELETE_IN_PROGRES stage + * - stacks that are using a different bootstrap qualifier + * + * It fails on the following stacks because we cannot get the template and therefore have an imcomplete + * understanding of what assets are being used. + * - stacks in REVIEW_IN_PROGRESS stage + */ +async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, qualifier?: string) { + const stackNames: string[] = []; + await paginateSdkCall(async (nextToken) => { + const stacks = await listStacksNotBeingReviewed(cfn, maxWaitTime, nextToken); + + // Deleted stacks are ignored + const ignoredStatues = ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_IN_PROGRESS', 'DELETE_FAILED']; + stackNames.push( + ...(stacks.StackSummaries ?? []) + .filter(s => !ignoredStatues.includes(s.StackStatus)) + .map(s => s.StackId ?? s.StackName), + ); + + return stacks.NextToken; + }); + + print(chalk.blue(`Parsing through ${stackNames.length} stacks`)); + + const templates: string[] = []; + for (const stack of stackNames) { + let summary; + summary = await cfn.getTemplateSummary({ + StackName: stack, + }).promise(); + + // Filter out stacks that we KNOW are using a different bootstrap qualifier + // This is necessary because a stack under a different bootstrap could coincidentally reference the same hash + // and cause a false negative (cause an asset to be preserved when its isolated) + // This is intentionally done in a way where we ONLY filter out stacks that are meant for a different qualifier + // because we are okay with false positives. + const bootstrapVersion = summary?.Parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); + const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); + if (qualifier && splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier) { + // This stack is definitely bootstrapped to a different qualifier so we can safely ignore it + continue; + } else { + const template = await cfn.getTemplate({ + StackName: stack, + }).promise(); + + templates.push(template.TemplateBody ?? '' + summary?.Parameters); + } + } + + print(chalk.red('Done parsing through stacks')); + + return templates; +} + +export async function refreshStacks(cfn: CloudFormation, activeAssets: ActiveAssetCache, maxWaitTime: number, qualifier?: string) { + try { + const stacks = await fetchAllStackTemplates(cfn, maxWaitTime, qualifier); + for (const stack of stacks) { + activeAssets.rememberStack(stack); + } + } catch (err) { + throw new Error(`Error refreshing stacks: ${err}`); + } +} + +/** + * Background Stack Refresh properties + */ +export interface BackgroundStackRefreshProps { + /** + * The CFN SDK handler + */ + readonly cfn: CloudFormation; + + /** + * Active Asset storage + */ + readonly activeAssets: ActiveAssetCache; + + /** + * Stack bootstrap qualifier + */ + readonly qualifier?: string; + + /** + * Maximum wait time when waiting for stacks to leave REVIEW_IN_PROGRESS stage. + * + * @default 60000 + */ + readonly maxWaitTime?: number; +} + +/** + * Class that controls scheduling of the background stack refresh + */ +export class BackgroundStackRefresh { + private _isRefreshing = false; + private timeout: NodeJS.Timeout | undefined; + + constructor(private readonly props: BackgroundStackRefreshProps) {} + + public async start() { + if (this.isRefreshing) { + return; + } + + const startTime = Date.now(); + this._isRefreshing = true; + + await refreshStacks(this.props.cfn, this.props.activeAssets, this.props.maxWaitTime ?? 60000, this.props.qualifier); + + this._isRefreshing = false; + this.timeout = setTimeout(this.start, Math.max(startTime + 300_000 - Date.now(), 0)); + } + + public stop() { + clearTimeout(this.timeout); + } + + public get isRefreshing(): boolean { + return this._isRefreshing; + } +} diff --git a/packages/aws-cdk/lib/api/index.ts b/packages/aws-cdk/lib/api/index.ts index 2bd0a29378d26..9f3ba4e355a7c 100644 --- a/packages/aws-cdk/lib/api/index.ts +++ b/packages/aws-cdk/lib/api/index.ts @@ -1,6 +1,6 @@ export * from './aws-auth/credentials'; export * from './bootstrap'; -export * from './garbage-collector'; +export * from './garbage-collection/garbage-collector'; export * from './deploy-stack'; export * from './toolkit-info'; export * from './aws-auth'; diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index a9feb43c646da..7fb920a4b0ae4 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -313,18 +313,8 @@ describe('Garbage Collection', () => { ], }); - // Mock the describeStacks call - const mockDescribeStacksStatus = jest.fn() - .mockResolvedValueOnce({ - Stacks: [{ StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }], - }) - .mockResolvedValueOnce({ - Stacks: [{ StackName: 'Stack1', StackStatus: 'UPDATE_COMPLETE' }], - }); - sdk.stubCloudFormation({ listStacks: mockListStacksStatus, - describeStacks: mockDescribeStacksStatus, getTemplateSummary: mockGetTemplateSummary, getTemplate: mockGetTemplate, }); @@ -344,8 +334,7 @@ describe('Garbage Collection', () => { await garbageCollector.garbageCollect(); - // describe and list are called as expected - expect(mockDescribeStacksStatus).toHaveBeenCalledTimes(2); + // list are called as expected expect(mockListStacksStatus).toHaveBeenCalledTimes(2); // everything else runs as expected: @@ -357,7 +346,7 @@ describe('Garbage Collection', () => { expect(mockDeleteObjects).toHaveBeenCalledTimes(0); }, 60000); - test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { + test('fails when stackStatus stuck in REVIEW_IN_PROGRESS', async () => { mockTheToolkitInfo({ Outputs: [ { @@ -376,15 +365,8 @@ describe('Garbage Collection', () => { ], }); - // Mock the describeStacks call - const mockDescribeStacksStatus = jest.fn() - .mockResolvedValue({ - Stacks: [{ StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }], - }); - sdk.stubCloudFormation({ listStacks: mockListStacksStatus, - describeStacks: mockDescribeStacksStatus, getTemplateSummary: mockGetTemplateSummary, getTemplate: mockGetTemplate, }); @@ -400,9 +382,9 @@ describe('Garbage Collection', () => { bootstrapStackName: 'GarbageStack', rollbackBufferDays: 3, type: 's3', - maxWaitTime: 100, // Wait for only 100ms in tests + maxWaitTime: 600, // Wait for only 600ms in tests }); - await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting for 1 minute/); + await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting/); }, 60000); }); From 796248e410681311fed4b3152926be485762a7c8 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 08:54:45 -0400 Subject: [PATCH 39/68] add debugs --- .../garbage-collection/garbage-collector.ts | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 5e291596aedb3..31411d52485b4 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -1,7 +1,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import { S3 } from 'aws-sdk'; import * as chalk from 'chalk'; -import { print } from '../../logging'; +import { debug, print } from '../../logging'; import { ISDK, Mode, SdkProvider } from '../aws-auth'; import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; @@ -49,12 +49,9 @@ class S3Asset { public async isolatedTagBefore(s3: S3, date: Date) { const tagValue = await this.getTag(s3, ISOLATED_TAG); - print(chalk.red('a', tagValue, 'b', date)); if (!tagValue || tagValue == '') { - print(chalk.red('no tag')); return false; } - print(chalk.red('tag', typeof(tagValue))); return new Date(tagValue) < date; } } @@ -128,8 +125,6 @@ export class GarbageCollector { this.permissionToTag = ['tag', 'full'].includes(props.action); this.maxWaitTime = props.maxWaitTime ?? 60000; - print(chalk.white(this.permissionToDelete, this.permissionToTag, props.action)); - this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; // TODO: ECR garbage collection @@ -142,7 +137,6 @@ export class GarbageCollector { * Perform garbage collection on the resolved environment. */ public async garbageCollect() { - print(chalk.black(this.garbageCollectS3Assets)); // SDKs const sdk = (await this.props.sdkProvider.forEnvironment(this.props.resolvedEnvironment, Mode.ForWriting)).sdk; const cfn = sdk.cloudFormation(); @@ -171,7 +165,7 @@ export class GarbageCollector { const currentTime = Date.now(); const graceDays = this.props.rollbackBufferDays; - print(chalk.white(`Parsing through ${numObjects} objects in batches`)); + debug(`Parsing through ${numObjects} objects in batches`); // Process objects in batches of 1000 // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario @@ -183,13 +177,13 @@ export class GarbageCollector { return !activeAssets.contains(obj.fileName()); }); - print(chalk.blue(`${isolated.length} isolated assets`)); + debug(`${isolated.length} isolated assets`); let deletables: S3Asset[] = isolated; let taggables: S3Asset[] = []; if (graceDays > 0) { - print(chalk.white('Filtering out assets that are not old enough to delete')); + debug('Filtering out assets that are not old enough to delete'); await this.parallelReadAllTags(s3, isolated); // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism deletables = await Promise.all(isolated.map(async (obj) => { @@ -204,8 +198,8 @@ export class GarbageCollector { })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); } - print(chalk.blue(`${deletables.length} deletable assets`)); - print(chalk.white(`${taggables.length} taggable assets`)); + debug(`${deletables.length} deletable assets`); + debug(`${taggables.length} taggable assets`); if (this.permissionToDelete && deletables.length > 0) { await this.parallelDelete(s3, bucket, deletables); @@ -257,7 +251,7 @@ export class GarbageCollector { ); } - print(chalk.green(`Tagged ${taggables.length} assets`)); + debug(`Tagged ${taggables.length} assets`); } /** @@ -277,7 +271,7 @@ export class GarbageCollector { }, }).promise(); - print(chalk.green(`Deleted ${deletables.length} assets`)); + debug(`Deleted ${deletables.length} assets`); } catch (err) { print(chalk.red(`Error deleting objects: ${err}`)); } From 6a406c3163d41461f57ce50433f25296d8a6c252 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 15:30:16 -0400 Subject: [PATCH 40/68] progress printer --- .../garbage-collection/garbage-collector.ts | 31 +++++++--- .../garbage-collection/progress-printer.ts | 60 +++++++++++++++++++ .../api/garbage-collection/stack-refresh.ts | 15 ++--- packages/aws-cdk/lib/cdk-toolkit.ts | 2 +- 4 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 31411d52485b4..422a9b7409c9b 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -5,6 +5,7 @@ import { debug, print } from '../../logging'; import { ISDK, Mode, SdkProvider } from '../aws-auth'; import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; +import { ProgressPrinter } from './progress-printer'; // Must use a require() otherwise esbuild complains // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -14,7 +15,7 @@ const ISOLATED_TAG = 'aws-cdk:isolated'; const P_LIMIT = 50; const DAY = 24 * 60 * 60 * 1000; // Number of milliseconds in a day -class S3Asset { +export class S3Asset { private cached_tags: S3.TagSet | undefined = undefined; public constructor(private readonly bucket: string, public readonly key: string, public readonly size: number) {} @@ -121,6 +122,8 @@ export class GarbageCollector { this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); this.garbageCollectEcrAssets = ['ecr', 'all'].includes(props.type); + debug(`${this.garbageCollectS3Assets} ${this.garbageCollectEcrAssets}`); + this.permissionToDelete = ['delete-tagged', 'full'].includes(props.action); this.permissionToTag = ['tag', 'full'].includes(props.action); this.maxWaitTime = props.maxWaitTime ?? 60000; @@ -155,16 +158,20 @@ export class GarbageCollector { qualifier, maxWaitTime: this.maxWaitTime, }); - const timeout = setTimeout(backgroundStackRefresh.start, Math.max(startTime + 300_000 - Date.now(), 0)) + const timeout = setTimeout(backgroundStackRefresh.start, Math.max(startTime + 300_000 - Date.now(), 0)); + + const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); + const numObjects = await this.numObjectsInBucket(s3, bucket); + const printer = new ProgressPrinter(numObjects, 1000); + debug(`Found bootstrap bucket ${bucket}`); + try { - const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); - const numObjects = await this.numObjectsInBucket(s3, bucket); const batches = 1; const batchSize = 1000; const currentTime = Date.now(); const graceDays = this.props.rollbackBufferDays; - + debug(`Parsing through ${numObjects} objects in batches`); // Process objects in batches of 1000 @@ -172,6 +179,7 @@ export class GarbageCollector { // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. for await (const batch of this.readBucketInBatches(s3, bucket, batchSize)) { print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); + printer.start(); const isolated = batch.filter((obj) => { return !activeAssets.contains(obj.fileName()); @@ -202,19 +210,22 @@ export class GarbageCollector { debug(`${taggables.length} taggable assets`); if (this.permissionToDelete && deletables.length > 0) { - await this.parallelDelete(s3, bucket, deletables); + await this.parallelDelete(s3, bucket, deletables, printer); } if (this.permissionToTag && taggables.length > 0) { - await this.parallelTag(s3, bucket, taggables, currentTime); + await this.parallelTag(s3, bucket, taggables, currentTime, printer); } + printer.reportScannedObjects(batch.length); + // TODO: untag } } catch (err: any) { throw new Error(err); } finally { backgroundStackRefresh.stop(); + printer.stop(); clearTimeout(timeout); } } @@ -231,7 +242,7 @@ export class GarbageCollector { * Tag objects in parallel using p-limit. The putObjectTagging API does not * support batch tagging so we must handle the parallelism client-side. */ - private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[], date: number) { + private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[], date: number, printer: ProgressPrinter) { const limit = pLimit(P_LIMIT); for (const obj of taggables) { @@ -251,13 +262,14 @@ export class GarbageCollector { ); } + printer.reportTaggedObjects(taggables); debug(`Tagged ${taggables.length} assets`); } /** * Delete objects in parallel. The deleteObjects API supports batches of 1000. */ - private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[]) { + private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[], printer: ProgressPrinter) { const objectsToDelete: S3.ObjectIdentifierList = deletables.map(asset => ({ Key: asset.key, })); @@ -272,6 +284,7 @@ export class GarbageCollector { }).promise(); debug(`Deleted ${deletables.length} assets`); + printer.reportDeletedObjects(deletables); } catch (err) { print(chalk.red(`Error deleting objects: ${err}`)); } diff --git a/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts new file mode 100644 index 0000000000000..3945fab6c9afe --- /dev/null +++ b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts @@ -0,0 +1,60 @@ +import * as chalk from 'chalk'; +import { print } from '../../logging'; +import { S3Asset } from './garbage-collector'; + +export class ProgressPrinter { + private totalObjects: number; + private objectsScanned: number; + private taggedObjects: number; + private taggedObjectsSizeMb: number; + private deletedObjects: number; + private deletedObjectsSizeMb: number; + private interval: number; + private setInterval?: NodeJS.Timer; + + constructor(totalObjects: number, interval?: number) { + this.totalObjects = totalObjects; + this.objectsScanned = 0; + this.taggedObjects = 0; + this.taggedObjectsSizeMb = 0; + this.deletedObjects = 0; + this.deletedObjectsSizeMb = 0; + this.interval = interval ?? 10_000; + } + + public reportScannedObjects(amt: number) { + this.objectsScanned += amt; + } + + public reportTaggedObjects(objects: S3Asset[]) { + this.taggedObjects += objects.length; + const sizeInBytes = objects.reduce((total, asset) => total + asset.size, 0); + this.taggedObjectsSizeMb += sizeInBytes / 1_048_576; + } + + public reportDeletedObjects(objects: S3Asset[]) { + this.deletedObjects += objects.length; + const sizeInBytes = objects.reduce((total, asset) => total + asset.size, 0); + this.deletedObjectsSizeMb += sizeInBytes / 1_048_576; + } + + public start() { + this.setInterval = setInterval(() => this.print(), this.interval); + } + + public stop() { + clearInterval(this.setInterval); + // print one last time + this.print(); + } + + private print() { + const percentage = ((this.objectsScanned / this.totalObjects) * 100).toFixed(2); + // print in MiB until we hit at least 1 GiB of data tagged/deleted + if (Math.max(this.taggedObjectsSizeMb, this.deletedObjectsSizeMb) >= 1000) { + print(chalk.green(`[${percentage}%] ${this.objectsScanned} files scanned: ${this.taggedObjects} objects (${(this.taggedObjectsSizeMb / 1000).toFixed(2)} GiB) tagged, ${this.deletedObjects} objects (${(this.deletedObjectsSizeMb / 1000).toFixed(2)} GiB) deleted.`)); + } else { + print(chalk.green(`[${percentage}%] ${this.objectsScanned} files scanned: ${this.taggedObjects} objects (${this.taggedObjectsSizeMb.toFixed(2)} MiB) tagged, ${this.deletedObjects} objects (${this.deletedObjectsSizeMb.toFixed(2)} MiB) deleted.`)); + } + } +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index f1f9cc9630f26..57ec21daf2227 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -1,7 +1,6 @@ import { CloudFormation } from 'aws-sdk'; -import * as chalk from 'chalk'; -import { print } from '../../logging'; import { sleep } from '../../../test/util'; +import { debug } from '../../logging'; export class ActiveAssetCache { private readonly stacks: Set = new Set(); @@ -36,10 +35,8 @@ async function paginateSdkCall(cb: (nextToken?: string) => Promise s.StackStatus == 'REVIEW_IN_PROGRESS')) { return stacks; @@ -60,7 +57,7 @@ async function listStacksNotBeingReviewed(cfn: CloudFormation, maxWaitTime: numb * understanding of what assets are being used. * - stacks in REVIEW_IN_PROGRESS stage */ -async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, qualifier?: string) { +async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, qualifier?: string) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { const stacks = await listStacksNotBeingReviewed(cfn, maxWaitTime, nextToken); @@ -76,7 +73,7 @@ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, return stacks.NextToken; }); - print(chalk.blue(`Parsing through ${stackNames.length} stacks`)); + debug(`Parsing through ${stackNames.length} stacks`); const templates: string[] = []; for (const stack of stackNames) { @@ -104,7 +101,7 @@ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, } } - print(chalk.red('Done parsing through stacks')); + debug('Done parsing through stacks'); return templates; } @@ -152,7 +149,7 @@ export interface BackgroundStackRefreshProps { */ export class BackgroundStackRefresh { private _isRefreshing = false; - private timeout: NodeJS.Timeout | undefined; + private timeout?: NodeJS.Timeout; constructor(private readonly props: BackgroundStackRefreshProps) {} diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index eaf884ec66da3..b7500d98e5640 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -12,7 +12,7 @@ import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap'; import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly'; import { CloudExecutable } from './api/cxapp/cloud-executable'; import { Deployments } from './api/deployments'; -import { GarbageCollector } from './api/garbage-collector'; +import { GarbageCollector } from './api/garbage-collection/garbage-collector'; import { HotswapMode } from './api/hotswap/common'; import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs'; import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor'; From 551aa86a7a87059fc218d4d5d41a79590f826f1a Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 15:31:27 -0400 Subject: [PATCH 41/68] lint --- .../aws-cdk/lib/api/garbage-collection/garbage-collector.ts | 6 +++--- .../aws-cdk/lib/api/garbage-collection/progress-printer.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 422a9b7409c9b..1b5b3e67e97bb 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -4,8 +4,8 @@ import * as chalk from 'chalk'; import { debug, print } from '../../logging'; import { ISDK, Mode, SdkProvider } from '../aws-auth'; import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; -import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; import { ProgressPrinter } from './progress-printer'; +import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; // Must use a require() otherwise esbuild complains // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -165,13 +165,13 @@ export class GarbageCollector { const printer = new ProgressPrinter(numObjects, 1000); debug(`Found bootstrap bucket ${bucket}`); - + try { const batches = 1; const batchSize = 1000; const currentTime = Date.now(); const graceDays = this.props.rollbackBufferDays; - + debug(`Parsing through ${numObjects} objects in batches`); // Process objects in batches of 1000 diff --git a/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts index 3945fab6c9afe..0f0158aa56d5d 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts @@ -1,6 +1,6 @@ import * as chalk from 'chalk'; -import { print } from '../../logging'; import { S3Asset } from './garbage-collector'; +import { print } from '../../logging'; export class ProgressPrinter { private totalObjects: number; From 411df38f3d03db21ccde3548e0a9d6a11158c9a9 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 15:52:26 -0400 Subject: [PATCH 42/68] various pr comments --- .../api/garbage-collection/stack-refresh.ts | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 57ec21daf2227..761a944c8cd5e 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -82,14 +82,7 @@ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, StackName: stack, }).promise(); - // Filter out stacks that we KNOW are using a different bootstrap qualifier - // This is necessary because a stack under a different bootstrap could coincidentally reference the same hash - // and cause a false negative (cause an asset to be preserved when its isolated) - // This is intentionally done in a way where we ONLY filter out stacks that are meant for a different qualifier - // because we are okay with false positives. - const bootstrapVersion = summary?.Parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); - const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); - if (qualifier && splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier) { + if (bootstrapFilter(summary.Parameters, qualifier)) { // This stack is definitely bootstrapped to a different qualifier so we can safely ignore it continue; } else { @@ -106,6 +99,25 @@ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, return templates; } +/** + * Filter out stacks that we KNOW are using a different bootstrap qualifier + * This is mostly necessary for the integration tests that can run the same app (with the same assets) + * under different qualifiers. + * This is necessary because a stack under a different bootstrap could coincidentally reference the same hash + * and cause a false negative (cause an asset to be preserved when its isolated) + * This is intentionally done in a way where we ONLY filter out stacks that are meant for a different qualifier + * because we are okay with false positives. + */ +function bootstrapFilter(parameters?: CloudFormation.ParameterDeclarations, qualifier?: string) { + const bootstrapVersion = parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); + const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); + // We find the qualifier in a specific part of the bootstrap version parameter + return (qualifier && + splitBootstrapVersion && + splitBootstrapVersion.length == 4 && + splitBootstrapVersion[2] != qualifier); +} + export async function refreshStacks(cfn: CloudFormation, activeAssets: ActiveAssetCache, maxWaitTime: number, qualifier?: string) { try { const stacks = await fetchAllStackTemplates(cfn, maxWaitTime, qualifier); @@ -148,30 +160,21 @@ export interface BackgroundStackRefreshProps { * Class that controls scheduling of the background stack refresh */ export class BackgroundStackRefresh { - private _isRefreshing = false; private timeout?: NodeJS.Timeout; constructor(private readonly props: BackgroundStackRefreshProps) {} public async start() { - if (this.isRefreshing) { - return; - } - const startTime = Date.now(); - this._isRefreshing = true; await refreshStacks(this.props.cfn, this.props.activeAssets, this.props.maxWaitTime ?? 60000, this.props.qualifier); - this._isRefreshing = false; + // If the last invocation of refreshStacks takes <5 minutes, the next invocation starts 5 minutes after the last one started. + // If the last invocation of refreshStacks takes >5 minutes, the next invocation starts immediately. this.timeout = setTimeout(this.start, Math.max(startTime + 300_000 - Date.now(), 0)); } public stop() { clearTimeout(this.timeout); } - - public get isRefreshing(): boolean { - return this._isRefreshing; - } } From a46aa35fb3b18ca283df7aa23f653b2d80bcaab0 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 16:21:55 -0400 Subject: [PATCH 43/68] add two missing unit tests --- .../api/garbage-collection/stack-refresh.ts | 2 +- .../test/api/garbage-collection.test.ts | 96 ++++++++++++++++++- 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 761a944c8cd5e..6c2edf15461a7 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -90,7 +90,7 @@ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, StackName: stack, }).promise(); - templates.push(template.TemplateBody ?? '' + summary?.Parameters); + templates.push((template.TemplateBody ?? '') + JSON.stringify(summary?.Parameters)); } } diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index 7fb920a4b0ae4..c9ccc9276a1a5 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -42,7 +42,12 @@ describe('Garbage Collection', () => { { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, ], }); - mockGetTemplateSummary = jest.fn().mockReturnValue({}); + mockGetTemplateSummary = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: "BootstrapVersion", + DefaultValue: "/cdk-bootstrap/abcde/version", + }], + }); mockGetTemplate = jest.fn().mockReturnValue({ TemplateBody: 'abcde', }); @@ -288,6 +293,95 @@ describe('Garbage Collection', () => { expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); }); + test('bootstrap filters out other bootstrap versions', async () => { + mockTheToolkitInfo({ + Parameters: [{ + ParameterKey: 'Qualifier', + ParameterValue: 'zzzzzz', + }], + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 3, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockGetTemplateSummary).toHaveBeenCalledTimes(2); + expect(mockGetTemplate).toHaveBeenCalledTimes(0); + }); + + test('parameter hashes are included', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + const mockGetTemplateSummaryAssets = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: "AssetParametersasset1", + DefaultValue: "asset1", + }], + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummaryAssets, + getTemplate: mockGetTemplate, + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 0, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + // no tagging + expect(mockGetObjectTagging).toHaveBeenCalledTimes(0); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + + // assets are to be deleted + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + // no 'asset1' + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + Quiet: true, + }, + }); + }); + test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { mockTheToolkitInfo({ Outputs: [ From b3c6b99d3ccf0ab6d621ef530b0214f16019440b Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 16:53:13 -0400 Subject: [PATCH 44/68] ignore assets that are changed after gc starts --- .../garbage-collection/garbage-collector.ts | 8 ++- .../test/api/garbage-collection.test.ts | 62 ++++++++++++++++++- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 1b5b3e67e97bb..91f56a5bd4f56 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -177,7 +177,7 @@ export class GarbageCollector { // Process objects in batches of 1000 // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. - for await (const batch of this.readBucketInBatches(s3, bucket, batchSize)) { + for await (const batch of this.readBucketInBatches(s3, bucket, batchSize, currentTime)) { print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); printer.start(); @@ -308,7 +308,7 @@ export class GarbageCollector { /** * Generator function that reads objects from the S3 Bucket in batches. */ - private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000): AsyncGenerator { + private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000, currentTime: number): AsyncGenerator { let continuationToken: string | undefined; do { @@ -323,7 +323,9 @@ export class GarbageCollector { response.Contents?.forEach((obj) => { const key = obj.Key ?? ''; const size = obj.Size ?? 0; - if (obj.Key) { + const lastModified = obj.LastModified?.getTime() ?? Date.now(); + // Store the object if it has a Key and if it has not been modified since the start of garbage collection + if (key && lastModified < currentTime) { batch.push(new S3Asset(bucket, key, size)); } }); diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index c9ccc9276a1a5..1ca6d20152193 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -54,9 +54,9 @@ describe('Garbage Collection', () => { mockListObjectsV2 = jest.fn().mockImplementation(() => { return Promise.resolve({ Contents: [ - { Key: 'asset1' }, - { Key: 'asset2' }, - { Key: 'asset3' }, + { Key: 'asset1', LastModified: new Date(0) }, + { Key: 'asset2', LastModified: new Date(0) }, + { Key: 'asset3', LastModified: new Date(0) }, ], KeyCount: 3, }); @@ -293,6 +293,62 @@ describe('Garbage Collection', () => { expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); }); + test('ignore objects that are modified after gc start', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + const mockListObjectsV2Future = jest.fn().mockImplementation(() => { + return Promise.resolve({ + Contents: [ + { Key: 'asset1', LastModified: new Date(0) }, + { Key: 'asset2', LastModified: new Date(0) }, + { Key: 'asset3', LastModified: new Date(new Date().setFullYear(new Date().getFullYear() + 1))}, // future date ignored everywhere + ], + KeyCount: 3, + }); + }); + + sdk.stubS3({ + listObjectsV2: mockListObjectsV2Future, + getObjectTagging: mockGetObjectTagging, + deleteObjects: mockDeleteObjects, + putObjectTagging: mockPutObjectTagging, + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'full', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 0, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + // assets are to be deleted + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + { Key: 'asset1' }, + { Key: 'asset2' }, + // no asset3 + ], + Quiet: true, + }, + }); + }); + test('bootstrap filters out other bootstrap versions', async () => { mockTheToolkitInfo({ Parameters: [{ From 0f233c500b96ece3c1d4c1525c0a6af24a71d20c Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 16:54:54 -0400 Subject: [PATCH 45/68] linters --- .../lib/api/garbage-collection/garbage-collector.ts | 2 +- .../lib/api/garbage-collection/stack-refresh.ts | 2 +- packages/aws-cdk/test/api/garbage-collection.test.ts | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 91f56a5bd4f56..2ad59cbca7545 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -324,7 +324,7 @@ export class GarbageCollector { const key = obj.Key ?? ''; const size = obj.Size ?? 0; const lastModified = obj.LastModified?.getTime() ?? Date.now(); - // Store the object if it has a Key and if it has not been modified since the start of garbage collection + // Store the object if it has a Key and if it has not been modified since the start of garbage collection if (key && lastModified < currentTime) { batch.push(new S3Asset(bucket, key, size)); } diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 6c2edf15461a7..e9c428c8ad8ff 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -112,7 +112,7 @@ function bootstrapFilter(parameters?: CloudFormation.ParameterDeclarations, qual const bootstrapVersion = parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); // We find the qualifier in a specific part of the bootstrap version parameter - return (qualifier && + return (qualifier && splitBootstrapVersion && splitBootstrapVersion.length == 4 && splitBootstrapVersion[2] != qualifier); diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index 1ca6d20152193..6cdda5a0b7977 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -44,8 +44,8 @@ describe('Garbage Collection', () => { }); mockGetTemplateSummary = jest.fn().mockReturnValue({ Parameters: [{ - ParameterKey: "BootstrapVersion", - DefaultValue: "/cdk-bootstrap/abcde/version", + ParameterKey: 'BootstrapVersion', + DefaultValue: '/cdk-bootstrap/abcde/version', }], }); mockGetTemplate = jest.fn().mockReturnValue({ @@ -308,7 +308,7 @@ describe('Garbage Collection', () => { Contents: [ { Key: 'asset1', LastModified: new Date(0) }, { Key: 'asset2', LastModified: new Date(0) }, - { Key: 'asset3', LastModified: new Date(new Date().setFullYear(new Date().getFullYear() + 1))}, // future date ignored everywhere + { Key: 'asset3', LastModified: new Date(new Date().setFullYear(new Date().getFullYear() + 1)) }, // future date ignored everywhere ], KeyCount: 3, }); @@ -393,8 +393,8 @@ describe('Garbage Collection', () => { const mockGetTemplateSummaryAssets = jest.fn().mockReturnValue({ Parameters: [{ - ParameterKey: "AssetParametersasset1", - DefaultValue: "asset1", + ParameterKey: 'AssetParametersasset1', + DefaultValue: 'asset1', }], }); From f76f4948d8569df6062e558c1221bb5db1be84fd Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 17:26:31 -0400 Subject: [PATCH 46/68] untag --- .../garbage-collection/garbage-collector.ts | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 2ad59cbca7545..dd4ada38023c7 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -31,7 +31,7 @@ export class S3Asset { const response = await s3.getObjectTagging({ Bucket: this.bucket, Key: this.key }).promise(); this.cached_tags = response.TagSet; - return response.TagSet; + return this.cached_tags; } private async getTag(s3: S3, tag: string) { @@ -181,9 +181,14 @@ export class GarbageCollector { print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); printer.start(); - const isolated = batch.filter((obj) => { - return !activeAssets.contains(obj.fileName()); - }); + const { isolated, notIsolated } = batch.reduce<{ isolated: S3Asset[]; notIsolated: S3Asset[] }>((acc, obj) => { + if (!activeAssets.contains(obj.fileName())) { + acc.isolated.push(obj); + } else { + acc.notIsolated.push(obj); + } + return acc; + }, { isolated: [], notIsolated: [] }); debug(`${isolated.length} isolated assets`); @@ -219,7 +224,17 @@ export class GarbageCollector { printer.reportScannedObjects(batch.length); - // TODO: untag + // We untag objects that are referenced in ActiveAssets and currently have the Isolated Tag. + let untaggables: S3Asset[] = await Promise.all(notIsolated.map(async (obj) => { + const noTag = await obj.noIsolatedTag(s3); + return noTag ? null : obj; + })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); + + debug(`${untaggables.length} assets to untag`); + + if (this.permissionToTag && untaggables.length > 0) { + await this.parallelUntag(s3, bucket, untaggables); + } } } catch (err: any) { throw new Error(err); @@ -238,6 +253,37 @@ export class GarbageCollector { } } + /** + * Untag assets that were previously tagged, but now currently referenced. + * Since this is treated as an implementation detail, we do not print the results in the printer. + */ + private async parallelUntag(s3: S3, bucket: string, untaggables: S3Asset[]) { + const limit = pLimit(P_LIMIT); + + for (const obj of untaggables) { + const tags = await obj.allTags(s3); + const updatedTags = tags.filter(tag => tag.Key !== ISOLATED_TAG); + await limit(() => + s3.deleteObjectTagging({ + Bucket: bucket, + Key: obj.key, + + }).promise(), + ); + await limit(() => + s3.putObjectTagging({ + Bucket: bucket, + Key: obj.key, + Tagging: { + TagSet: updatedTags, + }, + }).promise(), + ); + } + + debug(`Untagged ${untaggables.length} assets`); + } + /** * Tag objects in parallel using p-limit. The putObjectTagging API does not * support batch tagging so we must handle the parallelism client-side. From d861414f6c949cab7f8b457e4cec56af7d072b19 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 14 Oct 2024 17:57:50 -0400 Subject: [PATCH 47/68] add integ test --- .../garbage-collection.integtest.ts | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts index 8860eb66c177d..1ded54aa7f05f 100644 --- a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -1,4 +1,4 @@ -import { GetObjectTaggingCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'; +import { GetObjectTaggingCommand, ListObjectsV2Command, PutObjectTaggingCommand } from '@aws-sdk/client-s3'; import { integTest, randomString, withoutBootstrap } from '../../lib'; jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime @@ -144,3 +144,59 @@ integTest( }); }), ); + +integTest( + 'Garbage Collection untags in-use assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + + // Artificially add tagging to the asset in the bootstrap bucket + const result = await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })); + const key = result.Contents!.filter((c) => c.Key?.split('.')[1] == 'zip')[0].Key; // fancy footwork to make sure we have the asset key + await fixture.aws.s3.send(new PutObjectTaggingCommand({ + Bucket: bootstrapBucketName, + Key: key, + Tagging: { + TagSet: [{ + Key: 'aws-cdk:isolated', + Value: '12345', + }, { + Key: 'bogus', + Value: 'val', + }], + }, + })); + + await fixture.cdkGarbageCollect({ + rollbackBufferDays: 100, // this will ensure that we do not delete assets immediately (and just tag them) + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the isolated object tag is removed while the other tag remains + const newTags = await fixture.aws.s3.send(new GetObjectTaggingCommand({ Bucket: bootstrapBucketName, Key: key })); + + expect(newTags.TagSet).toEqual([{ + Key: 'bogus', + Value: 'val', + }]); + }), +); From d1ac1b11c1c164a402525c5aff6da418de1a986d Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 15 Oct 2024 13:45:41 -0400 Subject: [PATCH 48/68] make some calls synchronous to simplify code --- .../garbage-collection/garbage-collector.ts | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index dd4ada38023c7..fcf7c0a968a09 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -34,22 +34,26 @@ export class S3Asset { return this.cached_tags; } - private async getTag(s3: S3, tag: string) { - const tags = await this.allTags(s3); - return tags.find(t => t.Key === tag)?.Value; + private getTag(tag: string) { + if (!this.cached_tags) { + throw new Error("Cannot call getTag before allTags"); + } + return this.cached_tags.find(t => t.Key === tag)?.Value; } - private async hasTag(s3: S3, tag: string) { - const tags = await this.allTags(s3); - return tags.some(t => t.Key === tag); + private hasTag(tag: string) { + if (!this.cached_tags) { + throw new Error("Cannot call hasTag before allTags"); + } + return this.cached_tags.some(t => t.Key === tag); } - public async noIsolatedTag(s3: S3) { - return !(await this.hasTag(s3, ISOLATED_TAG)); + public hasIsolatedTag() { + return this.hasTag(ISOLATED_TAG); } - public async isolatedTagBefore(s3: S3, date: Date) { - const tagValue = await this.getTag(s3, ISOLATED_TAG); + public isolatedTagBefore(date: Date) { + const tagValue = this.getTag(ISOLATED_TAG); if (!tagValue || tagValue == '') { return false; } @@ -194,25 +198,26 @@ export class GarbageCollector { let deletables: S3Asset[] = isolated; let taggables: S3Asset[] = []; + let untaggables: S3Asset[] = []; if (graceDays > 0) { debug('Filtering out assets that are not old enough to delete'); - await this.parallelReadAllTags(s3, isolated); - // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism - deletables = await Promise.all(isolated.map(async (obj) => { - const shouldDelete = await obj.isolatedTagBefore(s3, new Date(currentTime - (graceDays * DAY))); - return shouldDelete ? obj : null; - })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); - - // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism - taggables = await Promise.all(isolated.map(async (obj) => { - const shouldTag = await obj.noIsolatedTag(s3); - return shouldTag ? obj : null; - })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); + await this.parallelReadAllTags(s3, batch); + + // We delete objects that are not referenced in ActiveAssets and have the Isolated Tag with a date + // earlier than the current time - grace period. + deletables = isolated.filter(obj => obj.isolatedTagBefore(new Date(currentTime - (graceDays * DAY)))); + + // We tag objects that are not referenced in ActiveAssets and do not have the Isolated Tag. + taggables = isolated.filter(obj => !obj.hasIsolatedTag()); + + // We untag objects that are referenced in ActiveAssets and currently have the Isolated Tag. + untaggables = notIsolated.filter(obj => obj.hasIsolatedTag()); } debug(`${deletables.length} deletable assets`); debug(`${taggables.length} taggable assets`); + debug(`${untaggables.length} assets to untag`); if (this.permissionToDelete && deletables.length > 0) { await this.parallelDelete(s3, bucket, deletables, printer); @@ -222,19 +227,11 @@ export class GarbageCollector { await this.parallelTag(s3, bucket, taggables, currentTime, printer); } - printer.reportScannedObjects(batch.length); - - // We untag objects that are referenced in ActiveAssets and currently have the Isolated Tag. - let untaggables: S3Asset[] = await Promise.all(notIsolated.map(async (obj) => { - const noTag = await obj.noIsolatedTag(s3); - return noTag ? null : obj; - })).then(results => results.filter((obj): obj is S3Asset => obj !== null)); - - debug(`${untaggables.length} assets to untag`); - if (this.permissionToTag && untaggables.length > 0) { await this.parallelUntag(s3, bucket, untaggables); } + + printer.reportScannedObjects(batch.length); } } catch (err: any) { throw new Error(err); From 4bf31b6fe49b3ecd3792c3a44d2bd039ddcfb590 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 15 Oct 2024 14:23:27 -0400 Subject: [PATCH 49/68] pr comments --- .../api/garbage-collection/garbage-collector.ts | 4 +--- .../lib/api/garbage-collection/stack-refresh.ts | 15 +++++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index fcf7c0a968a09..d6df995b2c92e 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -153,7 +153,6 @@ export class GarbageCollector { const activeAssets = new ActiveAssetCache(); // Grab stack templates first - const startTime = Date.now(); await refreshStacks(cfn, activeAssets, this.maxWaitTime, qualifier); // Start the background refresh const backgroundStackRefresh = new BackgroundStackRefresh({ @@ -162,7 +161,7 @@ export class GarbageCollector { qualifier, maxWaitTime: this.maxWaitTime, }); - const timeout = setTimeout(backgroundStackRefresh.start, Math.max(startTime + 300_000 - Date.now(), 0)); + backgroundStackRefresh.start(); const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); const numObjects = await this.numObjectsInBucket(s3, bucket); @@ -238,7 +237,6 @@ export class GarbageCollector { } finally { backgroundStackRefresh.stop(); printer.stop(); - clearTimeout(timeout); } } diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index e9c428c8ad8ff..745d20c80ae0b 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -35,14 +35,15 @@ async function paginateSdkCall(cb: (nextToken?: string) => Promise s.StackStatus == 'REVIEW_IN_PROGRESS')) { return stacks; } await sleep(Math.floor(Math.random() * sleepMs)); - sleepMs = Math.min(sleepMs * 2, maxWaitTime); + sleepMs = sleepMs * 2; } throw new Error(`Stacks still in REVIEW_IN_PROGRESS state after waiting for ${maxWaitTime} ms.`); @@ -62,7 +63,7 @@ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, await paginateSdkCall(async (nextToken) => { const stacks = await listStacksNotBeingReviewed(cfn, maxWaitTime, nextToken); - // Deleted stacks are ignored + // We ignore stacks with these statuses because their assets are no longer live const ignoredStatues = ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_IN_PROGRESS', 'DELETE_FAILED']; stackNames.push( ...(stacks.StackSummaries ?? []) @@ -165,13 +166,19 @@ export class BackgroundStackRefresh { constructor(private readonly props: BackgroundStackRefreshProps) {} public async start() { + // Since start is going to be called right after the first invocation of refreshStacks, + // lets wait some time before beginning the background refresh. + this.timeout = setTimeout(() => this.refresh(), 300_000); + } + + private async refresh() { const startTime = Date.now(); await refreshStacks(this.props.cfn, this.props.activeAssets, this.props.maxWaitTime ?? 60000, this.props.qualifier); // If the last invocation of refreshStacks takes <5 minutes, the next invocation starts 5 minutes after the last one started. // If the last invocation of refreshStacks takes >5 minutes, the next invocation starts immediately. - this.timeout = setTimeout(this.start, Math.max(startTime + 300_000 - Date.now(), 0)); + this.timeout = setTimeout(() => this.refresh(), Math.max(startTime + 300_000 - Date.now(), 0)); } public stop() { From 4610ac44ff89911ff83fa61bbe79a6587acc0900 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 15 Oct 2024 16:23:49 -0400 Subject: [PATCH 50/68] unit tests for large # of objects --- .../garbage-collection/garbage-collector.ts | 30 ++-- .../api/garbage-collection/stack-refresh.ts | 36 ++++- .../test/api/garbage-collection.test.ts | 150 +++++++++++++++++- 3 files changed, 201 insertions(+), 15 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index d6df995b2c92e..ac86785282e2a 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -181,6 +181,7 @@ export class GarbageCollector { // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. for await (const batch of this.readBucketInBatches(s3, bucket, batchSize, currentTime)) { + await backgroundStackRefresh.noOlderThan(600_000); // 10 mins print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); printer.start(); @@ -311,21 +312,30 @@ export class GarbageCollector { * Delete objects in parallel. The deleteObjects API supports batches of 1000. */ private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[], printer: ProgressPrinter) { + const batchSize = 1000; const objectsToDelete: S3.ObjectIdentifierList = deletables.map(asset => ({ Key: asset.key, })); try { - await s3.deleteObjects({ - Bucket: bucket, - Delete: { - Objects: objectsToDelete, - Quiet: true, - }, - }).promise(); - - debug(`Deleted ${deletables.length} assets`); - printer.reportDeletedObjects(deletables); + const batches = []; + for (let i = 0; i < objectsToDelete.length; i += batchSize) { + batches.push(objectsToDelete.slice(i, i + batchSize)); + } + // Delete objects in batches + for (const batch of batches) { + await s3.deleteObjects({ + Bucket: bucket, + Delete: { + Objects: batch, + Quiet: true, + }, + }).promise(); + + const deletedCount = batch.length; + debug(`Deleted ${deletedCount} assets`); + printer.reportDeletedObjects(deletables.slice(0, deletedCount)); + } } catch (err) { print(chalk.red(`Error deleting objects: ${err}`)); } diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 745d20c80ae0b..3bf92ba300a6c 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -162,8 +162,12 @@ export interface BackgroundStackRefreshProps { */ export class BackgroundStackRefresh { private timeout?: NodeJS.Timeout; + private lastRefreshTime: number; + private queuedPromises: Array<(value: unknown) => void> = []; - constructor(private readonly props: BackgroundStackRefreshProps) {} + constructor(private readonly props: BackgroundStackRefreshProps) { + this.lastRefreshTime = Date.now(); + } public async start() { // Since start is going to be called right after the first invocation of refreshStacks, @@ -175,12 +179,42 @@ export class BackgroundStackRefresh { const startTime = Date.now(); await refreshStacks(this.props.cfn, this.props.activeAssets, this.props.maxWaitTime ?? 60000, this.props.qualifier); + this.justRefreshedStacks(); // If the last invocation of refreshStacks takes <5 minutes, the next invocation starts 5 minutes after the last one started. // If the last invocation of refreshStacks takes >5 minutes, the next invocation starts immediately. this.timeout = setTimeout(() => this.refresh(), Math.max(startTime + 300_000 - Date.now(), 0)); } + private justRefreshedStacks() { + debug("just refreshed stacks"); + this.lastRefreshTime = Date.now(); + for (const p of this.queuedPromises.splice(0, this.queuedPromises.length)) { + p(undefined); + } + } + + /** + * Checks if the last successful background refresh happened within the specified time frame. + * If the last refresh is older than the specified time frame, it returns a Promise that resolves + * when the next background refresh completes or rejects if the refresh takes too long. + */ + public noOlderThan(ms: number) { + const horizon = Date.now() - ms; + + // The last refresh happened within the time frame + if (this.lastRefreshTime >= horizon) { + return Promise.resolve(); + } + + // The last refresh happened earlier than the time frame + // We will wait for the latest refresh to land or reject if it takes too long + return Promise.race([ + new Promise(resolve => this.queuedPromises.push(resolve)), + new Promise((_, reject) => setTimeout(() => reject(new Error('refreshStacks took too long; the background thread likely threw an error')), ms)), + ]); + } + public stop() { clearTimeout(this.timeout); } diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index 6cdda5a0b7977..d71be8d76c4ab 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -13,11 +13,14 @@ let mockGetTemplate: (params: AWS.CloudFormation.Types.GetTemplateInput) => AWS. let mockListObjectsV2: (params: AWS.S3.Types.ListObjectsV2Request) => AWS.S3.Types.ListObjectsV2Output; let mockGetObjectTagging: (params: AWS.S3.Types.GetObjectTaggingRequest) => AWS.S3.Types.GetObjectTaggingOutput; let mockDeleteObjects: (params: AWS.S3.Types.DeleteObjectsRequest) => AWS.S3.Types.DeleteObjectsOutput; +let mockDeleteObjectTagging: (params: AWS.S3.Types.DeleteObjectTaggingRequest) => AWS.S3.Types.DeleteObjectTaggingOutput; let mockPutObjectTagging: (params: AWS.S3.Types.PutObjectTaggingRequest) => AWS.S3.Types.PutObjectTaggingOutput; let stderrMock: jest.SpyInstance; let sdk: MockSdkProvider; +const ISOLATED_TAG = 'aws-cdk:isolated'; + function mockTheToolkitInfo(stackProps: Partial) { const mockSdk = new MockSdk(); (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.fromStack(mockBootstrapStack(mockSdk, stackProps))); @@ -63,11 +66,12 @@ describe('Garbage Collection', () => { }); mockGetObjectTagging = jest.fn().mockImplementation((params) => { return Promise.resolve({ - TagSet: params.Key === 'asset2' ? [{ Key: 'ISOLATED_TAG', Value: new Date().toISOString() }] : [], + TagSet: params.Key === 'asset2' ? [{ Key: ISOLATED_TAG, Value: new Date().toISOString() }] : [], }); }); mockPutObjectTagging = jest.fn(); mockDeleteObjects = jest.fn(); + mockDeleteObjectTagging = jest.fn(); mockDescribeStacks = jest.fn(); sdk.stubCloudFormation({ @@ -80,6 +84,7 @@ describe('Garbage Collection', () => { listObjectsV2: mockListObjectsV2, getObjectTagging: mockGetObjectTagging, deleteObjects: mockDeleteObjects, + deleteObjectTagging: mockDeleteObjectTagging, putObjectTagging: mockPutObjectTagging, }); }); @@ -161,7 +166,7 @@ describe('Garbage Collection', () => { // assets tagged expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); - expect(mockPutObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(2); // one asset already has the tag // no deleting expect(mockDeleteObjects).toHaveBeenCalledTimes(0); @@ -255,7 +260,7 @@ describe('Garbage Collection', () => { // tags objects expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); - expect(mockPutObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(2); // one object already has the tag // no deleting expect(mockDeleteObjects).toHaveBeenCalledTimes(0); @@ -490,7 +495,7 @@ describe('Garbage Collection', () => { // everything else runs as expected: // assets tagged expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); - expect(mockPutObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(2); // one object already has the tag // no deleting expect(mockDeleteObjects).toHaveBeenCalledTimes(0); @@ -538,3 +543,140 @@ describe('Garbage Collection', () => { await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting/); }, 60000); }); + +let mockListObjectsV2Large: (params: AWS.S3.Types.ListObjectsV2Request) => AWS.S3.Types.ListObjectsV2Output; +let mockGetObjectTaggingLarge: (params: AWS.S3.Types.GetObjectTaggingRequest) => AWS.S3.Types.GetObjectTaggingOutput; +describe('Garbage Collection with large # of objects', () => { + const keyCount = 10000; + + beforeEach(() => { + mockListStacks = jest.fn().mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE' }, + ], + }); + mockGetTemplateSummary = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: 'BootstrapVersion', + DefaultValue: '/cdk-bootstrap/abcde/version', + }], + }); + // add every 5th asset hash to the mock template body: 8000 assets are isolated + const mockTemplateBody = []; + for (let i = 0; i < keyCount; i+=5) { + mockTemplateBody.push(`asset${i}hash`); + } + mockGetTemplate = jest.fn().mockReturnValue({ + TemplateBody: mockTemplateBody.join('-'), + }); + + const contents: { Key: string; LastModified: Date }[] = []; + for (let i = 0; i < keyCount; i++) { + contents.push({ + Key: `asset${i}hash`, + LastModified: new Date(0), + }); + } + mockListObjectsV2Large = jest.fn().mockImplementation(() => { + return Promise.resolve({ + Contents: contents, + KeyCount: keyCount, + }); + }); + + // every other object has the isolated tag: of the 8000 isolated assets, 4000 already are tagged. + // of the 2000 in use assets, 1000 are tagged. + mockGetObjectTaggingLarge = jest.fn().mockImplementation((params) => { + return Promise.resolve({ + TagSet: Number(params.Key[params.Key.length - 5]) % 2 === 0 ? [{ Key: ISOLATED_TAG, Value: new Date(2000, 1, 1).toISOString() }] : [], + }); + }); + mockPutObjectTagging = jest.fn(); + mockDeleteObjects = jest.fn(); + mockDeleteObjectTagging = jest.fn(); + mockDescribeStacks = jest.fn(); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + describeStacks: mockDescribeStacks, + }); + sdk.stubS3({ + listObjectsV2: mockListObjectsV2Large, + getObjectTagging: mockGetObjectTaggingLarge, + deleteObjects: mockDeleteObjects, + deleteObjectTagging: mockDeleteObjectTagging, + putObjectTagging: mockPutObjectTagging, + }); + }); + + afterEach(() => { + mockGarbageCollect.mockClear(); + }); + + test('tag only', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'tag', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 1, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2Large).toHaveBeenCalledTimes(2); + + // tagging is performed + expect(mockGetObjectTaggingLarge).toHaveBeenCalledTimes(keyCount); + expect(mockDeleteObjectTagging).toHaveBeenCalledTimes(1000); // 1000 in use assets are erroneously tagged + expect(mockPutObjectTagging).toHaveBeenCalledTimes(5000); // 8000-4000 assets need to be tagged, + 1000 (since untag also calls this) + }); + + test('delete-tagged only', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = new GarbageCollector({ + sdkProvider: sdk, + action: 'delete-tagged', + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: 1, + type: 's3', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2Large).toHaveBeenCalledTimes(2); + + // delete previously tagged objects + expect(mockGetObjectTaggingLarge).toHaveBeenCalledTimes(keyCount); + expect(mockDeleteObjects).toHaveBeenCalledTimes(4); // 4000 isolated assets are already tagged, deleted in batches of 1000 + }); +}); From 770ab8d68f524e7e7f9a07a357ee5173f71d9838 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Tue, 15 Oct 2024 17:02:07 -0400 Subject: [PATCH 51/68] stupid stupid linter --- .../lib/api/garbage-collection/garbage-collector.ts | 12 ++++++------ .../lib/api/garbage-collection/stack-refresh.ts | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index ac86785282e2a..1762e0a95ca4c 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -36,14 +36,14 @@ export class S3Asset { private getTag(tag: string) { if (!this.cached_tags) { - throw new Error("Cannot call getTag before allTags"); + throw new Error('Cannot call getTag before allTags'); } return this.cached_tags.find(t => t.Key === tag)?.Value; } private hasTag(tag: string) { if (!this.cached_tags) { - throw new Error("Cannot call hasTag before allTags"); + throw new Error('Cannot call hasTag before allTags'); } return this.cached_tags.some(t => t.Key === tag); } @@ -161,7 +161,7 @@ export class GarbageCollector { qualifier, maxWaitTime: this.maxWaitTime, }); - backgroundStackRefresh.start(); + void backgroundStackRefresh.start(); const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); const numObjects = await this.numObjectsInBucket(s3, bucket); @@ -203,11 +203,11 @@ export class GarbageCollector { if (graceDays > 0) { debug('Filtering out assets that are not old enough to delete'); await this.parallelReadAllTags(s3, batch); - + // We delete objects that are not referenced in ActiveAssets and have the Isolated Tag with a date // earlier than the current time - grace period. deletables = isolated.filter(obj => obj.isolatedTagBefore(new Date(currentTime - (graceDays * DAY)))); - + // We tag objects that are not referenced in ActiveAssets and do not have the Isolated Tag. taggables = isolated.filter(obj => !obj.hasIsolatedTag()); @@ -266,7 +266,7 @@ export class GarbageCollector { }).promise(), ); - await limit(() => + await limit(() => s3.putObjectTagging({ Bucket: bucket, Key: obj.key, diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 3bf92ba300a6c..1ae4334f29fdb 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -187,7 +187,7 @@ export class BackgroundStackRefresh { } private justRefreshedStacks() { - debug("just refreshed stacks"); + debug('just refreshed stacks'); this.lastRefreshTime = Date.now(); for (const p of this.queuedPromises.splice(0, this.queuedPromises.length)) { p(undefined); From ffc0e18c95fd97d0e144bace057aaf166da9156e Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 08:54:46 -0400 Subject: [PATCH 52/68] pr comment --- .../garbage-collection/garbage-collector.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 1762e0a95ca4c..ef8a2da0f3efe 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -185,14 +185,7 @@ export class GarbageCollector { print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); printer.start(); - const { isolated, notIsolated } = batch.reduce<{ isolated: S3Asset[]; notIsolated: S3Asset[] }>((acc, obj) => { - if (!activeAssets.contains(obj.fileName())) { - acc.isolated.push(obj); - } else { - acc.notIsolated.push(obj); - } - return acc; - }, { isolated: [], notIsolated: [] }); + const { included: isolated, excluded: notIsolated } = partition(batch, asset => !activeAssets.contains(asset.fileName())); debug(`${isolated.length} isolated assets`); @@ -392,3 +385,20 @@ export class GarbageCollector { } while (continuationToken); } } + +function partition(xs: Iterable, pred: (x: A) => boolean): { included: A[]; excluded: A[] } { + const result = { + included: [] as A[], + excluded: [] as A[], + }; + + for (const x of xs) { + if (pred(x)) { + result.included.push(x); + } else { + result.excluded.push(x); + } + } + + return result; +} \ No newline at end of file From b9f92b547bac1447705f04063c9d5d3fa5480335 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 19:03:29 -0400 Subject: [PATCH 53/68] unit tests for background refresh --- .../api/garbage-collection/stack-refresh.ts | 2 +- .../test/api/garbage-collection.test.ts | 109 ++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 1ae4334f29fdb..6576125b4a218 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -172,7 +172,7 @@ export class BackgroundStackRefresh { public async start() { // Since start is going to be called right after the first invocation of refreshStacks, // lets wait some time before beginning the background refresh. - this.timeout = setTimeout(() => this.refresh(), 300_000); + this.timeout = setTimeout(() => this.refresh(), 300_000); // 5 minutes } private async refresh() { diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index d71be8d76c4ab..58bebce04a083 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -3,6 +3,7 @@ const mockGarbageCollect = jest.fn(); import { GarbageCollector, ToolkitInfo } from '../../lib/api'; +import { ActiveAssetCache, BackgroundStackRefresh, BackgroundStackRefreshProps } from '../../lib/api/garbage-collection/stack-refresh'; import { mockBootstrapStack, MockSdk, MockSdkProvider } from '../util/mock-sdk'; let garbageCollector: GarbageCollector; @@ -680,3 +681,111 @@ describe('Garbage Collection with large # of objects', () => { expect(mockDeleteObjects).toHaveBeenCalledTimes(4); // 4000 isolated assets are already tagged, deleted in batches of 1000 }); }); + +describe('BackgroundStackRefresh', () => { + let backgroundRefresh: BackgroundStackRefresh; + let refreshProps: BackgroundStackRefreshProps; + let setTimeoutSpy: jest.SpyInstance; + + beforeEach(() => { + jest.useFakeTimers(); + setTimeoutSpy = jest.spyOn(global, 'setTimeout'); + + mockListStacks = jest.fn().mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + mockGetTemplateSummary = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: 'BootstrapVersion', + DefaultValue: '/cdk-bootstrap/abcde/version', + }], + }); + mockGetTemplate = jest.fn().mockReturnValue({ + TemplateBody: 'abcde', + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + describeStacks: jest.fn(), + }); + + refreshProps = { + cfn: sdk.mockSdk.cloudFormation(), + activeAssets: new ActiveAssetCache(), + maxWaitTime: 60000, // 1 minute + }; + + backgroundRefresh = new BackgroundStackRefresh(refreshProps); + }); + + afterEach(() => { + jest.clearAllTimers(); + setTimeoutSpy.mockRestore(); + }); + + test('should start after a delay', () => { + backgroundRefresh.start(); + expect(setTimeoutSpy).toHaveBeenCalledTimes(1); + expect(setTimeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 300000); + }); + + test('should refresh stacks and schedule next refresh', async () => { + backgroundRefresh.start(); + + // Run the first timer (which should trigger the first refresh) + await jest.runOnlyPendingTimersAsync(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(setTimeoutSpy).toHaveBeenCalledTimes(2); // Once for start, once for next refresh + expect(setTimeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 300000); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + + expect(mockListStacks).toHaveBeenCalledTimes(2); + expect(setTimeoutSpy).toHaveBeenCalledTimes(3); // Two refreshes plus one more scheduled + }); + + test('should wait for the next refresh if called within time frame', async () => { + backgroundRefresh.start(); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + + const waitPromise = backgroundRefresh.noOlderThan(180000); // 3 minutes + jest.advanceTimersByTime(120000); // Advance time by 2 minutes + + await expect(waitPromise).resolves.toBeUndefined(); + }); + + test('should wait for the next refresh if refresh lands before the timeout', async () => { + backgroundRefresh.start(); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + jest.advanceTimersByTime(24000); // Advance time by 4 minutes + + const waitPromise = backgroundRefresh.noOlderThan(300000); // 5 minutes + jest.advanceTimersByTime(120000); // Advance time by 2 minutes, refresh should fire + + await expect(waitPromise).resolves.toBeUndefined(); + }); + + test('should reject if the refresh takes too long', async () => { + backgroundRefresh.start(); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + jest.advanceTimersByTime(120000); // Advance time by 2 minutes + + const waitPromise = backgroundRefresh.noOlderThan(0); // 0 seconds + jest.advanceTimersByTime(120000); // Advance time by 2 minutes + + await expect(waitPromise).rejects.toThrow('refreshStacks took too long; the background thread likely threw an error'); + }); +}); From d9d9b41a270605b7db404ef4f835207cb984755d Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 20:18:46 -0400 Subject: [PATCH 54/68] add prompt before deletion --- .../cli-integ/lib/with-cdk-app.ts | 2 +- .../garbage-collection/garbage-collector.ts | 49 ++++- packages/aws-cdk/lib/cdk-toolkit.ts | 8 + packages/aws-cdk/lib/cli.ts | 1 + .../test/api/garbage-collection.test.ts | 194 ++++++------------ 5 files changed, 119 insertions(+), 135 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index de74549de09cd..29ff598375150 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -489,7 +489,7 @@ export class TestFixture extends ShellHelper { } public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { - const args = ['gc', '--unstable=gc']; // TODO: remove when stabilizing + const args = ['gc', '--unstable=gc', '--skip-delete-prompt=true']; // TODO: remove when stabilizing if (options.rollbackBufferDays) { args.push('--rollback-buffer-days', String(options.rollbackBufferDays)); } diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index ef8a2da0f3efe..876a8ff693bb1 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -5,6 +5,7 @@ import { debug, print } from '../../logging'; import { ISDK, Mode, SdkProvider } from '../aws-auth'; import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; import { ProgressPrinter } from './progress-printer'; +import * as promptly from 'promptly'; import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; // Must use a require() otherwise esbuild complains @@ -81,6 +82,12 @@ interface GarbageCollectorProps { */ readonly rollbackBufferDays: number; + // /** + // * An asset must have been created this number of days ago before being elligible + // * for deletion. + // */ + // readonly createdAtBufferDays: number; + /** * The environment to deploy this stack in * @@ -109,6 +116,13 @@ interface GarbageCollectorProps { * @default 60000 */ readonly maxWaitTime?: number; + + /** + * Skips the prompt before actual deletion happens + * + * @default false + */ + readonly skipDeletePrompt?: boolean; } /** @@ -121,6 +135,7 @@ export class GarbageCollector { private permissionToTag: boolean; private bootstrapStackName: string; private maxWaitTime: number; + private skipDeletePrompt: boolean; public constructor(readonly props: GarbageCollectorProps) { this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); @@ -131,6 +146,7 @@ export class GarbageCollector { this.permissionToDelete = ['delete-tagged', 'full'].includes(props.action); this.permissionToTag = ['tag', 'full'].includes(props.action); this.maxWaitTime = props.maxWaitTime ?? 60000; + this.skipDeletePrompt = props.skipDeletePrompt ?? false; this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; @@ -213,6 +229,23 @@ export class GarbageCollector { debug(`${untaggables.length} assets to untag`); if (this.permissionToDelete && deletables.length > 0) { + if (!this.skipDeletePrompt) { + const message = [ + `Found ${deletables.length} objects to delete based off of the following criteria:`, + `- objects have been isolated for > ${this.props.rollbackBufferDays} days`, + 'Delete this batch (yes/no/delete-all)?', + ].join('\n'); + const response = await promptly.prompt(message, + { trim: true }, + ); + + // Anything other than yes/y/delete-all is treated as no + if (!response || !['yes', 'y', 'delete-all'].includes(response.toLowerCase())) { + throw new Error('Deletion aborted by user'); + } else if (response.toLowerCase() == 'delete-all') { + this.skipDeletePrompt = true; + } + } await this.parallelDelete(s3, bucket, deletables, printer); } @@ -345,8 +378,20 @@ export class GarbageCollector { } private async numObjectsInBucket(s3: S3, bucket: string): Promise { - const response = await s3.listObjectsV2({ Bucket: bucket }).promise(); - return response.KeyCount ?? 0; + let totalCount = 0; + let continuationToken: string | undefined; + + do { + const response = await s3.listObjectsV2({ + Bucket: bucket, + ContinuationToken: continuationToken + }).promise(); + + totalCount += response.KeyCount ?? 0; + continuationToken = response.NextContinuationToken; + } while (continuationToken); + + return totalCount; } /** diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index aba22f8972095..6fe95c5b5a979 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -803,6 +803,7 @@ export class CdkToolkit { rollbackBufferDays: options.rollbackBufferDays, action: options.action ?? 'full', type: options.type ?? 'all', + skipDeletePrompt: options.skipDeletePrompt ?? false, }); await gc.garbageCollect(); }; @@ -1559,6 +1560,13 @@ export interface GarbageCollectionOptions { * @default DEFAULT_TOOLKIT_STACK_NAME */ readonly bootstrapStackName?: string; + + /** + * Skips the prompt before actual deletion begins + * + * @default false + */ + readonly skipDeletePrompt?: boolean; } export interface MigrateOptions { diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index dcd8045a690e5..d21453fd6b6e8 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -120,6 +120,7 @@ async function parseCommandLineArguments(args: string[]) { .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) + .option('skip-delete-prompt', { type: 'boolean', desc: 'Skip manual prompt before deletion', default: false }) .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }), ) .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', (yargs: Argv) => yargs diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index 58bebce04a083..9d874b967104f 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -27,6 +27,28 @@ function mockTheToolkitInfo(stackProps: Partial) { (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.fromStack(mockBootstrapStack(mockSdk, stackProps))); } +function gc(props: { + type: 's3' | 'ecr' | 'all', + bufferDays: number, + action: 'full' | 'print' | 'tag' | 'delete-tagged', + maxWaitTime?: number, +}): GarbageCollector { + return new GarbageCollector({ + sdkProvider: sdk, + action: props.action, + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: props.bufferDays, + type: props.type, + maxWaitTime: props.maxWaitTime, + skipDeletePrompt: true, + }); +} + beforeEach(() => { sdk = new MockSdkProvider({ realSdk: false }); // By default, we'll return a non-found toolkit info @@ -104,17 +126,10 @@ describe('Garbage Collection', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 0, + garbageCollector = gc({ type: 's3', + bufferDays: 0, + action: 'full', }); await garbageCollector.garbageCollect(); @@ -148,17 +163,10 @@ describe('Garbage Collection', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = gc({ type: 's3', + bufferDays: 3, + action: 'full', }); await garbageCollector.garbageCollect(); @@ -183,17 +191,10 @@ describe('Garbage Collection', () => { ], }); - expect(() => new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + expect(() => garbageCollector = gc({ type: 'ecr', + bufferDays: 3, + action: 'full', })).toThrow(/ECR garbage collection is not yet supported/); }); @@ -207,17 +208,10 @@ describe('Garbage Collection', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'print', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 3, + action: 'print', }); await garbageCollector.garbageCollect(); @@ -242,17 +236,10 @@ describe('Garbage Collection', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'tag', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 3, + action: 'tag', }); await garbageCollector.garbageCollect(); @@ -277,17 +264,10 @@ describe('Garbage Collection', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'delete-tagged', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 3, + action: 'delete-tagged', }); await garbageCollector.garbageCollect(); @@ -327,17 +307,10 @@ describe('Garbage Collection', () => { putObjectTagging: mockPutObjectTagging, }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 0, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 0, + action: 'full', }); await garbageCollector.garbageCollect(); @@ -369,17 +342,10 @@ describe('Garbage Collection', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 3, + action: 'full', }); await garbageCollector.garbageCollect(); @@ -410,17 +376,10 @@ describe('Garbage Collection', () => { getTemplate: mockGetTemplate, }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 0, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 0, + action: 'full', }); await garbageCollector.garbageCollect(); @@ -475,19 +434,11 @@ describe('Garbage Collection', () => { getTemplate: mockGetTemplate, }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 3, + action: 'full', }); - await garbageCollector.garbageCollect(); // list are called as expected @@ -527,18 +478,11 @@ describe('Garbage Collection', () => { getTemplate: mockGetTemplate, }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'full', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 3, + garbageCollector = garbageCollector = gc({ type: 's3', - maxWaitTime: 600, // Wait for only 600ms in tests + bufferDays: 3, + action: 'full', + maxWaitTime: 600, // Wait only 600 ms in tests }); await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting/); @@ -626,17 +570,10 @@ describe('Garbage Collection with large # of objects', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'tag', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 1, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 1, + action: 'tag', }); await garbageCollector.garbageCollect(); @@ -659,17 +596,10 @@ describe('Garbage Collection with large # of objects', () => { ], }); - garbageCollector = new GarbageCollector({ - sdkProvider: sdk, - action: 'delete-tagged', - resolvedEnvironment: { - account: '123456789012', - region: 'us-east-1', - name: 'mock', - }, - bootstrapStackName: 'GarbageStack', - rollbackBufferDays: 1, + garbageCollector = garbageCollector = gc({ type: 's3', + bufferDays: 1, + action: 'delete-tagged', }); await garbageCollector.garbageCollect(); From 5d7942f03a2e3e4f254337c717d45482aeb98150 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 20:46:38 -0400 Subject: [PATCH 55/68] createdAtBufferDays --- .../garbage-collection/garbage-collector.ts | 16 ++--- packages/aws-cdk/lib/cdk-toolkit.ts | 6 ++ packages/aws-cdk/lib/cli.ts | 3 + .../test/api/garbage-collection.test.ts | 69 ++++++++++++++----- 4 files changed, 68 insertions(+), 26 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 876a8ff693bb1..1e0ab15e923d2 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -82,11 +82,10 @@ interface GarbageCollectorProps { */ readonly rollbackBufferDays: number; - // /** - // * An asset must have been created this number of days ago before being elligible - // * for deletion. - // */ - // readonly createdAtBufferDays: number; + /** + * Refuse deletion of any assets younger than this number of days. + */ + readonly createdAtBufferDays: number; /** * The environment to deploy this stack in @@ -412,9 +411,10 @@ export class GarbageCollector { response.Contents?.forEach((obj) => { const key = obj.Key ?? ''; const size = obj.Size ?? 0; - const lastModified = obj.LastModified?.getTime() ?? Date.now(); - // Store the object if it has a Key and if it has not been modified since the start of garbage collection - if (key && lastModified < currentTime) { + const lastModified = obj.LastModified ?? new Date(currentTime); + // Store the object if it has a Key and + // if it has not been modified since today - createdAtBufferDays + if (key && lastModified < new Date(currentTime - (this.props.createdAtBufferDays * DAY))) { batch.push(new S3Asset(bucket, key, size)); } }); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 6fe95c5b5a979..9eb1618bf6c51 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -801,6 +801,7 @@ export class CdkToolkit { resolvedEnvironment: environment, bootstrapStackName: options.bootstrapStackName, rollbackBufferDays: options.rollbackBufferDays, + createdAtBufferDays: options.createdAtBufferDays, action: options.action ?? 'full', type: options.type ?? 'all', skipDeletePrompt: options.skipDeletePrompt ?? false, @@ -1554,6 +1555,11 @@ export interface GarbageCollectionOptions { */ readonly rollbackBufferDays: number; + /** + * Refuse deletion of any assets younger than this number of days. + */ + readonly createdAtBufferDays: number; + /** * The stack name of the bootstrap stack. * diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index d21453fd6b6e8..9d6ab8975ae63 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -119,6 +119,7 @@ async function parseCommandLineArguments(args: string[]) { .option('action', { type: 'string', desc: 'The action (or sub-action) you want to perform. Valid entires are "print", "tag", "delete-tagged", "full".', default: 'full' }) .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) + .option('created-at-buffer-days', { type: 'number', desc: 'Skip deletion of any assets that are younger than this many days', default: 1 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) .option('skip-delete-prompt', { type: 'boolean', desc: 'Skip manual prompt before deletion', default: false }) .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }), @@ -695,7 +696,9 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise) { const mockSdk = new MockSdk(); @@ -29,7 +30,8 @@ function mockTheToolkitInfo(stackProps: Partial) { function gc(props: { type: 's3' | 'ecr' | 'all', - bufferDays: number, + rollbackBufferDays?: number, + createdAtBufferDays?: number, action: 'full' | 'print' | 'tag' | 'delete-tagged', maxWaitTime?: number, }): GarbageCollector { @@ -42,7 +44,8 @@ function gc(props: { name: 'mock', }, bootstrapStackName: 'GarbageStack', - rollbackBufferDays: props.bufferDays, + rollbackBufferDays: props.rollbackBufferDays ?? 0, + createdAtBufferDays: props.createdAtBufferDays ?? 0, type: props.type, maxWaitTime: props.maxWaitTime, skipDeletePrompt: true, @@ -80,9 +83,9 @@ describe('Garbage Collection', () => { mockListObjectsV2 = jest.fn().mockImplementation(() => { return Promise.resolve({ Contents: [ - { Key: 'asset1', LastModified: new Date(0) }, - { Key: 'asset2', LastModified: new Date(0) }, - { Key: 'asset3', LastModified: new Date(0) }, + { Key: 'asset1', LastModified: new Date(Date.now() - (2 * DAY)) }, + { Key: 'asset2', LastModified: new Date(Date.now() - (10 * DAY)) }, + { Key: 'asset3', LastModified: new Date(Date.now() - (100 * DAY)) }, ], KeyCount: 3, }); @@ -128,7 +131,7 @@ describe('Garbage Collection', () => { garbageCollector = gc({ type: 's3', - bufferDays: 0, + rollbackBufferDays: 0, action: 'full', }); await garbageCollector.garbageCollect(); @@ -165,7 +168,7 @@ describe('Garbage Collection', () => { garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'full', }); await garbageCollector.garbageCollect(); @@ -193,11 +196,41 @@ describe('Garbage Collection', () => { expect(() => garbageCollector = gc({ type: 'ecr', - bufferDays: 3, + rollbackBufferDays: 3, action: 'full', })).toThrow(/ECR garbage collection is not yet supported/); }); + test('createdAtBufferDays > 0 -- assets to be tagged', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 0, + createdAtBufferDays: 5, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + Quiet: true, + }, + }); + }); + test('action = print -- does not tag or delete', async () => { mockTheToolkitInfo({ Outputs: [ @@ -210,7 +243,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'print', }); await garbageCollector.garbageCollect(); @@ -238,7 +271,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'tag', }); await garbageCollector.garbageCollect(); @@ -266,7 +299,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'delete-tagged', }); await garbageCollector.garbageCollect(); @@ -309,7 +342,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 0, + rollbackBufferDays: 0, action: 'full', }); await garbageCollector.garbageCollect(); @@ -344,7 +377,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'full', }); await garbageCollector.garbageCollect(); @@ -378,7 +411,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 0, + rollbackBufferDays: 0, action: 'full', }); await garbageCollector.garbageCollect(); @@ -436,7 +469,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'full', }); await garbageCollector.garbageCollect(); @@ -480,7 +513,7 @@ describe('Garbage Collection', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 3, + rollbackBufferDays: 3, action: 'full', maxWaitTime: 600, // Wait only 600 ms in tests }); @@ -572,7 +605,7 @@ describe('Garbage Collection with large # of objects', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 1, + rollbackBufferDays: 1, action: 'tag', }); await garbageCollector.garbageCollect(); @@ -598,7 +631,7 @@ describe('Garbage Collection with large # of objects', () => { garbageCollector = garbageCollector = gc({ type: 's3', - bufferDays: 1, + rollbackBufferDays: 1, action: 'delete-tagged', }); await garbageCollector.garbageCollect(); From 4ce3a1aa5e3d4486c78267d4f7433c23e5a4ad37 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 20:48:28 -0400 Subject: [PATCH 56/68] lint --- .../garbage-collection/garbage-collector.ts | 12 +++++----- .../test/api/garbage-collection.test.ts | 24 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 1e0ab15e923d2..4688a2e06610d 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -1,11 +1,11 @@ import * as cxapi from '@aws-cdk/cx-api'; import { S3 } from 'aws-sdk'; import * as chalk from 'chalk'; +import * as promptly from 'promptly'; import { debug, print } from '../../logging'; import { ISDK, Mode, SdkProvider } from '../aws-auth'; import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; import { ProgressPrinter } from './progress-printer'; -import * as promptly from 'promptly'; import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; // Must use a require() otherwise esbuild complains @@ -229,7 +229,7 @@ export class GarbageCollector { if (this.permissionToDelete && deletables.length > 0) { if (!this.skipDeletePrompt) { - const message = [ + const message = [ `Found ${deletables.length} objects to delete based off of the following criteria:`, `- objects have been isolated for > ${this.props.rollbackBufferDays} days`, 'Delete this batch (yes/no/delete-all)?', @@ -379,17 +379,17 @@ export class GarbageCollector { private async numObjectsInBucket(s3: S3, bucket: string): Promise { let totalCount = 0; let continuationToken: string | undefined; - + do { const response = await s3.listObjectsV2({ Bucket: bucket, - ContinuationToken: continuationToken + ContinuationToken: continuationToken, }).promise(); - + totalCount += response.KeyCount ?? 0; continuationToken = response.NextContinuationToken; } while (continuationToken); - + return totalCount; } diff --git a/packages/aws-cdk/test/api/garbage-collection.test.ts b/packages/aws-cdk/test/api/garbage-collection.test.ts index e36d36331409d..339fc5d46e049 100644 --- a/packages/aws-cdk/test/api/garbage-collection.test.ts +++ b/packages/aws-cdk/test/api/garbage-collection.test.ts @@ -29,11 +29,11 @@ function mockTheToolkitInfo(stackProps: Partial) { } function gc(props: { - type: 's3' | 'ecr' | 'all', - rollbackBufferDays?: number, - createdAtBufferDays?: number, - action: 'full' | 'print' | 'tag' | 'delete-tagged', - maxWaitTime?: number, + type: 's3' | 'ecr' | 'all'; + rollbackBufferDays?: number; + createdAtBufferDays?: number; + action: 'full' | 'print' | 'tag' | 'delete-tagged'; + maxWaitTime?: number; }): GarbageCollector { return new GarbageCollector({ sdkProvider: sdk, @@ -214,7 +214,7 @@ describe('Garbage Collection', () => { garbageCollector = gc({ type: 's3', rollbackBufferDays: 0, - createdAtBufferDays: 5, + createdAtBufferDays: 5, action: 'full', }); await garbageCollector.garbageCollect(); @@ -692,13 +692,13 @@ describe('BackgroundStackRefresh', () => { }); test('should start after a delay', () => { - backgroundRefresh.start(); + void backgroundRefresh.start(); expect(setTimeoutSpy).toHaveBeenCalledTimes(1); expect(setTimeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 300000); }); test('should refresh stacks and schedule next refresh', async () => { - backgroundRefresh.start(); + void backgroundRefresh.start(); // Run the first timer (which should trigger the first refresh) await jest.runOnlyPendingTimersAsync(); @@ -715,7 +715,7 @@ describe('BackgroundStackRefresh', () => { }); test('should wait for the next refresh if called within time frame', async () => { - backgroundRefresh.start(); + void backgroundRefresh.start(); // Run the first timer (which triggers the first refresh) await jest.runOnlyPendingTimersAsync(); @@ -727,7 +727,7 @@ describe('BackgroundStackRefresh', () => { }); test('should wait for the next refresh if refresh lands before the timeout', async () => { - backgroundRefresh.start(); + void backgroundRefresh.start(); // Run the first timer (which triggers the first refresh) await jest.runOnlyPendingTimersAsync(); @@ -740,7 +740,7 @@ describe('BackgroundStackRefresh', () => { }); test('should reject if the refresh takes too long', async () => { - backgroundRefresh.start(); + void backgroundRefresh.start(); // Run the first timer (which triggers the first refresh) await jest.runOnlyPendingTimersAsync(); @@ -748,7 +748,7 @@ describe('BackgroundStackRefresh', () => { const waitPromise = backgroundRefresh.noOlderThan(0); // 0 seconds jest.advanceTimersByTime(120000); // Advance time by 2 minutes - + await expect(waitPromise).rejects.toThrow('refreshStacks took too long; the background thread likely threw an error'); }); }); From e4c290cefe25deb3eb59a3d80eb09c59a6116663 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 21:01:46 -0400 Subject: [PATCH 57/68] readme update --- packages/aws-cdk/README.md | 34 ++++++++++++++++--- .../garbage-collection/garbage-collector.ts | 2 ++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index fdccd26afb6bb..1625ff5defd1c 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -901,13 +901,39 @@ cdk gc --unstable=gc --type=s3 This will garbage collect S3 assets from the current bootstrapped environment(s) and immediately delete them. Note that, since the default bootstrap S3 Bucket is versioned, object deletion will be handled by the lifecycle policy on the bucket. -If you are concerned about deleting assets too aggressively, you can configure a buffer amount of days to keep -an unused asset before deletion. In this scenario, instead of deleting unused objects, `cdk gc` will tag -them with today's date instead. It will also check if any objects have been tagged by previous runs of `cdk gc` +Before we begin to delete your assets, you will be prompted: + +```console +cdk gc --unstable=gc --type=s3 + +Found X objects to delete based off of the following criteria: +- objects have been isolated for > 0 days +- objects were created > 1 days ago + +Delete this batch (yes/no/delete-all)? +``` + +Since it's quite possible that the bootstrap bucket has many objects, we work in batches of 1000 objects. To skip the +prompt either reply with `delete-all`, or use the `--skip-delete-prompt=true` option. + +```console +cdk gc --unstable=gc --type=s3 --skip-delete-prompt=true +``` + +If you are concerned about deleting assets too aggressively, there are multiple levers you can configure: + +- rollback-buffer-days: this is the amount of days an asset has to be marked as isolated before it is elligible for deletion. +- created-at-buffer-days: this is the amount of days an asset must live before it is elligible for deletion. + +When using `rollback-buffer-days`, instead of deleting unused objects, `cdk gc` will tag them with +today's date instead. It will also check if any objects have been tagged by previous runs of `cdk gc` and delete them if they have been tagged for longer than the buffer days. +When using `created-at-buffer-days`, we simply filter out any assets that have not persisted that number +of days. + ```console -cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 +cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 --created-at-buffer-days=1 ``` You can also configure the scope that `cdk gc` performs via the `--action` option. By default, all actions diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 4688a2e06610d..0a642997ba804 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -232,6 +232,8 @@ export class GarbageCollector { const message = [ `Found ${deletables.length} objects to delete based off of the following criteria:`, `- objects have been isolated for > ${this.props.rollbackBufferDays} days`, + `- objects were created > ${this.props.createdAtBufferDays} days ago`, + '', 'Delete this batch (yes/no/delete-all)?', ].join('\n'); const response = await promptly.prompt(message, From ba28cf2297cf1900fcbe452dbe807a7ddb171dba Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 21:09:46 -0400 Subject: [PATCH 58/68] typos --- .../lib/api/garbage-collection/stack-refresh.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 6576125b4a218..5b28e41ee60ae 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -50,14 +50,14 @@ async function listStacksNotBeingReviewed(cfn: CloudFormation, maxWaitTime: numb } /** - * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: - * - stacks in DELETE_COMPLETE or DELETE_IN_PROGRES stage - * - stacks that are using a different bootstrap qualifier - * - * It fails on the following stacks because we cannot get the template and therefore have an imcomplete - * understanding of what assets are being used. - * - stacks in REVIEW_IN_PROGRESS stage - */ + * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: + * - stacks in DELETE_COMPLETE or DELETE_IN_PROGRESS stage + * - stacks that are using a different bootstrap qualifier + * + * It fails on the following stacks because we cannot get the template and therefore have an imcomplete + * understanding of what assets are being used. + * - stacks in REVIEW_IN_PROGRESS stage + */ async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, qualifier?: string) { const stackNames: string[] = []; await paginateSdkCall(async (nextToken) => { From 099c327f192ab16ca5589b39bf84d662281ad9be Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Wed, 16 Oct 2024 19:50:03 -0700 Subject: [PATCH 59/68] pause --- .../garbage-collection/garbage-collector.ts | 4 ++++ .../garbage-collection/progress-printer.ts | 22 ++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 0a642997ba804..d1b3793b24f22 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -203,6 +203,8 @@ export class GarbageCollector { const { included: isolated, excluded: notIsolated } = partition(batch, asset => !activeAssets.contains(asset.fileName())); debug(`${isolated.length} isolated assets`); + debug(`${notIsolated.length} not isolated assets`); + debug(`${batch.length} objects total`); let deletables: S3Asset[] = isolated; let taggables: S3Asset[] = []; @@ -236,6 +238,7 @@ export class GarbageCollector { '', 'Delete this batch (yes/no/delete-all)?', ].join('\n'); + printer.pause(); const response = await promptly.prompt(message, { trim: true }, ); @@ -247,6 +250,7 @@ export class GarbageCollector { this.skipDeletePrompt = true; } } + printer.resume(); await this.parallelDelete(s3, bucket, deletables, printer); } diff --git a/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts index 0f0158aa56d5d..441bb5fb3977c 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts @@ -11,6 +11,7 @@ export class ProgressPrinter { private deletedObjectsSizeMb: number; private interval: number; private setInterval?: NodeJS.Timer; + private isPaused: boolean; constructor(totalObjects: number, interval?: number) { this.totalObjects = totalObjects; @@ -20,6 +21,7 @@ export class ProgressPrinter { this.deletedObjects = 0; this.deletedObjectsSizeMb = 0; this.interval = interval ?? 10_000; + this.isPaused = false; } public reportScannedObjects(amt: number) { @@ -39,13 +41,27 @@ export class ProgressPrinter { } public start() { - this.setInterval = setInterval(() => this.print(), this.interval); + this.setInterval = setInterval(() => { + if (!this.isPaused) { + this.print(); + } + }, this.interval); + } + + public pause() { + this.isPaused = true; + } + + public resume() { + this.isPaused = false; } public stop() { clearInterval(this.setInterval); - // print one last time - this.print(); + // print one last time if not paused + if (!this.isPaused) { + this.print(); + } } private print() { From b1098233465d7a0b6d0be94611c908f5ad6076be Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Thu, 17 Oct 2024 10:38:05 -0700 Subject: [PATCH 60/68] integ test fix --- packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 29ff598375150..70e8f719fe5fc 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -489,7 +489,12 @@ export class TestFixture extends ShellHelper { } public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { - const args = ['gc', '--unstable=gc', '--skip-delete-prompt=true']; // TODO: remove when stabilizing + const args = [ + 'gc', + '--unstable=gc', // TODO: remove when stabilizing + '--skip-delete-prompt=true', + '--created-at-buffer-days=0', // Otherwise all assets created during integ tests are too young + ]; if (options.rollbackBufferDays) { args.push('--rollback-buffer-days', String(options.rollbackBufferDays)); } From 09f212b7c41dba9f577ca02d3650714ffcef6386 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy <36202692+kaizencc@users.noreply.github.com> Date: Mon, 21 Oct 2024 09:50:26 -0400 Subject: [PATCH 61/68] Update packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts --- packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 5b28e41ee60ae..5d6e01b432486 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -187,7 +187,6 @@ export class BackgroundStackRefresh { } private justRefreshedStacks() { - debug('just refreshed stacks'); this.lastRefreshTime = Date.now(); for (const p of this.queuedPromises.splice(0, this.queuedPromises.length)) { p(undefined); From 533797ad91e1296d9d0a0fa6e739f3e2717a1d06 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy <36202692+kaizencc@users.noreply.github.com> Date: Mon, 21 Oct 2024 10:10:30 -0400 Subject: [PATCH 62/68] Update packages/aws-cdk/lib/cli.ts Co-authored-by: Rico Hermans --- packages/aws-cdk/lib/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 9d6ab8975ae63..aeb52753abb29 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -119,7 +119,7 @@ async function parseCommandLineArguments(args: string[]) { .option('action', { type: 'string', desc: 'The action (or sub-action) you want to perform. Valid entires are "print", "tag", "delete-tagged", "full".', default: 'full' }) .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) - .option('created-at-buffer-days', { type: 'number', desc: 'Skip deletion of any assets that are younger than this many days', default: 1 }) + .option('created-buffer-days', { type: 'number', desc: 'Never delete assets younger than this (in days)', default: 1 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) .option('skip-delete-prompt', { type: 'boolean', desc: 'Skip manual prompt before deletion', default: false }) .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }), From 6056bf6fc1f3bc7c664dab0a4b5a6411ba27f8a9 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy <36202692+kaizencc@users.noreply.github.com> Date: Mon, 21 Oct 2024 10:10:38 -0400 Subject: [PATCH 63/68] Update packages/aws-cdk/lib/cli.ts Co-authored-by: Rico Hermans --- packages/aws-cdk/lib/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index aeb52753abb29..abc8a5ea333b3 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -122,7 +122,7 @@ async function parseCommandLineArguments(args: string[]) { .option('created-buffer-days', { type: 'number', desc: 'Never delete assets younger than this (in days)', default: 1 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) .option('skip-delete-prompt', { type: 'boolean', desc: 'Skip manual prompt before deletion', default: false }) - .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack to create', requiresArg: true }), + .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack, if different from the default "CDKToolkit"', requiresArg: true }), ) .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', (yargs: Argv) => yargs .option('all', { type: 'boolean', default: false, desc: 'Deploy all available stacks' }) From 3736e37f9080ab924b8e9d80fec5acbb9de337cf Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 21 Oct 2024 07:11:03 -0700 Subject: [PATCH 64/68] start is not async --- .../aws-cdk/lib/api/garbage-collection/garbage-collector.ts | 2 +- packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index d1b3793b24f22..7edb79f1927db 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -176,7 +176,7 @@ export class GarbageCollector { qualifier, maxWaitTime: this.maxWaitTime, }); - void backgroundStackRefresh.start(); + backgroundStackRefresh.start(); const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); const numObjects = await this.numObjectsInBucket(s3, bucket); diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts index 5d6e01b432486..14ba12069a130 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -169,7 +169,7 @@ export class BackgroundStackRefresh { this.lastRefreshTime = Date.now(); } - public async start() { + public start() { // Since start is going to be called right after the first invocation of refreshStacks, // lets wait some time before beginning the background refresh. this.timeout = setTimeout(() => this.refresh(), 300_000); // 5 minutes From 9deae15840ed0674496aa554cf7f1e775af122dd Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 21 Oct 2024 07:22:23 -0700 Subject: [PATCH 65/68] rename cli options --- .../cli-integ/lib/with-cdk-app.ts | 4 ++-- packages/aws-cdk/README.md | 10 +++++----- .../garbage-collection/garbage-collector.ts | 18 +++++++++--------- packages/aws-cdk/lib/cdk-toolkit.ts | 8 ++++---- packages/aws-cdk/lib/cli.ts | 6 +++--- .../test/api/garbage-collection.test.ts | 4 ++-- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 70e8f719fe5fc..33fea7ed959cd 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -492,8 +492,8 @@ export class TestFixture extends ShellHelper { const args = [ 'gc', '--unstable=gc', // TODO: remove when stabilizing - '--skip-delete-prompt=true', - '--created-at-buffer-days=0', // Otherwise all assets created during integ tests are too young + '--confirm=false', + '--created-buffer-days=0', // Otherwise all assets created during integ tests are too young ]; if (options.rollbackBufferDays) { args.push('--rollback-buffer-days', String(options.rollbackBufferDays)); diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 1625ff5defd1c..deec121451f77 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -914,26 +914,26 @@ Delete this batch (yes/no/delete-all)? ``` Since it's quite possible that the bootstrap bucket has many objects, we work in batches of 1000 objects. To skip the -prompt either reply with `delete-all`, or use the `--skip-delete-prompt=true` option. +prompt either reply with `delete-all`, or use the `--confirm=false` option. ```console -cdk gc --unstable=gc --type=s3 --skip-delete-prompt=true +cdk gc --unstable=gc --type=s3 --confirm=false ``` If you are concerned about deleting assets too aggressively, there are multiple levers you can configure: - rollback-buffer-days: this is the amount of days an asset has to be marked as isolated before it is elligible for deletion. -- created-at-buffer-days: this is the amount of days an asset must live before it is elligible for deletion. +- created-buffer-days: this is the amount of days an asset must live before it is elligible for deletion. When using `rollback-buffer-days`, instead of deleting unused objects, `cdk gc` will tag them with today's date instead. It will also check if any objects have been tagged by previous runs of `cdk gc` and delete them if they have been tagged for longer than the buffer days. -When using `created-at-buffer-days`, we simply filter out any assets that have not persisted that number +When using `created-buffer-days`, we simply filter out any assets that have not persisted that number of days. ```console -cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 --created-at-buffer-days=1 +cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 --created-buffer-days=1 ``` You can also configure the scope that `cdk gc` performs via the `--action` option. By default, all actions diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index 7edb79f1927db..e0bf9ff395728 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -85,7 +85,7 @@ interface GarbageCollectorProps { /** * Refuse deletion of any assets younger than this number of days. */ - readonly createdAtBufferDays: number; + readonly createdBufferDays: number; /** * The environment to deploy this stack in @@ -121,7 +121,7 @@ interface GarbageCollectorProps { * * @default false */ - readonly skipDeletePrompt?: boolean; + readonly confirm?: boolean; } /** @@ -134,7 +134,7 @@ export class GarbageCollector { private permissionToTag: boolean; private bootstrapStackName: string; private maxWaitTime: number; - private skipDeletePrompt: boolean; + private confirm: boolean; public constructor(readonly props: GarbageCollectorProps) { this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); @@ -145,7 +145,7 @@ export class GarbageCollector { this.permissionToDelete = ['delete-tagged', 'full'].includes(props.action); this.permissionToTag = ['tag', 'full'].includes(props.action); this.maxWaitTime = props.maxWaitTime ?? 60000; - this.skipDeletePrompt = props.skipDeletePrompt ?? false; + this.confirm = props.confirm ?? true; this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; @@ -230,11 +230,11 @@ export class GarbageCollector { debug(`${untaggables.length} assets to untag`); if (this.permissionToDelete && deletables.length > 0) { - if (!this.skipDeletePrompt) { + if (this.confirm) { const message = [ `Found ${deletables.length} objects to delete based off of the following criteria:`, `- objects have been isolated for > ${this.props.rollbackBufferDays} days`, - `- objects were created > ${this.props.createdAtBufferDays} days ago`, + `- objects were created > ${this.props.createdBufferDays} days ago`, '', 'Delete this batch (yes/no/delete-all)?', ].join('\n'); @@ -247,7 +247,7 @@ export class GarbageCollector { if (!response || !['yes', 'y', 'delete-all'].includes(response.toLowerCase())) { throw new Error('Deletion aborted by user'); } else if (response.toLowerCase() == 'delete-all') { - this.skipDeletePrompt = true; + this.confirm = true; } } printer.resume(); @@ -419,8 +419,8 @@ export class GarbageCollector { const size = obj.Size ?? 0; const lastModified = obj.LastModified ?? new Date(currentTime); // Store the object if it has a Key and - // if it has not been modified since today - createdAtBufferDays - if (key && lastModified < new Date(currentTime - (this.props.createdAtBufferDays * DAY))) { + // if it has not been modified since today - createdBufferDays + if (key && lastModified < new Date(currentTime - (this.props.createdBufferDays * DAY))) { batch.push(new S3Asset(bucket, key, size)); } }); diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 9eb1618bf6c51..64a9a0b4dd20c 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -801,10 +801,10 @@ export class CdkToolkit { resolvedEnvironment: environment, bootstrapStackName: options.bootstrapStackName, rollbackBufferDays: options.rollbackBufferDays, - createdAtBufferDays: options.createdAtBufferDays, + createdBufferDays: options.createdBufferDays, action: options.action ?? 'full', type: options.type ?? 'all', - skipDeletePrompt: options.skipDeletePrompt ?? false, + confirm: options.confirm ?? true, }); await gc.garbageCollect(); }; @@ -1558,7 +1558,7 @@ export interface GarbageCollectionOptions { /** * Refuse deletion of any assets younger than this number of days. */ - readonly createdAtBufferDays: number; + readonly createdBufferDays: number; /** * The stack name of the bootstrap stack. @@ -1572,7 +1572,7 @@ export interface GarbageCollectionOptions { * * @default false */ - readonly skipDeletePrompt?: boolean; + readonly confirm?: boolean; } export interface MigrateOptions { diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index abc8a5ea333b3..090be38ebf6a2 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -121,7 +121,7 @@ async function parseCommandLineArguments(args: string[]) { .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) .option('created-buffer-days', { type: 'number', desc: 'Never delete assets younger than this (in days)', default: 1 }) .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) - .option('skip-delete-prompt', { type: 'boolean', desc: 'Skip manual prompt before deletion', default: false }) + .option('confirm', { alias: 'c', type: 'boolean', desc: 'Confirm via manual prompt before deletion', default: true }) .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack, if different from the default "CDKToolkit"', requiresArg: true }), ) .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', (yargs: Argv) => yargs @@ -696,9 +696,9 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Mon, 21 Oct 2024 08:59:00 -0700 Subject: [PATCH 66/68] global unstable --- packages/aws-cdk/lib/cli.ts | 7 +++---- packages/aws-cdk/lib/settings.ts | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 090be38ebf6a2..1cc58f1312435 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -86,6 +86,7 @@ async function parseCommandLineArguments(args: string[]) { .option('notices', { type: 'boolean', desc: 'Show relevant notices' }) .option('no-color', { type: 'boolean', desc: 'Removes colors and other style from console output', default: false }) .option('ci', { type: 'boolean', desc: 'Force CI detection. If CI=true then logs will be sent to stdout instead of stderr', default: process.env.CI !== undefined }) + .option('unstable', { type: 'array', desc: 'Opt in to specific unstable features. Can be specified multiple times.', default: [] }) .command(['list [STACKS..]', 'ls [STACKS..]'], 'Lists all stacks in the app', (yargs: Argv) => yargs .option('long', { type: 'boolean', default: false, alias: 'l', desc: 'Display environment information for each stack' }) .option('show-dependencies', { type: 'boolean', default: false, alias: 'd', desc: 'Display stack dependency information for each stack' }), @@ -115,13 +116,11 @@ async function parseCommandLineArguments(args: string[]) { .option('previous-parameters', { type: 'boolean', default: true, desc: 'Use previous values for existing parameters (you must specify all parameters on every deployment if this is disabled)' }), ) .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs - .option('unstable', { type: 'array', desc: 'Opt in to specific unstable features. Can be specified multiple times.', default: [] }) .option('action', { type: 'string', desc: 'The action (or sub-action) you want to perform. Valid entires are "print", "tag", "delete-tagged", "full".', default: 'full' }) .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) .option('created-buffer-days', { type: 'number', desc: 'Never delete assets younger than this (in days)', default: 1 }) - .option('qualifier', { type: 'string', desc: 'String which must be unique for each bootstrap stack. You must configure it on your CDK app if you change this from the default.', default: undefined }) - .option('confirm', { alias: 'c', type: 'boolean', desc: 'Confirm via manual prompt before deletion', default: true }) + .option('confirm', { type: 'boolean', desc: 'Confirm via manual prompt before deletion', default: true }) .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack, if different from the default "CDKToolkit"', requiresArg: true }), ) .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', (yargs: Argv) => yargs @@ -689,7 +688,7 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise Date: Mon, 21 Oct 2024 09:01:49 -0700 Subject: [PATCH 67/68] Apply suggestions from code review --- .../aws-cdk/lib/api/garbage-collection/garbage-collector.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts index e0bf9ff395728..7fe512805e0fc 100644 --- a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -117,9 +117,9 @@ interface GarbageCollectorProps { readonly maxWaitTime?: number; /** - * Skips the prompt before actual deletion happens + * Confirm with the user before actual deletion happens * - * @default false + * @default true */ readonly confirm?: boolean; } @@ -247,7 +247,7 @@ export class GarbageCollector { if (!response || !['yes', 'y', 'delete-all'].includes(response.toLowerCase())) { throw new Error('Deletion aborted by user'); } else if (response.toLowerCase() == 'delete-all') { - this.confirm = true; + this.confirm = false; } } printer.resume(); From 59edf16956576c3d5d911169f0dbbee0495c7c51 Mon Sep 17 00:00:00 2001 From: Kaizen Conroy Date: Mon, 21 Oct 2024 10:59:16 -0700 Subject: [PATCH 68/68] remove console statement --- packages/aws-cdk/lib/settings.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/aws-cdk/lib/settings.ts b/packages/aws-cdk/lib/settings.ts index 1ba8f66d64036..74684dc3501d3 100644 --- a/packages/aws-cdk/lib/settings.ts +++ b/packages/aws-cdk/lib/settings.ts @@ -292,7 +292,7 @@ export class Settings { assetParallelism: argv['asset-parallelism'], assetPrebuild: argv['asset-prebuild'], ignoreNoStacks: argv['ignore-no-stacks'], - unstable: argv['unstable'], + unstable: argv.unstable, }); } @@ -308,7 +308,6 @@ export class Settings { const context: any = {}; for (const assignment of ((argv as any).context || [])) { - console.log((argv as any).context, assignment); const parts = assignment.split(/=(.*)/, 2); if (parts.length === 2) { debug('CLI argument context: %s=%s', parts[0], parts[1]);