From 3dc79e7042c1dd2eb9b40e04796a837439700918 Mon Sep 17 00:00:00 2001 From: HBobertz Date: Tue, 28 Jan 2025 23:30:59 -0500 Subject: [PATCH 1/6] update markdown files --- ARCHITECTURE.MD | 139 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 35 ++++++++---- 2 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 ARCHITECTURE.MD diff --git a/ARCHITECTURE.MD b/ARCHITECTURE.MD new file mode 100644 index 00000000..97bb3515 --- /dev/null +++ b/ARCHITECTURE.MD @@ -0,0 +1,139 @@ +# cdk-from-cfn Architecture Overview + +This document provides a high level overview of the key components in the cdk-from-cfn project and their responsibilities + +## Processing Flow + +The easiest way to understand the purpose behind the core components of `cdk-from-cfj`, is to look at the high-level processing flow +of transforming a Cloudformation Template into a CDK Application: +``` +CloudFormation Template + ↓ (Parser) +Cloudformation Parse Tree + ↓ (IR) +Cloudformation Program Ir + ↓ (Synthesizer) +Generated CDK Code +``` + +Put simply, the idea behind the cdk-from-cfn processing flow is to transform a +CFN template into an IR, and then use that IR plus a language synthesizer to generate a CDK Application. + +To do so, the parser reads the text from the provided Cloudformation template and generates a simple, un-validated, data structure +called the `CloudformationParseTree` which is essentially a 1:1 in memory representation of the provided template + +However the `CloudformationParseTree` is not sufficient for application synthesis and needs to be enriched with additional +information and validated against the schema for correctness. This is the purpose of the `IR` which generates a `CloudformationProgramIr`. + +A language specific synthesizer can then read the IR and use it to generate CDK stack code. + +In rust code this flow would look like: +```rust +// 1. Parse CloudFormation template +let cfn_tree: CloudformationParseTree = serde_yaml::from_str(template)?; + +// 2. Convert to IR +let ir = CloudformationProgramIr::from(cfn_tree, schema)?; + +// 3. Synthesize to target language +let synthesizer = TypeScript {}; +ir.synthesize(synthesizer, output, stack_name)?; +``` + +## Core Components Overview + +### Parser + +Responsible for de-serializing the Cloudformation template into a `CloudformationParseTree` data structure. The parser's sole responsibility is creating a 1:1 representation of the template structure, without performing any validation or enhancement. + +Key responsibilities: +- Reads YAML/JSON Cloudformation templates +- Creates corresponding Rust data structures +- Parses all template sections (Resources, Parameters, Conditions, etc.) +- Handles Cloudformation intrinsic functions (Fn::Join, Ref, etc.) + +```rust +// CloudFormation Template +{ + "Resources": { + "MyBucket": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketName": { + "Fn::Join": ["-", ["mybucket", {"Ref": "AWS::Region"}]] + } + } + } + } +} + +// Parse Tree - Just maps the structure directly +CloudformationParseTree { + resources: { + "MyBucket" => ResourceAttributes { + resource_type: "AWS::S3::Bucket", + properties: { + "BucketName" => ResourceValue::IntrinsicFunction( + Fn::Join { + sep: "-", + list: ["mybucket", Ref("AWS::Region")] + } + ) + } + } + } +} +``` + +### IR (Intermediate Representation) + +Responsible for converting the raw ParseTree into a validated and enriched `CloudformationProgramIr` data structure. The IR's main purpose is to add necessary CDK-specific information and validate resources against the CDK schema before code generation. + +Key responsibilities: +- Validates resources and properties against CDK schema +- Resolves references and tracks dependencies between resources +- Enhances references with type and origin information +- Orders resources based on dependencies +- Adds necessary CDK imports and type information + +```rust +// IR - Adds understanding and validation +CloudformationProgramIr { + resources: [ + ResourceInstruction { + name: "MyBucket", + resource_type: ResourceType::AWS { + service: "S3", + type_name: "Bucket" + }, + properties: { + "bucketName": ResourceIr::Join( + "-", + [ + ResourceIr::String("mybucket"), + ResourceIr::Ref(Reference { + origin: Origin::PseudoParameter(Region), + name: "AWS::Region" + }) + ] + ) + }, + // Additional validated information: + references: ["AWS::Region"], // Tracks dependencies + condition: None, + deletion_policy: None + } + ] +} +``` + +### Synthesizer + +Responsible for converting the validated IR into CDK code in a specific target language. The synthesizer's main purpose is to generate idiomatic code that properly represents the CloudFormation resources as CDK constructs. + +Key responsibilities: +- Converts IR constructs to language-specific CDK code +- Manages imports and module dependencies +- Handles language-specific code patterns (builders, properties, etc.) +- Generates properly formatted and structured code +- Implements string interpolation and type conversion for each language diff --git a/README.md b/README.md index fdce0554..ed3b3c89 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,38 @@ # cdk-from-cfn -In a world where people want to use the full extent of the cdk, there **was** no product that would transform all your -JSON/YAML into beautiful typescript...until now. +`cdk-from-cfn` is a command-line tool that converts AWS CloudFormation templates into AWS CDK code. It supports multiple programming languages including TypeScript, Python, Java, Go, and C#. -`cdk-from-cfn` will take your JSON/YAML and output the equivalent typescript. +## Installation -## User Guide +```console +$ cargo install cdk-from-cfn +``` + +## Usage ```console -$ cargo build --release -$ ./target/release/cdk-from-cfn [INPUT] [OUTPUT] +$ cdk-from-cfn [INPUT] [OUTPUT] --language --stack-name ``` - `INPUT` is the input file path (STDIN by default). - `OUTPUT` is the output file path; if not specified, output will be printed on your command line (STDOUT by default). -### Cargo Features +## Node Usage + +cdk-from-cfn is also vended as a module on [npm](https://www.npmjs.com/package/cdk-from-cfn) and exposes some apis which can be used +in a node project. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. + +```typescript +import * as cdk_from_cfn from 'cdk-from-cfn'; + +// get supported languages +cdk_from_cfn.supported_languages(); + +// transmute cfn template into cdk app +cdk_from_cfn.transmute(template, language, stackName) +``` + +## Language and Feature support Name | Enabled by default | Description -------------|:------------------:|--------------------------------------------- @@ -39,7 +56,7 @@ $ cargo build --release --no-default-features --features=golang Finished release [optimized] target(s) in 0.17s ``` -## Implemented +### Implemented - [x] Fn::FindInMap - [x] Fn::Join @@ -63,7 +80,7 @@ Finished release [optimized] target(s) in 0.17s - [x] Deletion policy - [x] Fn::Cidr support -## Remaining +### Remaining There are known unsupported features. Working on them in priority order: From 3eaaebdf02485ac0d9af5d67b3a4765c388c3539 Mon Sep 17 00:00:00 2001 From: HBobertz Date: Wed, 29 Jan 2025 00:03:43 -0500 Subject: [PATCH 2/6] update readmes --- ARCHITECTURE.MD | 2 +- CONTRIBUTING.md | 62 +++++++++++++++++++++---------------------------- 2 files changed, 28 insertions(+), 36 deletions(-) diff --git a/ARCHITECTURE.MD b/ARCHITECTURE.MD index 97bb3515..94b0bbfe 100644 --- a/ARCHITECTURE.MD +++ b/ARCHITECTURE.MD @@ -97,7 +97,7 @@ Key responsibilities: - Adds necessary CDK imports and type information ```rust -// IR - Adds understanding and validation +// IR - Validates against the schema and adds additional information for app generation CloudformationProgramIr { resources: [ ResourceInstruction { diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 23513295..67f8404d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,51 +1,43 @@ -# Contributing +# Contributing to cdk-from-cfn -This document describes how to set up a development environment and submit your changes. +We welcome contributions to the cdk-from-cfn project! This guide will help you get started with contributing. -## Getting Started +## Development Environment Setup -### Setup - -#### Required - -- [Rust](https://www.rust-lang.org/tools/install) +### Prerequisites +- [Rust](https://www.rust-lang.org/tools/install) (Latest stable version) +- [Wasm-Pack](https://github.com/rustwasm/wasm-pack?tab=readme-ov-file) - [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) -- [GitHub account](https://github.com/join) - -#### Recommended +- [GitHub Account](https://github.com/join) -- [Brew](https://docs.brew.sh/Installation) -- IDE - - [VSCode](https://code.visualstudio.com/download) - - [IntelliJ](https://www.jetbrains.com/idea/download) -- [GitHub CLI](https://cli.github.com/) +### Getting the Project -```console -// Using github cli +```bash +# Clone your fork gh repo fork cdklabs/cdk-from-cfn +# Or manually clone +git clone https://github.com/YOUR_USERNAME/cdk-from-cfn.git cd cdk-from-cfn -cargo build ``` -### Tests +## Building the Project +```bash +# build the debug target +cargo build -```console -cargo test -``` +# build the release target +cargo build --release -```console -./tasks/coverage.sh +# build the wasm release +wasm-pack build --all-features --target=nodejs ``` -### Making changes - -Following guidance is for making changes in your fork and pushing changes to your fork's remote: +## Testing the Project -```console -git status -git add -git commit -m "" -git push -``` +```bash +# run all tests +cargo test -Once you have done the above, you can then [create a PR from your fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). +# run clippy to lint +cargo clippy +``` \ No newline at end of file From 3f0d94a00d0c327ccd5760b403972206191b4d6e Mon Sep 17 00:00:00 2001 From: HBobertz Date: Wed, 29 Jan 2025 00:08:30 -0500 Subject: [PATCH 3/6] update readme again --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ed3b3c89..dab05971 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,9 @@ $ cdk-from-cfn [INPUT] [OUTPUT] --language --stack-name - `INPUT` is the input file path (STDIN by default). - `OUTPUT` is the output file path; if not specified, output will be printed on your command line (STDOUT by default). -## Node Usage +## Node.js Module Usage -cdk-from-cfn is also vended as a module on [npm](https://www.npmjs.com/package/cdk-from-cfn) and exposes some apis which can be used -in a node project. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. +cdk-from-cfn utilizes WASM bindings to vend a cdk-from-cfn library as a module on [npm](https://www.npmjs.com/package/cdk-from-cfn) which exposes some apis which can be used in a node project. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. ```typescript import * as cdk_from_cfn from 'cdk-from-cfn'; From 7d77204840bd44143f2609469d2aa03b1a780031 Mon Sep 17 00:00:00 2001 From: HBobertz Date: Wed, 29 Jan 2025 00:13:17 -0500 Subject: [PATCH 4/6] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dab05971..ace8e43e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ $ cdk-from-cfn [INPUT] [OUTPUT] --language --stack-name ## Node.js Module Usage -cdk-from-cfn utilizes WASM bindings to vend a cdk-from-cfn library as a module on [npm](https://www.npmjs.com/package/cdk-from-cfn) which exposes some apis which can be used in a node project. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. +cdk-from-cfn leverages WebAssembly (WASM) bindings to provide a cross-platform [npm](https://www.npmjs.com/package/cdk-from-cfn) module, which exposes some apis to be used in Node.js projects. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. ```typescript import * as cdk_from_cfn from 'cdk-from-cfn'; From 14323a22646e7eaa60e0c5e470053ac865d7b3e8 Mon Sep 17 00:00:00 2001 From: HBobertz Date: Wed, 29 Jan 2025 00:13:38 -0500 Subject: [PATCH 5/6] remove word --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ace8e43e..7bdb491c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ $ cdk-from-cfn [INPUT] [OUTPUT] --language --stack-name ## Node.js Module Usage -cdk-from-cfn leverages WebAssembly (WASM) bindings to provide a cross-platform [npm](https://www.npmjs.com/package/cdk-from-cfn) module, which exposes some apis to be used in Node.js projects. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. +cdk-from-cfn leverages WebAssembly (WASM) bindings to provide a cross-platform [npm](https://www.npmjs.com/package/cdk-from-cfn) module, which exposes apis to be used in Node.js projects. Simply take a dependency on `cdk-from-cfn` in your package.json and utilize it as you would a normal module. i.e. ```typescript import * as cdk_from_cfn from 'cdk-from-cfn'; From d4cbf4a357845d73bac7bed372bde30f5195dca8 Mon Sep 17 00:00:00 2001 From: HBobertz Date: Thu, 30 Jan 2025 10:06:35 -0500 Subject: [PATCH 6/6] update docs --- ARCHITECTURE.MD | 54 ++++++++++++++++++++++++++++++++++----- src/synthesizer/README.md | 2 +- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/ARCHITECTURE.MD b/ARCHITECTURE.MD index 94b0bbfe..83fcaed5 100644 --- a/ARCHITECTURE.MD +++ b/ARCHITECTURE.MD @@ -49,8 +49,7 @@ Responsible for de-serializing the Cloudformation template into a `Cloudformatio Key responsibilities: - Reads YAML/JSON Cloudformation templates - Creates corresponding Rust data structures -- Parses all template sections (Resources, Parameters, Conditions, etc.) -- Handles Cloudformation intrinsic functions (Fn::Join, Ref, etc.) +- Parses all template sections (Resources, Parameters, Conditions, intrinsic functions, etc.) ```rust // CloudFormation Template @@ -87,7 +86,16 @@ CloudformationParseTree { ### IR (Intermediate Representation) -Responsible for converting the raw ParseTree into a validated and enriched `CloudformationProgramIr` data structure. The IR's main purpose is to add necessary CDK-specific information and validate resources against the CDK schema before code generation. +Responsible for converting the raw ParseTree into a validated and enriched `CloudformationProgramIr`. The IR's main purpose is to validate the template against the schema, as well as add necessary CDK-specific information and validate resources against the CDK schema before code generation. Some of the key enrichments added by the IR from the parse tree are as follows: + +1. Type Resolution + * Comparing the resource properties from the ParseTree against the schema, the IR will concretely type the resource properties so they can be concretely typed in the generated application +2. Dependency Graph and Reference Tracking + * Performs a topological sort to determine resource ordering for code synthesis +3. Import Instructions + * Generates all modules modules which will later need to be imported in the cdk app. +4. Conditional Normalization + * CFN allows multiple ways to define conditionals (short hands or intrinsic functions), so the IR normalizes these into `ConditionIr`s as these will be turned into language specific boolean expressions by the synthesizer. Key responsibilities: - Validates resources and properties against CDK schema @@ -95,6 +103,7 @@ Key responsibilities: - Enhances references with type and origin information - Orders resources based on dependencies - Adds necessary CDK imports and type information +- Normalizes Conditionals ```rust // IR - Validates against the schema and adds additional information for app generation @@ -129,11 +138,44 @@ CloudformationProgramIr { ### Synthesizer -Responsible for converting the validated IR into CDK code in a specific target language. The synthesizer's main purpose is to generate idiomatic code that properly represents the CloudFormation resources as CDK constructs. +Responsible for converting the validated IR into CDK code in a specific target language. The synthesizer's main purpose is to generate idiomatic code that properly represents the CloudFormation resources as CDK constructs. At a high level the synthesizer's job is quite simple. Every language is different however generally speaking a synthesizer will loop through the various properties of the IR and generate language specific code in the following order: + +``` +1. Import Generation +2. Stack Class/Props Generation +3. Resource Generation +4. Mappings Generation +5. Output Generation +``` + +However because every language is different that does not mean every synthesizer will generate code in the same way/order. For example a simple s3 bucket in Typescript may look like this + +```typescript +// Stack properties defined as an interface +interface StackProps { + bucketName?: string; +} + +// Resource creation +new s3.Bucket(this, 'MyBucket', { + bucketName: props.bucketName +}); +``` + +where the same app in python may look like this +```python +# Stack properties defined as parameters since python does not have interfaces +def __init__(self, scope, id, *, bucket_name=None): + +# Resource creation +s3.Bucket(self, 'MyBucket', + bucket_name=bucket_name +) +``` + Key responsibilities: - Converts IR constructs to language-specific CDK code -- Manages imports and module dependencies +- Generates language specific imports - Handles language-specific code patterns (builders, properties, etc.) -- Generates properly formatted and structured code - Implements string interpolation and type conversion for each language diff --git a/src/synthesizer/README.md b/src/synthesizer/README.md index d9724b8c..4fa865dd 100644 --- a/src/synthesizer/README.md +++ b/src/synthesizer/README.md @@ -10,7 +10,7 @@ their specific use-cases, reducing compilation time and binary size. Languages considered "stable" are however enabled by default, while "experimental" targets are opt-in. -The `Synthesizer` API is very simple, reciving a CloudFormation Template IR +The `Synthesizer` API is very simple, receiving a CloudFormation Template IR object, and a `Writer` to which the generated code should be written. The `cdk_from_cfn::code` module provides assistance for generating code, in particular for maintaining correct indentation levels.