Skip to content

Commit

Permalink
Address review comments
Browse files Browse the repository at this point in the history 
Signed-off-by: Mikayla Thompson <thomika@amazon.com>
  • Loading branch information
@mikaylathompson
mikaylathompson committed Nov 14, 2023
1 parent 54c262d commit 09d33a8
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 78 deletions.
59 changes: 53 additions & 6 deletions deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
import {Domain, EngineVersion, TLSSecurityPolicy, ZoneAwarenessConfig} from "aws-cdk-lib/aws-opensearchservice";
import {RemovalPolicy, SecretValue, Stack} from "aws-cdk-lib";
import {IKey, Key} from "aws-cdk-lib/aws-kms";
import {PolicyStatement} from "aws-cdk-lib/aws-iam";
import {AnyPrincipal, Effect, PolicyStatement} from "aws-cdk-lib/aws-iam";
import {ILogGroup, LogGroup} from "aws-cdk-lib/aws-logs";
import {ISecret, Secret} from "aws-cdk-lib/aws-secretsmanager";
import {StackPropsExt} from "./stack-composer";
Expand All @@ -25,7 +25,8 @@ export interface OpensearchDomainStackProps extends StackPropsExt {
readonly dedicatedManagerNodeCount?: number,
readonly warmInstanceType?: string,
readonly warmNodes?: number
readonly accessPolicies?: PolicyStatement[],
readonly accessPolicyJson?: object,
readonly openAccessPolicyEnabled?: boolean
readonly useUnsignedBasicAuth?: boolean,
readonly fineGrainedManagerUserARN?: string,
readonly fineGrainedManagerUserName?: string,
Expand All @@ -36,7 +37,7 @@ export interface OpensearchDomainStackProps extends StackPropsExt {
readonly ebsEnabled?: boolean,
readonly ebsIops?: number,
readonly ebsVolumeSize?: number,
readonly ebsVolumeType?: EbsDeviceVolumeType,
readonly ebsVolumeTypeName?: string,
readonly encryptionAtRestEnabled?: boolean,
readonly encryptionAtRestKmsKeyARN?: string,
readonly appLogEnabled?: boolean,
Expand All @@ -49,11 +50,49 @@ export interface OpensearchDomainStackProps extends StackPropsExt {
readonly domainRemovalPolicy?: RemovalPolicy,
readonly domainAccessSecurityGroupParameter?: string,
readonly endpointParameterName?: string

}


export class OpenSearchDomainStack extends Stack {

getEbsVolumeType(ebsVolumeTypeName: string) : EbsDeviceVolumeType|undefined {
const ebsVolumeType: EbsDeviceVolumeType|undefined = ebsVolumeTypeName ? EbsDeviceVolumeType[ebsVolumeTypeName as keyof typeof EbsDeviceVolumeType] : undefined
if (ebsVolumeTypeName && !ebsVolumeType) {
throw new Error("Provided ebsVolumeType does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.EbsDeviceVolumeType.html")
}
return ebsVolumeType
}

createOpenAccessPolicy(domainName: string) {
const openPolicy = new PolicyStatement({
effect: Effect.ALLOW,
principals: [new AnyPrincipal()],
actions: ["es:*"],
resources: [`arn:aws:es:${this.region}:${this.account}:domain/${domainName}/*`]
})
return openPolicy
}

parseAccessPolicies(jsonObject: { [x: string]: any; }): PolicyStatement[] {
let accessPolicies: PolicyStatement[] = []
const statements = jsonObject['Statement']
if (!statements || statements.length < 1) {
throw new Error ("Provided accessPolicies JSON must have the 'Statement' element present and not be empty, for reference https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_statement.html")
}
// Access policies can provide a single Statement block or an array of Statement blocks
if (Array.isArray(statements)) {
for (let statementBlock of statements) {
const statement = PolicyStatement.fromJson(statementBlock)
accessPolicies.push(statement)
}
}
else {
const statement = PolicyStatement.fromJson(statements)
accessPolicies.push(statement)
}
return accessPolicies
}

createSSMParameters(domain: Domain, endpointParameterName: string|undefined, adminUserName: string|undefined, adminUserSecret: ISecret|undefined, stage: string, deployId: string) {

Expand Down Expand Up @@ -95,7 +134,6 @@ export class OpenSearchDomainStack extends Stack {
StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/${domainAccessSecurityGroupParameter}`))

// Map objects from props

let adminUserName: string|undefined = props.fineGrainedManagerUserName
// Enable demo mode setting
if (props.enableDemoAdmin) {
Expand Down Expand Up @@ -128,10 +166,19 @@ export class OpenSearchDomainStack extends Stack {
}
}

const ebsVolumeType = props.ebsVolumeTypeName ? this.getEbsVolumeType(props.ebsVolumeTypeName) : undefined

let accessPolicies: PolicyStatement[] | undefined
if (props.openAccessPolicyEnabled) {
accessPolicies = [this.createOpenAccessPolicy(props.domainName)]
} else {
accessPolicies = props.accessPolicyJson ? this.parseAccessPolicies(props.accessPolicyJson) : undefined
}

const domain = new Domain(this, 'Domain', {
version: props.version,
domainName: props.domainName,
accessPolicies: props.accessPolicies,
accessPolicies: accessPolicies,
useUnsignedBasicAuth: props.useUnsignedBasicAuth,
capacity: {
dataNodeInstanceType: props.dataNodeInstanceType,
Expand All @@ -157,7 +204,7 @@ export class OpenSearchDomainStack extends Stack {
enabled: props.ebsEnabled,
iops: props.ebsIops,
volumeSize: props.ebsVolumeSize,
volumeType: props.ebsVolumeType
volumeType: ebsVolumeType
},
logging: {
appLogEnabled: props.appLogEnabled,
Expand Down
1 change: 0 additions & 1 deletion deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ export class CaptureProxyStack extends MigrationServiceCore {
this.createService({
serviceName: "capture-proxy",
dockerFilePath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/build/docker/trafficCaptureProxyServer"),
// TODO: add otel collector endpoint
dockerImageCommand: ['/bin/sh', '-c', command],
securityGroups: securityGroups,
taskRolePolicies: [mskClusterConnectPolicy, mskTopicProducerPolicy],
Expand Down
23 changes: 6 additions & 17 deletions deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,27 +24,12 @@ export class MigrationAnalyticsStack extends MigrationServiceCore {
constructor(scope: Construct, id: string, props: MigrationAnalyticsProps) {
super(scope, id, props)

// Bastion Security Group
const bastionSecurityGroup = new SecurityGroup(
this,
"analyticsDashboardBastionSecurityGroup",
{
vpc: props.vpc,
allowAllOutbound: true,
securityGroupName: "analyticsDashboardBastionSecurityGroup",
}
);
let securityGroups = [
SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
SecurityGroup.fromSecurityGroupId(this, "migrationAnalyticsSGId", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/analyticsDomainSGId`)),
bastionSecurityGroup
]
securityGroups[1].addIngressRule(bastionSecurityGroup, Port.tcp(443))
const migrationAnalyticsSecurityGroup = SecurityGroup.fromSecurityGroupId(this, "migrationAnalyticsSGId", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/analyticsDomainSGId`))

// Bastion host to access Opensearch Dashboards
new BastionHostLinux(this, "AnalyticsDashboardBastionHost", {
vpc: props.vpc,
securityGroup: bastionSecurityGroup,
securityGroup: migrationAnalyticsSecurityGroup,
machineImage: MachineImage.latestAmazonLinux2023(),
blockDevices: [
{
Expand Down Expand Up @@ -82,6 +67,10 @@ export class MigrationAnalyticsStack extends MigrationServiceCore {

const analyticsDomainEndpoint = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/analyticsDomainEndpoint`)

let securityGroups = [
SecurityGroup.fromSecurityGroupId(this, "serviceConnectSG", StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`)),
migrationAnalyticsSecurityGroup
]
this.createService({
serviceName: `otel-collector`,
dockerFilePath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/otelcol"),
Expand Down
2 changes: 1 addition & 1 deletion deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ export class TrafficReplayerStack extends MigrationServiceCore {
const osClusterEndpoint = props.customTargetEndpoint ? props.customTargetEndpoint : `https://${cdkDomainEndpoint}:443`
const brokerEndpoints = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/mskBrokers`);
const groupId = props.customKafkaGroupId ? props.customKafkaGroupId : `logging-group-${deployId}`
// TODO: add otel collector endpoint

let replayerCommand = `/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer ${osClusterEndpoint} --insecure --kafka-traffic-brokers ${brokerEndpoints} --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id ${groupId} --kafka-traffic-enable-msk-auth`
if (props.enableClusterFGACAuth) {
const osUserAndSecret = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${deployId}/osUserAndSecretArn`);
Expand Down
62 changes: 9 additions & 53 deletions deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import {Application} from "@aws-cdk/aws-servicecatalogappregistry-alpha";
export interface StackPropsExt extends StackProps {
readonly stage: string,
readonly defaultDeployId: string
readonly addOnMigrationDeployId?: string,
readonly addOnMigrationDeployId?: string
}

export interface StackComposerProps extends StackProps {
Expand Down Expand Up @@ -67,26 +67,6 @@ export class StackComposer {
return option
}

private parseAccessPolicies(jsonObject: { [x: string]: any; }): PolicyStatement[] {
let accessPolicies: PolicyStatement[] = []
const statements = jsonObject['Statement']
if (!statements || statements.length < 1) {
throw new Error ("Provided accessPolicies JSON must have the 'Statement' element present and not be empty, for reference https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_statement.html")
}
// Access policies can provide a single Statement block or an array of Statement blocks
if (Array.isArray(statements)) {
for (let statementBlock of statements) {
const statement = PolicyStatement.fromJson(statementBlock)
accessPolicies.push(statement)
}
}
else {
const statement = PolicyStatement.fromJson(statements)
accessPolicies.push(statement)
}
return accessPolicies
}


private getEngineVersion(engineVersionString: string) : EngineVersion {
let version: EngineVersion
Expand Down Expand Up @@ -136,6 +116,7 @@ export class StackComposer {
const enforceHTTPS = this.getContextForType('enforceHTTPS', 'boolean', defaultValues, contextJSON)
const ebsEnabled = this.getContextForType('ebsEnabled', 'boolean', defaultValues, contextJSON)
const ebsIops = this.getContextForType('ebsIops', 'number', defaultValues, contextJSON)
const ebsVolumeTypeName = this.getContextForType('ebsVolumeType', 'string', defaultValues, contextJSON)
const ebsVolumeSize = this.getContextForType('ebsVolumeSize', 'number', defaultValues, contextJSON)
const encryptionAtRestEnabled = this.getContextForType('encryptionAtRestEnabled', 'boolean', defaultValues, contextJSON)
const encryptionAtRestKmsKeyARN = this.getContextForType("encryptionAtRestKmsKeyARN", 'string', defaultValues, contextJSON)
Expand All @@ -147,6 +128,7 @@ export class StackComposer {
const vpcSecurityGroupIds = this.getContextForType('vpcSecurityGroupIds', 'object', defaultValues, contextJSON)
const vpcSubnetIds = this.getContextForType('vpcSubnetIds', 'object', defaultValues, contextJSON)
const openAccessPolicyEnabled = this.getContextForType('openAccessPolicyEnabled', 'boolean', defaultValues, contextJSON)
const accessPolicyJson = this.getContextForType('accessPolicies', 'object', defaultValues, contextJSON)
const availabilityZoneCount = this.getContextForType('availabilityZoneCount', 'number', defaultValues, contextJSON)
const migrationAssistanceEnabled = this.getContextForType('migrationAssistanceEnabled', 'boolean', defaultValues, contextJSON)
const mskARN = this.getContextForType('mskARN', 'string', defaultValues, contextJSON)
Expand Down Expand Up @@ -179,15 +161,13 @@ export class StackComposer {
const analyticsDomainDedicatedManagerNodeCount = this.getContextForType('analyticsDomainDedicatedManagerNodeCount', 'number', defaultValues, contextJSON)
const analyticsDomainWarmNodeType = this.getContextForType('analyticsDomainWarmNodeType', 'string', defaultValues, contextJSON)
const analyticsDomainWarmNodeCount = this.getContextForType('analyticsDomainWarmNodeCount', 'number', defaultValues, contextJSON)
const analyticsDomainEnforceHTTPS = this.getContextForType('analyticsDomainEnforceHTTPS', 'boolean', defaultValues, contextJSON)
const analyticsDomainEbsEnabled = this.getContextForType('analyticsDomainEbsEnabled', 'boolean', defaultValues, contextJSON)
const analyticsDomainEbsIops = this.getContextForType('analyticsDomainEbsIops', 'number', defaultValues, contextJSON)
const analyticsDomainEbsVolumeSize = this.getContextForType('analyticsDomainEbsVolumeSize', 'number', defaultValues, contextJSON)
const analyticsDomainEncryptionAtRestEnabled = this.getContextForType('analyticsDomainEncryptionAtRestEnabled', 'boolean', defaultValues, contextJSON)
const analyticsDomainEbsVolumeTypeName = this.getContextForType('analyticsDomainEbsVolumeType', 'string', defaultValues, contextJSON)
const analyticsDomainEncryptionAtRestKmsKeyARN = this.getContextForType("analyticsDomainEncryptionAtRestKmsKeyARN", 'string', defaultValues, contextJSON)
const analyticsDomainLoggingAppLogEnabled = this.getContextForType('analyticsDomainLoggingAppLogEnabled', 'boolean', defaultValues, contextJSON)
const analyticsDomainLoggingAppLogGroupARN = this.getContextForType('analyticsDomainLoggingAppLogGroupARN', 'string', defaultValues, contextJSON)
const analyticsDomainNoneToNodeEncryptionEnabled = this.getContextForType('analyticsDomainNodeToNodeEncryptionEnabled', 'boolean', defaultValues, contextJSON)

if (!stage) {
throw new Error("Required context field 'stage' is not present")
Expand All @@ -202,37 +182,12 @@ export class StackComposer {
const engineVersion = this.getContextForType('engineVersion', 'string', defaultValues, contextJSON)
version = this.getEngineVersion(engineVersion)

if (openAccessPolicyEnabled) {
const openPolicy = new PolicyStatement({
effect: Effect.ALLOW,
principals: [new AnyPrincipal()],
actions: ["es:*"],
resources: [`arn:aws:es:${region}:${account}:domain/${domainName}/*`]
})
accessPolicies = [openPolicy]
} else {
const accessPolicyJson = this.getContextForType('accessPolicies', 'object', defaultValues, contextJSON)
accessPolicies = accessPolicyJson ? this.parseAccessPolicies(accessPolicyJson) : undefined
}

const tlsSecurityPolicyName = this.getContextForType('tlsSecurityPolicy', 'string', defaultValues, contextJSON)
const tlsSecurityPolicy: TLSSecurityPolicy|undefined = tlsSecurityPolicyName ? TLSSecurityPolicy[tlsSecurityPolicyName as keyof typeof TLSSecurityPolicy] : undefined
if (tlsSecurityPolicyName && !tlsSecurityPolicy) {
throw new Error("Provided tlsSecurityPolicy does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_opensearchservice.TLSSecurityPolicy.html")
}

const ebsVolumeTypeName = this.getContextForType('ebsVolumeType', 'string', defaultValues, contextJSON)
const ebsVolumeType: EbsDeviceVolumeType|undefined = ebsVolumeTypeName ? EbsDeviceVolumeType[ebsVolumeTypeName as keyof typeof EbsDeviceVolumeType] : undefined
if (ebsVolumeTypeName && !ebsVolumeType) {
throw new Error("Provided ebsVolumeType does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.EbsDeviceVolumeType.html")
}

const analyticsDomainEbsVolumeTypeName = this.getContextForType('analyticsDomainEbsVolumeType', 'string', defaultValues, contextJSON)
const analyticsDomainEbsVolumeType: EbsDeviceVolumeType|undefined = analyticsDomainEbsVolumeTypeName ? EbsDeviceVolumeType[analyticsDomainEbsVolumeTypeName as keyof typeof EbsDeviceVolumeType] : undefined
if (analyticsDomainEbsVolumeTypeName && !analyticsDomainEbsVolumeType) {
throw new Error("Provided ebsVolumeType does not match a selectable option, for reference https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.EbsDeviceVolumeType.html")
}

const domainRemovalPolicyName = this.getContextForType('domainRemovalPolicy', 'string', defaultValues, contextJSON)
const domainRemovalPolicy = domainRemovalPolicyName ? RemovalPolicy[domainRemovalPolicyName as keyof typeof RemovalPolicy] : undefined
if (domainRemovalPolicyName && !domainRemovalPolicy) {
Expand Down Expand Up @@ -267,7 +222,8 @@ export class StackComposer {
dedicatedManagerNodeCount: dedicatedManagerNodeCount,
warmInstanceType: warmNodeType,
warmNodes: warmNodeCount,
accessPolicies: accessPolicies,
openAccessPolicyEnabled: openAccessPolicyEnabled,
accessPolicyJson: accessPolicyJson,
useUnsignedBasicAuth: useUnsignedBasicAuth,
fineGrainedManagerUserARN: fineGrainedManagerUserARN,
fineGrainedManagerUserName: fineGrainedManagerUserName,
Expand All @@ -278,7 +234,7 @@ export class StackComposer {
ebsEnabled: ebsEnabled,
ebsIops: ebsIops,
ebsVolumeSize: ebsVolumeSize,
ebsVolumeType: ebsVolumeType,
ebsVolumeTypeName: ebsVolumeTypeName,
encryptionAtRestEnabled: encryptionAtRestEnabled,
encryptionAtRestKmsKeyARN: encryptionAtRestKmsKeyARN,
appLogEnabled: loggingAppLogEnabled,
Expand Down Expand Up @@ -373,10 +329,10 @@ export class StackComposer {
ebsEnabled: analyticsDomainEbsEnabled,
ebsIops: analyticsDomainEbsIops,
ebsVolumeSize: analyticsDomainEbsVolumeSize,
ebsVolumeType: analyticsDomainEbsVolumeType,
ebsVolumeTypeName: analyticsDomainEbsVolumeTypeName,
stage: stage,
defaultDeployId: defaultDeployId,
accessPolicies: [openAccessPolicy],
openAccessPolicyEnabled: true,
...props
})
migrationAnalyticsStack = new MigrationAnalyticsStack(scope, "migration-analytics", {
Expand Down
Loading

0 comments on commit 09d33a8

Please sign in to comment.