From 3b31f16fe448e3dd01048dcaa2d9b1ad74a1577c Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 13 Feb 2021 14:19:21 +0100 Subject: [PATCH] Add comments to the disabled/unsupported integration test aliase files (#411) Add some additional comments so we know *why* the various tests aren't running. Looks like most of them just need policy updates --- tests/integration/targets/aws_config/aliases | 6 +++++- tests/integration/targets/aws_eks_cluster/aliases | 4 +++- tests/integration/targets/aws_secret/aliases | 4 +++- tests/integration/targets/aws_waf_web_acl/aliases | 6 +++++- .../integration/targets/cloudformation_stack_set/aliases | 5 ++++- .../integration/targets/cloudfront_distribution/aliases | 4 +++- tests/integration/targets/cloudtrail/aliases | 4 +++- tests/integration/targets/connection_aws_ssm/aliases | 8 ++++++-- tests/integration/targets/dms_endpoint/aliases | 4 +++- .../targets/dms_replication_subnet_group/aliases | 4 +++- tests/integration/targets/ec2_asg/aliases | 6 +++++- tests/integration/targets/ecs_cluster/aliases | 6 +++++- tests/integration/targets/ecs_tag/aliases | 5 ++++- tests/integration/targets/efs/aliases | 6 +++++- tests/integration/targets/elb_network_lb/aliases | 4 +++- tests/integration/targets/elb_target_info/aliases | 4 +++- tests/integration/targets/iam_group/aliases | 5 +++++ tests/integration/targets/iam_password_policy/aliases | 6 +++++- tests/integration/targets/iam_policy/aliases | 9 +++++++-- tests/integration/targets/iam_role/aliases | 8 +++++++- tests/integration/targets/iam_saml_federation/aliases | 4 +++- tests/integration/targets/iam_user/aliases | 8 +++++++- tests/integration/targets/rds_instance/aliases | 4 +++- tests/integration/targets/s3_lifecycle/aliases | 5 ++++- tests/integration/targets/s3_logging/aliases | 6 ++++-- tests/integration/targets/sns/aliases | 4 +++- tests/integration/targets/sns_topic/aliases | 4 +++- 27 files changed, 114 insertions(+), 29 deletions(-) diff --git a/tests/integration/targets/aws_config/aliases b/tests/integration/targets/aws_config/aliases index c598f3e8b47..932c8c7bb8d 100644 --- a/tests/integration/targets/aws_config/aliases +++ b/tests/integration/targets/aws_config/aliases @@ -1,6 +1,10 @@ -cloud/aws +# reason: missing-policy +# We don't have CI or 'unsupported' policy for AWS config disabled + +cloud/aws shippable/aws/group2 + aws_config_aggregation_authorization aws_config_aggregator aws_config_delivery_channel diff --git a/tests/integration/targets/aws_eks_cluster/aliases b/tests/integration/targets/aws_eks_cluster/aliases index 56927195182..3035c760534 100644 --- a/tests/integration/targets/aws_eks_cluster/aliases +++ b/tests/integration/targets/aws_eks_cluster/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: unknown unsupported + +cloud/aws diff --git a/tests/integration/targets/aws_secret/aliases b/tests/integration/targets/aws_secret/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/aws_secret/aliases +++ b/tests/integration/targets/aws_secret/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/aws_waf_web_acl/aliases b/tests/integration/targets/aws_waf_web_acl/aliases index ed291667234..5d782911356 100644 --- a/tests/integration/targets/aws_waf_web_acl/aliases +++ b/tests/integration/targets/aws_waf_web_acl/aliases @@ -1,6 +1,10 @@ +# reason: broken +# ansible/ansible#38258 +unsupported + cloud/aws + aws_waf_info aws_waf_web_acl aws_waf_web_match aws_waf_web_rule -unsupported diff --git a/tests/integration/targets/cloudformation_stack_set/aliases b/tests/integration/targets/cloudformation_stack_set/aliases index 56927195182..47eb21d5303 100644 --- a/tests/integration/targets/cloudformation_stack_set/aliases +++ b/tests/integration/targets/cloudformation_stack_set/aliases @@ -1,2 +1,5 @@ +# reason: broken +# Tests rely on additional non-standard secondary_aws_ credentials +disabled + cloud/aws -unsupported diff --git a/tests/integration/targets/cloudfront_distribution/aliases b/tests/integration/targets/cloudfront_distribution/aliases index 56927195182..e04e1b287a4 100644 --- a/tests/integration/targets/cloudfront_distribution/aliases +++ b/tests/integration/targets/cloudfront_distribution/aliases @@ -1,2 +1,4 @@ +# reason: broken +disabled + cloud/aws -unsupported diff --git a/tests/integration/targets/cloudtrail/aliases b/tests/integration/targets/cloudtrail/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/cloudtrail/aliases +++ b/tests/integration/targets/cloudtrail/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/connection_aws_ssm/aliases b/tests/integration/targets/connection_aws_ssm/aliases index 0031909d6da..ef80db1cad5 100644 --- a/tests/integration/targets/connection_aws_ssm/aliases +++ b/tests/integration/targets/connection_aws_ssm/aliases @@ -1,7 +1,11 @@ +# reason: unstable +# reason: slow +disabled + cloud/aws -destructive shippable/aws/group4 + +destructive non_local needs/root needs/target/connection -disabled diff --git a/tests/integration/targets/dms_endpoint/aliases b/tests/integration/targets/dms_endpoint/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/dms_endpoint/aliases +++ b/tests/integration/targets/dms_endpoint/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/dms_replication_subnet_group/aliases b/tests/integration/targets/dms_replication_subnet_group/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/dms_replication_subnet_group/aliases +++ b/tests/integration/targets/dms_replication_subnet_group/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/ec2_asg/aliases b/tests/integration/targets/ec2_asg/aliases index 56927195182..ae2bc1e9bd0 100644 --- a/tests/integration/targets/ec2_asg/aliases +++ b/tests/integration/targets/ec2_asg/aliases @@ -1,2 +1,6 @@ -cloud/aws +# reason: slow +# reason: broken +# Tests take around 30 minutes unsupported + +cloud/aws diff --git a/tests/integration/targets/ecs_cluster/aliases b/tests/integration/targets/ecs_cluster/aliases index 4b1bea7a189..f120eea7d6f 100644 --- a/tests/integration/targets/ecs_cluster/aliases +++ b/tests/integration/targets/ecs_cluster/aliases @@ -1,6 +1,10 @@ +# reason: slow +# Tests take around 15 minutes to run +unsupported + cloud/aws + ecs_service_info ecs_task ecs_taskdefinition ecs_taskdefinition_info -unsupported diff --git a/tests/integration/targets/ecs_tag/aliases b/tests/integration/targets/ecs_tag/aliases index fe51f28bd2e..4413c442ed8 100644 --- a/tests/integration/targets/ecs_tag/aliases +++ b/tests/integration/targets/ecs_tag/aliases @@ -1,3 +1,6 @@ +# reason: missing-policy +unsupported + cloud/aws + ecs_tag -unsupported diff --git a/tests/integration/targets/efs/aliases b/tests/integration/targets/efs/aliases index 3f9eda99494..d0d7616f534 100644 --- a/tests/integration/targets/efs/aliases +++ b/tests/integration/targets/efs/aliases @@ -1,3 +1,7 @@ -cloud/aws +# reason: missing-policy +# ansible/ansible#36520 - attempts were made to get it running in CI but were unsuccessful unsupported + +cloud/aws + efs_info diff --git a/tests/integration/targets/elb_network_lb/aliases b/tests/integration/targets/elb_network_lb/aliases index 56927195182..2b0eed263eb 100644 --- a/tests/integration/targets/elb_network_lb/aliases +++ b/tests/integration/targets/elb_network_lb/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: unsupported + +cloud/aws diff --git a/tests/integration/targets/elb_target_info/aliases b/tests/integration/targets/elb_target_info/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/elb_target_info/aliases +++ b/tests/integration/targets/elb_target_info/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/iam_group/aliases b/tests/integration/targets/iam_group/aliases index 67ae2cc73b6..2da398045bd 100644 --- a/tests/integration/targets/iam_group/aliases +++ b/tests/integration/targets/iam_group/aliases @@ -1,2 +1,7 @@ +# reason: missing-policy +# It should be possible to test iam_groups by limiting which policies can be +# attached to the groups as well as which users can be added to the groups. +# Careful review is needed prior to adding this to the main CI. unsupported + cloud/aws diff --git a/tests/integration/targets/iam_password_policy/aliases b/tests/integration/targets/iam_password_policy/aliases index 56927195182..de4c8915ab9 100644 --- a/tests/integration/targets/iam_password_policy/aliases +++ b/tests/integration/targets/iam_password_policy/aliases @@ -1,2 +1,6 @@ -cloud/aws +# reason: missing-policy +# IAM Password Policies configure account-wide settings, this makes then +# difficult to safely test unsupported + +cloud/aws diff --git a/tests/integration/targets/iam_policy/aliases b/tests/integration/targets/iam_policy/aliases index 3f812e19142..9b3bde40b45 100644 --- a/tests/integration/targets/iam_policy/aliases +++ b/tests/integration/targets/iam_policy/aliases @@ -1,3 +1,8 @@ -iam_policy_info -cloud/aws +# reason: missing-policy +# It's not possible to control what permissions are granted to a policy. +# This makes securely testing iam_policy very difficult unsupported + +cloud/aws + +iam_policy_info diff --git a/tests/integration/targets/iam_role/aliases b/tests/integration/targets/iam_role/aliases index 3d7a2c9f14c..483c861158c 100644 --- a/tests/integration/targets/iam_role/aliases +++ b/tests/integration/targets/iam_role/aliases @@ -1,3 +1,9 @@ -iam_role_info +# reason: missing-policy +# It should be possible to test iam_role by limiting which policies can be +# attached to the roles. +# Careful review is needed prior to adding this to the main CI. unsupported + cloud/aws + +iam_role_info diff --git a/tests/integration/targets/iam_saml_federation/aliases b/tests/integration/targets/iam_saml_federation/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/iam_saml_federation/aliases +++ b/tests/integration/targets/iam_saml_federation/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/iam_user/aliases b/tests/integration/targets/iam_user/aliases index c7a4b8abe0e..cf11724d72d 100644 --- a/tests/integration/targets/iam_user/aliases +++ b/tests/integration/targets/iam_user/aliases @@ -1,3 +1,9 @@ +# reason: missing-policy +# It should be possible to test iam_user by limiting which policies can be +# attached to the users. +# Careful review is needed prior to adding this to the main CI. +unsupported + cloud/aws + iam_user_info -unsupported diff --git a/tests/integration/targets/rds_instance/aliases b/tests/integration/targets/rds_instance/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/rds_instance/aliases +++ b/tests/integration/targets/rds_instance/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/s3_lifecycle/aliases b/tests/integration/targets/s3_lifecycle/aliases index 0cc87f1a382..9897349ac59 100644 --- a/tests/integration/targets/s3_lifecycle/aliases +++ b/tests/integration/targets/s3_lifecycle/aliases @@ -1,3 +1,6 @@ +# reason: broken +# ansible/ansible#59311 +disabled + cloud/aws shippable/aws/group1 -disabled diff --git a/tests/integration/targets/s3_logging/aliases b/tests/integration/targets/s3_logging/aliases index 3431a6a5421..df4bfd632c6 100644 --- a/tests/integration/targets/s3_logging/aliases +++ b/tests/integration/targets/s3_logging/aliases @@ -1,4 +1,6 @@ +# reason: unstable +# When running tests a failure rate of around 20% was observed +unsupported + cloud/aws #shippable/aws/group1 -# when running tests we saw an ~20% failure rate -unsupported diff --git a/tests/integration/targets/sns/aliases b/tests/integration/targets/sns/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/sns/aliases +++ b/tests/integration/targets/sns/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws diff --git a/tests/integration/targets/sns_topic/aliases b/tests/integration/targets/sns_topic/aliases index 56927195182..e4280272565 100644 --- a/tests/integration/targets/sns_topic/aliases +++ b/tests/integration/targets/sns_topic/aliases @@ -1,2 +1,4 @@ -cloud/aws +# reason: missing-policy unsupported + +cloud/aws