diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 79b85d11ac..e90f44dae7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,23 +4,25 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "daily" commit-message: prefix: chore include: scope reviewers: - aws/serverless-application-experience-sbt + open-pull-requests-limit: 10 - package-ecosystem: "pip" directory: "/requirements" schedule: - interval: "weekly" + interval: "daily" target-branch: "develop" commit-message: prefix: chore include: scope reviewers: - aws/serverless-application-experience-sbt + open-pull-requests-limit: 10 ignore: # Ignored intentionally since we have a GHA that updates to more # completely diff --git a/installer/pyinstaller/build-linux.sh b/installer/pyinstaller/build-linux.sh index 041d4d190d..2b728866da 100755 --- a/installer/pyinstaller/build-linux.sh +++ b/installer/pyinstaller/build-linux.sh @@ -26,7 +26,7 @@ else is_nightly="false" fi -set -eu +set -eux yum install -y zlib-devel libffi-devel bzip2-devel @@ -83,6 +83,7 @@ cp -r ./venv/lib/python*/site-packages/* ./output/python-libraries echo "Installing PyInstaller" ./venv/bin/pip install -r src/requirements/pyinstaller-build.txt +./venv/bin/pip check echo "Building Binary" cd src diff --git a/installer/pyinstaller/build-mac.sh b/installer/pyinstaller/build-mac.sh index eafcb14f31..7a69a6b0a4 100644 --- a/installer/pyinstaller/build-mac.sh +++ b/installer/pyinstaller/build-mac.sh @@ -45,7 +45,7 @@ else is_nightly="false" fi -set -eu +set -eux echo "Making Folders" mkdir -p .build/src @@ -93,6 +93,7 @@ cp -r ./venv/lib/python*/site-packages/* ./output/python-libraries echo "Installing PyInstaller" ./venv/bin/pip install -r src/requirements/pyinstaller-build.txt +./venv/bin/pip check # Building the binary using pyinstaller echo "Building Binary" diff --git a/requirements/base.txt b/requirements/base.txt index 38532b4714..57a67e078b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,7 @@ chevron~=0.12 # 8.1.4 of Click has an issue with the typing breaking the linter - https://github.com/pallets/click/issues/2558 -click~=8.0,!=8.1.4 +# Allow click to be greater than 8.1.4 when https://github.com/pallets/click/pull/2565 is released. +click~=8.0,<8.1.4 Flask<2.3 #Need to add latest lambda changes which will return invoke mode details boto3>=1.26.109,==1.* @@ -17,13 +18,13 @@ serverlessrepo==0.1.10 aws_lambda_builders==1.34.0 tomlkit==0.11.8 watchdog==2.1.2 -rich~=13.3.3 +rich~=13.4.2 pyopenssl~=23.2.0 # Pin to <4.18 to until SAM-T no longer uses RefResolver jsonschema<4.18 # Needed for supporting Protocol in Python 3.7, Protocol class became public with python3.8 -typing_extensions~=4.4.0 +typing_extensions>=4.4.0,<5 # NOTE: regex is not a direct dependency of SAM CLI, exclude version 2021.10.8 due to not working on M1 Mac - https://github.com/mrabarnett/mrab-regex/issues/399 regex!=2021.10.8 @@ -31,4 +32,4 @@ regex!=2021.10.8 tzlocal==3.0 #Adding cfn-lint dependency for SAM validate -cfn-lint~=0.77.9 +cfn-lint~=0.78.1 diff --git a/requirements/dev.txt b/requirements/dev.txt index 2421d05095..55f6a457c1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ pytest-cov==4.1.0 # mypy adds new rules in new minor versions, which could cause our PR check to fail # here we fix its version and upgrade it manually in the future mypy==1.3.0 -boto3-stubs[apigateway,cloudformation,ecr,iam,lambda,s3,schemas,secretsmanager,signer,stepfunctions,sts,xray]==1.26.131 +boto3-stubs[apigateway,cloudformation,ecr,iam,lambda,s3,schemas,secretsmanager,signer,stepfunctions,sts,xray]==1.28.2 types-pywin32==306.0.0.2 types-PyYAML==6.0.12 types-chevron==0.14.2.4 @@ -19,7 +19,7 @@ types-colorama==0.4.15.11 types-dateparser==1.1.4.9 types-docutils==0.20.0.1 types-jsonschema==4.17.0.8 -types-pyOpenSSL==23.2.0.0 +types-pyOpenSSL==23.2.0.1 types-requests==2.31.0.1 types-urllib3==1.26.25.13 diff --git a/requirements/reproducible-linux.txt b/requirements/reproducible-linux.txt index 7ed8c51903..8e993c68de 100644 --- a/requirements/reproducible-linux.txt +++ b/requirements/reproducible-linux.txt @@ -29,16 +29,16 @@ binaryornot==0.4.4 \ --hash=sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061 \ --hash=sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4 # via cookiecutter -boto3==1.26.134 \ - --hash=sha256:2da4a4caa789312ae73d29be9d3e79ce3328e3aaf7e9de0da6f243455ad3aae6 \ - --hash=sha256:a49b47621c71adfa952127222809ae50867ae4fd249bb932eb1a98519baefa40 +boto3==1.28.2 \ + --hash=sha256:0d53fe604dc30edded21906bc56b30a7684f0715f4f6897307d53f8184997368 \ + --hash=sha256:9933e40dc9ac72deac45cecce2df020e3bf8d0d537538d2b361c17d1cee807cc # via # aws-sam-cli (setup.py) # aws-sam-translator # serverlessrepo -botocore==1.29.135 \ - --hash=sha256:06502a4473924ef60ac0de908385a5afab9caee6c5b49cf6a330fab0d76ddf5f \ - --hash=sha256:0c61d4e5e04fe5329fa65da6b31492ef9d0d5174d72fc2af69de2ed0f87804ca +botocore==1.31.2 \ + --hash=sha256:67a475bec9e52d495a358b34e219ef7f62907e83b87e5bc712528f998bd46dab \ + --hash=sha256:d368ac0b58e2b9025b9c397e4a4f86d71788913ee619263506885a866a4f6811 # via # boto3 # s3transfer @@ -112,9 +112,9 @@ cffi==1.15.1 \ --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 # via cryptography -cfn-lint==0.77.9 \ - --hash=sha256:7c1e631b723b521234d92d4081934291b256dba28d723ddb7ff105215fe40020 \ - --hash=sha256:f95b503f7465ee1f2f89ddf32289ea03a517f08c366bb8e6a5d6773a11e5a1aa +cfn-lint==0.78.1 \ + --hash=sha256:2dacb19d5f70c0d49f466302507707cfa4914f65b8fc9310ae3771a273cec044 \ + --hash=sha256:46118362b2e13b79ba3ae6b3c28b7df5fcd437c06f5bcc3384d13a2defdb7d06 # via aws-sam-cli (setup.py) chardet==5.1.0 \ --hash=sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5 \ @@ -212,26 +212,30 @@ cookiecutter==2.1.1 \ --hash=sha256:9f3ab027cec4f70916e28f03470bdb41e637a3ad354b4d65c765d93aad160022 \ --hash=sha256:f3982be8d9c53dac1261864013fdec7f83afd2e42ede6f6dd069c5e149c540d5 # via aws-sam-cli (setup.py) -cryptography==41.0.0 \ - --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \ - --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \ - --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \ - --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \ - --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \ - --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \ - --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \ - --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \ - --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \ - --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \ - --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \ - --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \ - --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \ - --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \ - --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \ - --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \ - --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \ - --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \ - --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be +cryptography==41.0.2 \ + --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ + --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ + --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ + --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ + --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ + --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ + --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ + --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ + --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ + --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ + --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ + --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ + --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ + --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ + --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ + --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ + --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ + --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ + --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ + --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ + --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ + --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ + --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 # via pyopenssl dateparser==1.1.8 \ --hash=sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f \ @@ -291,6 +295,7 @@ jsonschema==4.17.3 \ --hash=sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d \ --hash=sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6 # via + # aws-sam-cli (setup.py) # aws-sam-translator # cfn-lint junit-xml==1.9 \ @@ -467,9 +472,9 @@ python-slugify==8.0.1 \ --hash=sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395 \ --hash=sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27 # via cookiecutter -pytz==2023.2 \ - --hash=sha256:8a8baaf1e237175b02f5c751eea67168043a749c843989e2b3015aa1ad9db68b \ - --hash=sha256:a27dcf612c05d2ebde626f7d506555f10dfc815b3eddccfaadfc7d99b11c9a07 +pytz==2023.3 \ + --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ + --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb # via dateparser pyyaml==5.4.1 \ --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ @@ -606,9 +611,9 @@ requests==2.31.0 \ # aws-sam-cli (setup.py) # cookiecutter # docker -rich==13.3.3 \ - --hash=sha256:540c7d6d26a1178e8e8b37e9ba44573a3cd1464ff6348b99ee7061b95d1c6333 \ - --hash=sha256:dc84400a9d842b3a9c5ff74addd8eb798d155f36c1c91303888e0a66850d2a15 +rich==13.4.2 \ + --hash=sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec \ + --hash=sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898 # via aws-sam-cli (setup.py) ruamel-yaml==0.17.32 \ --hash=sha256:23cd2ed620231677564646b0c6a89d138b6822a0d78656df7abda5879ec4f447 \ @@ -684,9 +689,9 @@ tomlkit==0.11.8 \ --hash=sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171 \ --hash=sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3 # via aws-sam-cli (setup.py) -typing-extensions==4.4.0 \ - --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ - --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e +typing-extensions==4.7.1 \ + --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \ + --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2 # via # aws-sam-cli (setup.py) # aws-sam-translator diff --git a/requirements/reproducible-mac.txt b/requirements/reproducible-mac.txt index 4ac1c38a21..7e7740fe0c 100644 --- a/requirements/reproducible-mac.txt +++ b/requirements/reproducible-mac.txt @@ -47,16 +47,16 @@ binaryornot==0.4.4 \ --hash=sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061 \ --hash=sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4 # via cookiecutter -boto3==1.26.134 \ - --hash=sha256:2da4a4caa789312ae73d29be9d3e79ce3328e3aaf7e9de0da6f243455ad3aae6 \ - --hash=sha256:a49b47621c71adfa952127222809ae50867ae4fd249bb932eb1a98519baefa40 +boto3==1.28.2 \ + --hash=sha256:0d53fe604dc30edded21906bc56b30a7684f0715f4f6897307d53f8184997368 \ + --hash=sha256:9933e40dc9ac72deac45cecce2df020e3bf8d0d537538d2b361c17d1cee807cc # via # aws-sam-cli (setup.py) # aws-sam-translator # serverlessrepo -botocore==1.29.135 \ - --hash=sha256:06502a4473924ef60ac0de908385a5afab9caee6c5b49cf6a330fab0d76ddf5f \ - --hash=sha256:0c61d4e5e04fe5329fa65da6b31492ef9d0d5174d72fc2af69de2ed0f87804ca +botocore==1.31.2 \ + --hash=sha256:67a475bec9e52d495a358b34e219ef7f62907e83b87e5bc712528f998bd46dab \ + --hash=sha256:d368ac0b58e2b9025b9c397e4a4f86d71788913ee619263506885a866a4f6811 # via # boto3 # s3transfer @@ -130,9 +130,9 @@ cffi==1.15.1 \ --hash=sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01 \ --hash=sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0 # via cryptography -cfn-lint==0.77.9 \ - --hash=sha256:7c1e631b723b521234d92d4081934291b256dba28d723ddb7ff105215fe40020 \ - --hash=sha256:f95b503f7465ee1f2f89ddf32289ea03a517f08c366bb8e6a5d6773a11e5a1aa +cfn-lint==0.78.1 \ + --hash=sha256:2dacb19d5f70c0d49f466302507707cfa4914f65b8fc9310ae3771a273cec044 \ + --hash=sha256:46118362b2e13b79ba3ae6b3c28b7df5fcd437c06f5bcc3384d13a2defdb7d06 # via aws-sam-cli (setup.py) chardet==5.1.0 \ --hash=sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5 \ @@ -230,26 +230,30 @@ cookiecutter==2.1.1 \ --hash=sha256:9f3ab027cec4f70916e28f03470bdb41e637a3ad354b4d65c765d93aad160022 \ --hash=sha256:f3982be8d9c53dac1261864013fdec7f83afd2e42ede6f6dd069c5e149c540d5 # via aws-sam-cli (setup.py) -cryptography==41.0.0 \ - --hash=sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55 \ - --hash=sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895 \ - --hash=sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be \ - --hash=sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928 \ - --hash=sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d \ - --hash=sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8 \ - --hash=sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237 \ - --hash=sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9 \ - --hash=sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78 \ - --hash=sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d \ - --hash=sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0 \ - --hash=sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46 \ - --hash=sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5 \ - --hash=sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4 \ - --hash=sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d \ - --hash=sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75 \ - --hash=sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb \ - --hash=sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2 \ - --hash=sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be +cryptography==41.0.2 \ + --hash=sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711 \ + --hash=sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7 \ + --hash=sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd \ + --hash=sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e \ + --hash=sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58 \ + --hash=sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0 \ + --hash=sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d \ + --hash=sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83 \ + --hash=sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831 \ + --hash=sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766 \ + --hash=sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b \ + --hash=sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c \ + --hash=sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182 \ + --hash=sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f \ + --hash=sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa \ + --hash=sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4 \ + --hash=sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a \ + --hash=sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2 \ + --hash=sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76 \ + --hash=sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5 \ + --hash=sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee \ + --hash=sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f \ + --hash=sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14 # via pyopenssl dateparser==1.1.8 \ --hash=sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f \ @@ -317,6 +321,7 @@ jsonschema==4.17.3 \ --hash=sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d \ --hash=sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6 # via + # aws-sam-cli (setup.py) # aws-sam-translator # cfn-lint junit-xml==1.9 \ @@ -497,9 +502,9 @@ python-slugify==8.0.1 \ --hash=sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395 \ --hash=sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27 # via cookiecutter -pytz==2023.2 \ - --hash=sha256:8a8baaf1e237175b02f5c751eea67168043a749c843989e2b3015aa1ad9db68b \ - --hash=sha256:a27dcf612c05d2ebde626f7d506555f10dfc815b3eddccfaadfc7d99b11c9a07 +pytz==2023.3 \ + --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ + --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb # via dateparser pyyaml==5.4.1 \ --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ @@ -636,9 +641,9 @@ requests==2.31.0 \ # aws-sam-cli (setup.py) # cookiecutter # docker -rich==13.3.3 \ - --hash=sha256:540c7d6d26a1178e8e8b37e9ba44573a3cd1464ff6348b99ee7061b95d1c6333 \ - --hash=sha256:dc84400a9d842b3a9c5ff74addd8eb798d155f36c1c91303888e0a66850d2a15 +rich==13.4.2 \ + --hash=sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec \ + --hash=sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898 # via aws-sam-cli (setup.py) ruamel-yaml==0.17.32 \ --hash=sha256:23cd2ed620231677564646b0c6a89d138b6822a0d78656df7abda5879ec4f447 \ @@ -713,9 +718,9 @@ tomlkit==0.11.8 \ --hash=sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171 \ --hash=sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3 # via aws-sam-cli (setup.py) -typing-extensions==4.4.0 \ - --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ - --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e +typing-extensions==4.7.1 \ + --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \ + --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2 # via # aws-sam-cli (setup.py) # aws-sam-translator diff --git a/samcli/__init__.py b/samcli/__init__.py index 1fea4bd55f..15e5e50faa 100644 --- a/samcli/__init__.py +++ b/samcli/__init__.py @@ -2,4 +2,4 @@ SAM CLI version """ -__version__ = "1.90.0" +__version__ = "1.91.0" diff --git a/samcli/cli/cli_config_file.py b/samcli/cli/cli_config_file.py index e4606e4555..bfc295c01b 100644 --- a/samcli/cli/cli_config_file.py +++ b/samcli/cli/cli_config_file.py @@ -10,49 +10,67 @@ import logging import os from pathlib import Path +from typing import Any, Callable, Dict, List, Optional import click +from click.core import ParameterSource from samcli.cli.context import get_cmd_names from samcli.commands.exceptions import ConfigException from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, DEFAULT_ENV, SamConfig -__all__ = ("TomlProvider", "configuration_option", "get_ctx_defaults") +__all__ = ("ConfigProvider", "configuration_option", "get_ctx_defaults") LOG = logging.getLogger(__name__) -class TomlProvider: +class ConfigProvider: """ - A parser for toml configuration files + A parser for sam configuration files """ def __init__(self, section=None, cmd_names=None): """ - The constructor for TomlProvider class - :param section: section defined in the configuration file nested within `cmd` - :param cmd_names: cmd_name defined in the configuration file + The constructor for ConfigProvider class + + Parameters + ---------- + section + The section defined in the configuration file nested within `cmd` + cmd_names + The cmd_name defined in the configuration file """ self.section = section self.cmd_names = cmd_names - def __call__(self, config_path, config_env, cmd_names): + def __call__(self, config_path: Path, config_env: str, cmd_names: List[str]) -> dict: """ Get resolved config based on the `file_path` for the configuration file, `config_env` targeted inside the config file and corresponding `cmd_name` as denoted by `click`. - :param config_path: The path of configuration file. - :param config_env: The name of the sectional config_env within configuration file. - :param list cmd_names: sam command name as defined by click - :returns dictionary containing the configuration parameters under specified config_env + Parameters + ---------- + config_path: Path + The path of configuration file. + config_env: str + The name of the sectional config_env within configuration file. + cmd_names: List[str] + The sam command name as defined by click. + + Returns + ------- + dict + A dictionary containing the configuration parameters under specified config_env. """ - resolved_config = {} + resolved_config: dict = {} # Use default sam config file name if config_path only contain the directory config_file_path = ( - Path(os.path.abspath(config_path)) if config_path else Path(os.getcwd(), DEFAULT_CONFIG_FILE_NAME) + Path(os.path.abspath(config_path)) + if config_path + else Path(os.getcwd(), SamConfig.get_default_file(os.getcwd())) ) config_file_name = config_file_path.name config_file_dir = config_file_path.parents[0] @@ -105,32 +123,56 @@ def __call__(self, config_path, config_env, cmd_names): return resolved_config -def configuration_callback(cmd_name, option_name, saved_callback, provider, ctx, param, value): +def configuration_callback( + cmd_name: str, + option_name: str, + saved_callback: Optional[Callable], + provider: Callable, + ctx: click.Context, + param: click.Parameter, + value, +): """ Callback for reading the config file. Also takes care of calling user specified custom callback afterwards. - :param cmd_name: `sam` command name derived from click. - :param option_name: The name of the option. This is used for error messages. - :param saved_callback: User-specified callback to be called later. - :param provider: A callable that parses the configuration file and returns a dictionary + Parameters + ---------- + cmd_name: str + The `sam` command name derived from click. + option_name: str + The name of the option. This is used for error messages. + saved_callback: Optional[Callable] + User-specified callback to be called later. + provider: Callable + A callable that parses the configuration file and returns a dictionary of the configuration parameters. Will be called as `provider(file_path, config_env, cmd_name)`. - :param ctx: Click context - :param param: Click parameter - :param value: Specified value for config_env - :returns specified callback or the specified value for config_env. + ctx: click.Context + Click context + param: click.Parameter + Click parameter + value + Specified value for config_env + + Returns + ------- + The specified callback or the specified value for config_env. """ # ctx, param and value are default arguments for click specified callbacks. ctx.default_map = ctx.default_map or {} - cmd_name = cmd_name or ctx.info_name + cmd_name = cmd_name or str(ctx.info_name) param.default = None config_env_name = ctx.params.get("config_env") or DEFAULT_ENV - config_file = ctx.params.get("config_file") or DEFAULT_CONFIG_FILE_NAME config_dir = getattr(ctx, "samconfig_dir", None) or os.getcwd() + config_file = ( # If given by default, check for other `samconfig` extensions first. Else use user-provided value + SamConfig.get_default_file(config_dir=config_dir) + if getattr(ctx.get_parameter_source("config_file"), "name", "") == ParameterSource.DEFAULT.name + else ctx.params.get("config_file") or SamConfig.get_default_file(config_dir=config_dir) + ) # If --config-file is an absolute path, use it, if not, start from config_dir config_file_path = config_file if os.path.isabs(config_file) else os.path.join(config_dir, config_file) if ( @@ -154,21 +196,35 @@ def configuration_callback(cmd_name, option_name, saved_callback, provider, ctx, return saved_callback(ctx, param, config_env_name) if saved_callback else config_env_name -def get_ctx_defaults(cmd_name, provider, ctx, config_env_name, config_file=None): +def get_ctx_defaults( + cmd_name: str, provider: Callable, ctx: click.Context, config_env_name: str, config_file: Optional[str] = None +) -> Any: """ Get the set of the parameters that are needed to be set into the click command. + This function also figures out the command name by looking up current click context's parent and constructing the parsed command name that is used in default configuration file. If a given cmd_name is start-api, the parsed name is "local_start_api". provider is called with `config_file`, `config_env_name` and `parsed_cmd_name`. - :param cmd_name: `sam` command name - :param provider: provider to be called for reading configuration file - :param ctx: Click context - :param config_env_name: config-env within configuration file, sam configuration file will be relative to the - supplied original template if its path is not specified - :param config_file: configuration file name - :return: dictionary of defaults for parameters + Parameters + ---------- + cmd_name: str + The `sam` command name. + provider: Callable + The provider to be called for reading configuration file. + ctx: click.Context + Click context + config_env_name: str + The config-env within configuration file, sam configuration file will be relative to the + supplied original template if its path is not specified. + config_file: Optional[str] + The configuration file name. + + Returns + ------- + Any + A dictionary of defaults for parameters. """ return provider(config_file, config_env_name, get_cmd_names(cmd_name, ctx)) @@ -180,30 +236,38 @@ def configuration_option(*param_decls, **attrs): """ Adds configuration file support to a click application. - NOTE: This decorator should be added to the top of parameter chain, right below click.command, before - any options are declared. - - Example: - >>> @click.command("hello") - @configuration_option(provider=TomlProvider(section="parameters")) - @click.option('--name', type=click.String) - def hello(name): - print("Hello " + name) - This will create a hidden click option whose callback function loads configuration parameters from default configuration environment [default] in default configuration file [samconfig.toml] in the template file directory. - :param preconfig_decorator_list: A list of click option decorator which need to place before this function. For - exmple, if we want to add option "--config-file" and "--config-env" to allow customized configuration file + + Note + ---- + This decorator should be added to the top of parameter chain, right below click.command, before + any options are declared. + + Example + ------- + >>> @click.command("hello") + @configuration_option(provider=ConfigProvider(section="parameters")) + @click.option('--name', type=click.String) + def hello(name): + print("Hello " + name) + + Parameters + ---------- + preconfig_decorator_list: list + A list of click option decorator which need to place before this function. For + example, if we want to add option "--config-file" and "--config-env" to allow customized configuration file and configuration environment, we will use configuration_option as below: @configuration_option( preconfig_decorator_list=[decorator_customize_config_file, decorator_customize_config_env], - provider=TomlProvider(section=CONFIG_SECTION), + provider=ConfigProvider(section=CONFIG_SECTION), ) By default, we enable these two options. - :param provider: A callable that parses the configuration file and returns a dictionary + provider: Callable + A callable that parses the configuration file and returns a dictionary of the configuration parameters. Will be called as - `provider(file_path, config_env, cmd_name) + `provider(file_path, config_env, cmd_name)` """ def decorator_configuration_setup(f): @@ -240,17 +304,25 @@ def decorator(f): return composed_decorator(decorator_list) -def decorator_customize_config_file(f): +def decorator_customize_config_file(f: Callable) -> Callable: """ CLI option to customize configuration file name. By default it is 'samconfig.toml' in project directory. Ex: --config-file samconfig.toml - :param f: Callback function passed by Click - :return: Callback function + + Parameters + ---------- + f: Callable + Callback function passed by Click + + Returns + ------- + Callable + A Callback function """ - config_file_attrs = {} + config_file_attrs: Dict[str, Any] = {} config_file_param_decls = ("--config-file",) config_file_attrs["help"] = "Configuration file containing default parameter values." - config_file_attrs["default"] = "samconfig.toml" + config_file_attrs["default"] = DEFAULT_CONFIG_FILE_NAME config_file_attrs["show_default"] = True config_file_attrs["is_eager"] = True config_file_attrs["required"] = False @@ -258,17 +330,25 @@ def decorator_customize_config_file(f): return click.option(*config_file_param_decls, **config_file_attrs)(f) -def decorator_customize_config_env(f): +def decorator_customize_config_env(f: Callable) -> Callable: """ CLI option to customize configuration environment name. By default it is 'default'. Ex: --config-env default - :param f: Callback function passed by Click - :return: Callback function + + Parameters + ---------- + f: Callable + Callback function passed by Click + + Returns + ------- + Callable + A Callback function """ - config_env_attrs = {} + config_env_attrs: Dict[str, Any] = {} config_env_param_decls = ("--config-env",) config_env_attrs["help"] = "Environment name specifying default parameter values in the configuration file." - config_env_attrs["default"] = "default" + config_env_attrs["default"] = DEFAULT_ENV config_env_attrs["show_default"] = True config_env_attrs["is_eager"] = True config_env_attrs["required"] = False diff --git a/samcli/commands/_utils/custom_options/hook_name_option.py b/samcli/commands/_utils/custom_options/hook_name_option.py index 745ab6c64a..a2cb334157 100644 --- a/samcli/commands/_utils/custom_options/hook_name_option.py +++ b/samcli/commands/_utils/custom_options/hook_name_option.py @@ -141,7 +141,7 @@ def _get_customer_input_beta_features_option(default_map, experimental_entry, op if beta_features is not None: return beta_features - # Get the beta-features flag value from the SamConfig toml file if provided. + # Get the beta-features flag value from the SamConfig file if provided. beta_features = default_map.get("beta_features") if beta_features is not None: return beta_features diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index a60f9954e1..86327d411d 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -27,7 +27,7 @@ from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options, print_cmdline_args from samcli.commands.build.core.command import BuildCommand from samcli.lib.telemetry.metric import track_command -from samcli.cli.cli_config_file import configuration_option, TomlProvider +from samcli.cli.cli_config_file import configuration_option, ConfigProvider from samcli.lib.utils.version_checker import check_newer_version from samcli.commands.build.click_container import ContainerOptions from samcli.commands.build.utils import MountMode @@ -69,7 +69,7 @@ short_help=HELP_TEXT, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=True, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"], diff --git a/samcli/commands/delete/delete_context.py b/samcli/commands/delete/delete_context.py index 08d327eceb..1424c87f4a 100644 --- a/samcli/commands/delete/delete_context.py +++ b/samcli/commands/delete/delete_context.py @@ -9,7 +9,7 @@ from botocore.exceptions import NoCredentialsError, NoRegionError from click import confirm, prompt -from samcli.cli.cli_config_file import TomlProvider +from samcli.cli.cli_config_file import ConfigProvider from samcli.commands.delete.exceptions import CfDeleteFailedStatusError from samcli.commands.exceptions import AWSServiceClientError, RegionError from samcli.lib.bootstrap.companion_stack.companion_stack_builder import CompanionStack @@ -82,8 +82,8 @@ def parse_config_file(self): """ Read the provided config file if it exists and assign the options values. """ - toml_provider = TomlProvider(CONFIG_SECTION, [CONFIG_COMMAND]) - config_options = toml_provider( + config_provider = ConfigProvider(CONFIG_SECTION, [CONFIG_COMMAND]) + config_options = config_provider( config_path=self.config_file, config_env=self.config_env, cmd_names=[CONFIG_COMMAND] ) if not config_options: diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index aeefe4d25f..557a601261 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.cdk_support_decorators import unsupported_command_cdk from samcli.commands._utils.click_mutex import ClickMutex @@ -75,7 +75,7 @@ description=DESCRIPTION, requires_credentials=True, ) -@configuration_option(provider=TomlProvider(section=CONFIG_SECTION)) +@configuration_option(provider=ConfigProvider(section=CONFIG_SECTION)) @click.option( "--guided", "-g", diff --git a/samcli/commands/deploy/guided_config.py b/samcli/commands/deploy/guided_config.py index b9d8ea59b5..78866944cd 100644 --- a/samcli/commands/deploy/guided_config.py +++ b/samcli/commands/deploy/guided_config.py @@ -7,7 +7,8 @@ from samcli.cli.context import get_cmd_names from samcli.commands.deploy.exceptions import GuidedDeployFailedError -from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, DEFAULT_ENV, SamConfig +from samcli.lib.config.exceptions import SamConfigFileReadException +from samcli.lib.config.samconfig import DEFAULT_ENV, SamConfig class GuidedConfig: @@ -19,20 +20,25 @@ def get_config_ctx(self, config_file=None): ctx = click.get_current_context() samconfig_dir = getattr(ctx, "samconfig_dir", None) + config_dir = samconfig_dir if samconfig_dir else SamConfig.config_dir(template_file_path=self.template_file) samconfig = SamConfig( - config_dir=samconfig_dir if samconfig_dir else SamConfig.config_dir(template_file_path=self.template_file), - filename=config_file or DEFAULT_CONFIG_FILE_NAME, + config_dir=config_dir, + filename=config_file or SamConfig.get_default_file(config_dir=config_dir), ) return ctx, samconfig def read_config_showcase(self, config_file=None): - _, samconfig = self.get_config_ctx(config_file) - - status = "Found" if samconfig.exists() else "Not found" msg = ( "Syntax invalid in samconfig.toml; save values " "through sam deploy --guided to overwrite file with a valid set of values." ) + try: + _, samconfig = self.get_config_ctx(config_file) + except SamConfigFileReadException: + raise GuidedDeployFailedError(msg) + + status = "Found" if samconfig.exists() else "Not found" + config_sanity = samconfig.sanity_check() click.secho("\nConfiguring SAM deploy\n======================", fg="yellow") click.echo(f"\n\tLooking for config file [{config_file}] : {status}") diff --git a/samcli/commands/init/command.py b/samcli/commands/init/command.py index 2806a96c72..6a416f3119 100644 --- a/samcli/commands/init/command.py +++ b/samcli/commands/init/command.py @@ -7,7 +7,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import common_options, pass_context, print_cmdline_args from samcli.commands._utils.click_mutex import ClickMutex from samcli.commands.init.core.command import InitCommand @@ -113,7 +113,7 @@ def wrapped(*args, **kwargs): description=DESCRIPTION, requires_credentials=False, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--no-interactive", is_flag=True, diff --git a/samcli/commands/list/endpoints/command.py b/samcli/commands/list/endpoints/command.py index 6de1c41bb2..f11543ef8c 100644 --- a/samcli/commands/list/endpoints/command.py +++ b/samcli/commands/list/endpoints/command.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.command_exception_handler import command_exception_handler from samcli.commands._utils.options import parameter_override_option, template_option_without_build @@ -21,7 +21,7 @@ @click.command(name="endpoints", help=HELP_TEXT) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @parameter_override_option @stack_name_option @output_option diff --git a/samcli/commands/list/resources/command.py b/samcli/commands/list/resources/command.py index 5dd2b41034..dacfac30e2 100644 --- a/samcli/commands/list/resources/command.py +++ b/samcli/commands/list/resources/command.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.command_exception_handler import command_exception_handler from samcli.commands._utils.options import parameter_override_option, template_option_without_build @@ -20,7 +20,7 @@ @click.command(name="resources", help=HELP_TEXT) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @parameter_override_option @stack_name_option @output_option diff --git a/samcli/commands/list/stack_outputs/command.py b/samcli/commands/list/stack_outputs/command.py index 3800c009b2..e988f98045 100644 --- a/samcli/commands/list/stack_outputs/command.py +++ b/samcli/commands/list/stack_outputs/command.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.command_exception_handler import command_exception_handler from samcli.commands.list.cli_common.options import output_option @@ -23,7 +23,7 @@ required=True, type=click.STRING, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @output_option @aws_creds_options @common_options diff --git a/samcli/commands/local/generate_event/event_generation.py b/samcli/commands/local/generate_event/event_generation.py index 5715bded73..9bf8e49e7a 100644 --- a/samcli/commands/local/generate_event/event_generation.py +++ b/samcli/commands/local/generate_event/event_generation.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.options import debug_option from samcli.lib.generated_sample_events import events from samcli.lib.telemetry.metric import track_command @@ -160,7 +160,7 @@ def get_command(self, ctx, cmd_name): callback=command_callback, ) - cmd = configuration_option(provider=TomlProvider(section="parameters"))(debug_option(cmd)) + cmd = configuration_option(provider=ConfigProvider(section="parameters"))(debug_option(cmd)) return cmd def list_commands(self, ctx): diff --git a/samcli/commands/local/invoke/cli.py b/samcli/commands/local/invoke/cli.py index a9a3fc9571..0442e7b7ed 100644 --- a/samcli/commands/local/invoke/cli.py +++ b/samcli/commands/local/invoke/cli.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled @@ -43,7 +43,7 @@ short_help=HELP_TEXT, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=False, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"] ) diff --git a/samcli/commands/local/lib/swagger/parser.py b/samcli/commands/local/lib/swagger/parser.py index 9c46e0c631..68ada78024 100644 --- a/samcli/commands/local/lib/swagger/parser.py +++ b/samcli/commands/local/lib/swagger/parser.py @@ -81,7 +81,7 @@ def get_authorizers(self, event_type: str = Route.API) -> Dict[str, Authorizer]: authorizers: Dict[str, Authorizer] = {} authorizer_dict = {} - document_version = self.swagger.get(SwaggerParser._SWAGGER) or self.swagger.get(SwaggerParser._OPENAPI) or "" + document_version = self._get_document_version() if document_version.startswith(SwaggerParser._2_X_VERSION): LOG.debug("Parsing Swagger document using 2.0 specification") @@ -240,6 +240,19 @@ def _get_lambda_identity_sources( return identity_sources + def _get_document_version(self) -> str: + """ + Helper method to fetch the Swagger document version + + Returns + ------- + str + A string representing a version, blank if not found + """ + document_version = self.swagger.get(SwaggerParser._SWAGGER) or self.swagger.get(SwaggerParser._OPENAPI) or "" + + return str(document_version) + def get_default_authorizer(self, event_type: str) -> Union[str, None]: """ Parses the body definition to find root level Authorizer definitions @@ -254,7 +267,7 @@ def get_default_authorizer(self, event_type: str) -> Union[str, None]: Union[str, None] Returns the name of the authorizer, if there is one defined, otherwise None """ - document_version = self.swagger.get(SwaggerParser._SWAGGER) or self.swagger.get(SwaggerParser._OPENAPI) or "" + document_version = self._get_document_version() authorizers = self.swagger.get(SwaggerParser._SWAGGER_SECURITY, []) if not authorizers: diff --git a/samcli/commands/local/start_api/cli.py b/samcli/commands/local/start_api/cli.py index 9de4d7982c..3e2f02b5f3 100644 --- a/samcli/commands/local/start_api/cli.py +++ b/samcli/commands/local/start_api/cli.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled @@ -58,7 +58,7 @@ requires_credentials=False, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=False, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"] ) diff --git a/samcli/commands/local/start_lambda/cli.py b/samcli/commands/local/start_lambda/cli.py index 9aaec50976..ded8b786fc 100644 --- a/samcli/commands/local/start_lambda/cli.py +++ b/samcli/commands/local/start_lambda/cli.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled @@ -52,7 +52,7 @@ requires_credentials=False, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=False, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"] ) diff --git a/samcli/commands/logs/command.py b/samcli/commands/logs/command.py index 1146767a60..f8df636b59 100644 --- a/samcli/commands/logs/command.py +++ b/samcli/commands/logs/command.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -49,7 +49,7 @@ description=DESCRIPTION, requires_credentials=True, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--name", "-n", diff --git a/samcli/commands/package/command.py b/samcli/commands/package/command.py index 41fb10b133..151661c91e 100644 --- a/samcli/commands/package/command.py +++ b/samcli/commands/package/command.py @@ -3,7 +3,7 @@ """ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.cdk_support_decorators import unsupported_command_cdk from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -67,7 +67,7 @@ def resources_and_properties_help_string(): description=DESCRIPTION, requires_credentials=True, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_click_option(include_build=True) @click.option( "--output-template-file", diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index d357adf5dd..7ba17e43fe 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -7,7 +7,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.click_mutex import ClickMutex from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -28,7 +28,7 @@ HELP_TEXT = """ This command generates the required AWS infrastructure resources to connect to your CI/CD system. -This step must be run for each deployment stage in your pipeline, prior to running the sam pipline init command. +This step must be run for each deployment stage in your pipeline, prior to running the sam pipeline init command. """ PIPELINE_CONFIG_DIR = os.path.join(".aws-sam", "pipeline") @@ -38,7 +38,7 @@ @click.command("bootstrap", short_help=SHORT_HELP, help=HELP_TEXT, context_settings=dict(max_content_width=120)) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--interactive/--no-interactive", is_flag=True, diff --git a/samcli/commands/pipeline/init/cli.py b/samcli/commands/pipeline/init/cli.py index 9e42f6e74b..ec675fb608 100644 --- a/samcli/commands/pipeline/init/cli.py +++ b/samcli/commands/pipeline/init/cli.py @@ -5,7 +5,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import common_options as cli_framework_options from samcli.cli.main import pass_context from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -24,7 +24,7 @@ @click.command("init", help=HELP_TEXT, short_help=SHORT_HELP) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--bootstrap", is_flag=True, diff --git a/samcli/commands/publish/command.py b/samcli/commands/publish/command.py index 091a044192..4ebfb3e8e5 100644 --- a/samcli/commands/publish/command.py +++ b/samcli/commands/publish/command.py @@ -7,7 +7,7 @@ import click from serverlessrepo.publish import CREATE_APPLICATION -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -44,7 +44,7 @@ @click.command("publish", help=HELP_TEXT, short_help=SHORT_HELP) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_common_option @click.option("--semantic-version", help=SEMANTIC_VERSION_HELP) @aws_creds_options diff --git a/samcli/commands/remote/exceptions.py b/samcli/commands/remote/exceptions.py index 68b0d5983f..e2a5c66b98 100644 --- a/samcli/commands/remote/exceptions.py +++ b/samcli/commands/remote/exceptions.py @@ -20,7 +20,7 @@ class UnsupportedServiceForRemoteInvoke(UserException): pass -class NoExecutorFoundForRemoteInvoke(UserException): +class ResourceNotSupportedForRemoteInvoke(UserException): pass diff --git a/samcli/commands/remote/invoke/cli.py b/samcli/commands/remote/invoke/cli.py index 3f3a771ea1..0318566b4a 100644 --- a/samcli/commands/remote/invoke/cli.py +++ b/samcli/commands/remote/invoke/cli.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.context import Context from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.cli.types import RemoteInvokeOutputFormatType @@ -42,7 +42,7 @@ requires_credentials=True, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option("--stack-name", required=False, help="Name of the stack to get the resource information from") @click.argument("resource-id", required=False) @click.option( diff --git a/samcli/commands/remote/remote_invoke_context.py b/samcli/commands/remote/remote_invoke_context.py index 90242b5142..b710df8410 100644 --- a/samcli/commands/remote/remote_invoke_context.py +++ b/samcli/commands/remote/remote_invoke_context.py @@ -11,8 +11,8 @@ AmbiguousResourceForRemoteInvoke, InvalidRemoteInvokeParameters, InvalidStackNameProvidedForRemoteInvoke, - NoExecutorFoundForRemoteInvoke, NoResourceFoundForRemoteInvoke, + ResourceNotSupportedForRemoteInvoke, UnsupportedServiceForRemoteInvoke, ) from samcli.lib.remote_invoke.remote_invoke_executor_factory import RemoteInvokeExecutorFactory @@ -70,8 +70,8 @@ def __exit__(self, *args) -> None: def run(self, remote_invoke_input: RemoteInvokeExecutionInfo) -> None: """ Instantiates remote invoke executor with populated resource summary information, executes it with the provided - input & returns its response back to the caller. If no executor can be instantiated it raises - NoExecutorFoundForRemoteInvoke exception. + input & returns its response back to the caller. If resource is not supported by command, raises + ResourceNotSupportedForRemoteInvoke exception. Parameters ---------- @@ -93,8 +93,8 @@ def run(self, remote_invoke_input: RemoteInvokeExecutionInfo) -> None: DefaultRemoteInvokeLogConsumer(self.stderr), ) if not remote_invoke_executor: - raise NoExecutorFoundForRemoteInvoke( - f"Resource type {self._resource_summary.resource_type} is not supported for remote invoke" + raise ResourceNotSupportedForRemoteInvoke( + f"Resource type {self._resource_summary.resource_type} is not supported for remote invoke." ) remote_invoke_executor.execute(remote_invoke_input) diff --git a/samcli/commands/sync/command.py b/samcli/commands/sync/command.py index 81f1222207..ddc5e2a165 100644 --- a/samcli/commands/sync/command.py +++ b/samcli/commands/sync/command.py @@ -5,7 +5,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.context import Context from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options @@ -114,7 +114,7 @@ requires_credentials=True, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_option_without_build @click.option( "--code", diff --git a/samcli/commands/traces/command.py b/samcli/commands/traces/command.py index 183a2bd156..d82b6871ab 100644 --- a/samcli/commands/traces/command.py +++ b/samcli/commands/traces/command.py @@ -5,7 +5,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -28,7 +28,7 @@ @click.command("traces", help=HELP_TEXT, short_help="Fetch AWS X-Ray traces") -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--trace-id", "-ti", diff --git a/samcli/commands/validate/validate.py b/samcli/commands/validate/validate.py index 1c5c2b28b1..221284b1ac 100644 --- a/samcli/commands/validate/validate.py +++ b/samcli/commands/validate/validate.py @@ -8,7 +8,7 @@ from botocore.exceptions import NoCredentialsError from samtranslator.translator.arn_generator import NoRegionFound -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.context import Context from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options @@ -35,7 +35,7 @@ requires_credentials=False, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_option_without_build @aws_creds_options @cli_framework_options diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index 50ddbd74e0..24df7dbad9 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -407,14 +407,18 @@ def _build_lambda_image(self, function_name: str, metadata: Dict, architecture: "dockerfile": dockerfile, "tag": docker_tag, "buildargs": docker_build_args, - "decode": True, "platform": get_docker_platform(architecture), "rm": True, } if docker_build_target: build_args["target"] = cast(str, docker_build_target) - build_logs = self._docker_client.api.build(**build_args) + try: + (build_image, build_logs) = self._docker_client.images.build(**build_args) + LOG.debug("%s image is built for %s function", build_image, function_name) + except docker.errors.BuildError as ex: + LOG.error("Failed building function %s", function_name) + raise DockerBuildFailed(str(ex)) from ex # The Docker-py low level api will stream logs back but if an exception is raised by the api # this is raised when accessing the generator. So we need to wrap accessing build_logs in a try: except. diff --git a/samcli/lib/config/exceptions.py b/samcli/lib/config/exceptions.py index 50297ce722..c179b4a13c 100644 --- a/samcli/lib/config/exceptions.py +++ b/samcli/lib/config/exceptions.py @@ -4,4 +4,12 @@ class SamConfigVersionException(Exception): - pass + """Exception for the `samconfig` file being not present or in unrecognized format""" + + +class FileParseException(Exception): + """Exception when a file is incorrectly parsed by a FileManager object.""" + + +class SamConfigFileReadException(Exception): + """Exception when a `samconfig` file is read incorrectly.""" diff --git a/samcli/lib/config/file_manager.py b/samcli/lib/config/file_manager.py new file mode 100644 index 0000000000..0629ace318 --- /dev/null +++ b/samcli/lib/config/file_manager.py @@ -0,0 +1,342 @@ +""" +Class to represent the parsing of different file types into Python objects. +""" + + +import json +import logging +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Dict, Type + +import tomlkit +from ruamel.yaml import YAML, YAMLError +from ruamel.yaml.compat import StringIO + +from samcli.lib.config.exceptions import FileParseException + +LOG = logging.getLogger(__name__) +COMMENT_KEY = "__comment__" + + +class FileManager(ABC): + """ + Abstract class to be overridden by file managers for specific file extensions. + """ + + @staticmethod + @abstractmethod + def read(filepath: Path) -> Any: + """ + Read a file at a given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary-like representation of the contents at the filepath location. + """ + raise NotImplementedError("Read method not implemented.") + + @staticmethod + @abstractmethod + def write(document: dict, filepath: Path): + """ + Write a dictionary or dictionary-like object to a given file. + + Parameters + ---------- + document: dict + The object to write. + filepath: Path + The final location for the document to be written. + """ + raise NotImplementedError("Write method not implemented.") + + @staticmethod + @abstractmethod + def put_comment(document: Any, comment: str) -> Any: + """ + Put a comment in a document object. + + Parameters + ---------- + document: Any + The object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new document, with the comment added to it. + """ + raise NotImplementedError("Put comment method not implemented.") + + +class TomlFileManager(FileManager): + """ + Static class to read and write toml files. + """ + + file_format = "TOML" + + @staticmethod + def read(filepath: Path) -> Any: + """ + Read a TOML file at the given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary-like tomlkit.TOMLDocument object, which represents the contents of the TOML file at the + provided location. + """ + toml_doc = tomlkit.document() + try: + txt = filepath.read_text() + toml_doc = tomlkit.loads(txt) + except OSError as e: + LOG.debug(f"OSError occurred while reading {TomlFileManager.file_format} file: {str(e)}") + except tomlkit.exceptions.TOMLKitError as e: + raise FileParseException(e) from e + + return toml_doc + + @staticmethod + def write(document: dict, filepath: Path): + """ + Write the contents of a dictionary or tomlkit.TOMLDocument to a TOML file at the provided location. + + Parameters + ---------- + document: dict + The object to write. + filepath: Path + The final location for the TOML file to be written. + """ + if not document: + LOG.debug("Nothing for TomlFileManager to write.") + return + + toml_document = TomlFileManager._to_toml(document) + + if toml_document.get(COMMENT_KEY, None): # Remove dunder comments that may be residue from other formats + toml_document.add(tomlkit.comment(toml_document.get(COMMENT_KEY, ""))) + toml_document.pop(COMMENT_KEY) + + filepath.write_text(tomlkit.dumps(toml_document)) + + @staticmethod + def put_comment(document: dict, comment: str) -> Any: + """ + Put a comment in a document object. + + Parameters + ---------- + document: Any + The tomlkit.TOMLDocument object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new TOMLDocument, with the comment added to it. + """ + document = TomlFileManager._to_toml(document) + document.add(tomlkit.comment(comment)) + return document + + @staticmethod + def _to_toml(document: dict) -> tomlkit.TOMLDocument: + """Ensure that a dictionary-like object is a TOMLDocument.""" + return tomlkit.parse(tomlkit.dumps(document)) + + +class YamlFileManager(FileManager): + """ + Static class to read and write yaml files. + """ + + yaml = YAML() + file_format = "YAML" + + @staticmethod + def read(filepath: Path) -> Any: + """ + Read a YAML file at the given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary-like yaml object, which represents the contents of the YAML file at the + provided location. + """ + yaml_doc = {} + try: + yaml_doc = YamlFileManager.yaml.load(filepath.read_text()) + except OSError as e: + LOG.debug(f"OSError occurred while reading {YamlFileManager.file_format} file: {str(e)}") + except YAMLError as e: + raise FileParseException(e) from e + + return yaml_doc + + @staticmethod + def write(document: dict, filepath: Path): + """ + Write the contents of a dictionary to a YAML file at the provided location. + + Parameters + ---------- + document: dict + The object to write. + filepath: Path + The final location for the YAML file to be written. + """ + if not document: + LOG.debug("No document given to YamlFileManager to write.") + return + + yaml_doc = YamlFileManager._to_yaml(document) + + if yaml_doc.get(COMMENT_KEY, None): # Comment appears at the top of doc + yaml_doc.yaml_set_start_comment(document[COMMENT_KEY]) + yaml_doc.pop(COMMENT_KEY) + + YamlFileManager.yaml.dump(yaml_doc, filepath) + + @staticmethod + def put_comment(document: Any, comment: str) -> Any: + """ + Put a comment in a document object. + + Parameters + ---------- + document: Any + The yaml object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new yaml document, with the comment added to it. + """ + document = YamlFileManager._to_yaml(document) + document.yaml_set_start_comment(comment) + return document + + @staticmethod + def _to_yaml(document: dict) -> Any: + """ + Ensure a dictionary-like object is a YAML document. + + Parameters + ---------- + document: dict + A dictionary-like object to parse. + + Returns + ------- + Any + A dictionary-like YAML object, as derived from `yaml.load()`. + """ + with StringIO() as stream: + YamlFileManager.yaml.dump(document, stream) + return YamlFileManager.yaml.load(stream.getvalue()) + + +class JsonFileManager(FileManager): + """ + Static class to read and write json files. + """ + + file_format = "JSON" + INDENT_SIZE = 2 + + @staticmethod + def read(filepath: Path) -> Any: + """ + Read a JSON file at a given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary representation of the contents of the JSON document. + """ + json_file = {} + try: + json_file = json.loads(filepath.read_text()) + except OSError as e: + LOG.debug(f"OSError occurred while reading {JsonFileManager.file_format} file: {str(e)}") + except json.JSONDecodeError as e: + raise FileParseException(e) from e + return json_file + + @staticmethod + def write(document: dict, filepath: Path): + """ + Write a dictionary or dictionary-like object to a JSON file. + + Parameters + ---------- + document: dict + The JSON object to write. + filepath: Path + The final location for the document to be written. + """ + if not document: + LOG.debug("No document given to JsonFileManager to write.") + return + + with filepath.open("w") as file: + json.dump(document, file, indent=JsonFileManager.INDENT_SIZE) + + @staticmethod + def put_comment(document: Any, comment: str) -> Any: + """ + Put a comment in a JSON object. + + Parameters + ---------- + document: Any + The JSON object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new JSON dictionary object, with the comment added to it. + """ + document.update({COMMENT_KEY: comment}) + return document + + +FILE_MANAGER_MAPPER: Dict[str, Type[FileManager]] = { # keys ordered by priority + ".toml": TomlFileManager, + ".yaml": YamlFileManager, + ".yml": YamlFileManager, + # ".json": JsonFileManager, # JSON support disabled +} diff --git a/samcli/lib/config/samconfig.py b/samcli/lib/config/samconfig.py index 2c3ba303a6..2cd0d41199 100644 --- a/samcli/lib/config/samconfig.py +++ b/samcli/lib/config/samconfig.py @@ -7,26 +7,25 @@ from pathlib import Path from typing import Any, Iterable -import tomlkit - -from samcli.lib.config.exceptions import SamConfigVersionException +from samcli.lib.config.exceptions import FileParseException, SamConfigFileReadException, SamConfigVersionException +from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER from samcli.lib.config.version import SAM_CONFIG_VERSION, VERSION_KEY +from samcli.lib.telemetry.event import EventTracker LOG = logging.getLogger(__name__) -DEFAULT_CONFIG_FILE_EXTENSION = "toml" -DEFAULT_CONFIG_FILE_NAME = f"samconfig.{DEFAULT_CONFIG_FILE_EXTENSION}" +DEFAULT_CONFIG_FILE_EXTENSION = ".toml" +DEFAULT_CONFIG_FILE = "samconfig" +DEFAULT_CONFIG_FILE_NAME = DEFAULT_CONFIG_FILE + DEFAULT_CONFIG_FILE_EXTENSION DEFAULT_ENV = "default" DEFAULT_GLOBAL_CMDNAME = "global" class SamConfig: """ - Class to interface with `samconfig.toml` file. + Class to represent `samconfig` config options. """ - document = None - def __init__(self, config_dir, filename=None): """ Initialize the class @@ -39,11 +38,23 @@ def __init__(self, config_dir, filename=None): Optional. Name of the configuration file. It is recommended to stick with default so in the future we could automatically support auto-resolving multiple config files within same directory. """ - self.filepath = Path(config_dir, filename or DEFAULT_CONFIG_FILE_NAME) + self.document = {} + self.filepath = Path(config_dir, filename or self.get_default_file(config_dir=config_dir)) + file_extension = self.filepath.suffix + self.file_manager = FILE_MANAGER_MAPPER.get(file_extension, None) + if not self.file_manager: + LOG.warning( + f"The config file extension '{file_extension}' is not supported. " + f"Supported formats are: [{'|'.join(FILE_MANAGER_MAPPER.keys())}]" + ) + raise SamConfigFileReadException( + f"The config file {self.filepath} uses an unsupported extension, and cannot be read." + ) + self._read() + EventTracker.track_event("SamConfigFileExtension", file_extension) def get_stage_configuration_names(self): - self._read() - if isinstance(self.document, dict): + if self.document: return [stage for stage, value in self.document.items() if isinstance(value, dict)] return [] @@ -69,23 +80,19 @@ def get_all(self, cmd_names, section, env=DEFAULT_ENV): ------ KeyError If the config file does *not* have the specific section - - tomlkit.exceptions.TOMLKitError - If the configuration file is invalid """ env = env or DEFAULT_ENV - self._read() - if isinstance(self.document, dict): - toml_content = self.document.get(env, {}) - params = toml_content.get(self.to_key(cmd_names), {}).get(section, {}) - if DEFAULT_GLOBAL_CMDNAME in toml_content: - global_params = toml_content.get(DEFAULT_GLOBAL_CMDNAME, {}).get(section, {}) - global_params.update(params.copy()) - params = global_params.copy() - return params - return {} + self.document = self._read() + + config_content = self.document.get(env, {}) + params = config_content.get(self.to_key(cmd_names), {}).get(section, {}) + if DEFAULT_GLOBAL_CMDNAME in config_content: + global_params = config_content.get(DEFAULT_GLOBAL_CMDNAME, {}).get(section, {}) + global_params.update(params.copy()) + params = global_params.copy() + return params def put(self, cmd_names, section, key, value, env=DEFAULT_ENV): """ @@ -102,20 +109,10 @@ def put(self, cmd_names, section, key, value, env=DEFAULT_ENV): key : str Key to write the data under value : Any - Value to write. Could be any of the supported TOML types. + Value to write. Could be any of the supported types. env : str Optional, Name of the environment - - Raises - ------ - tomlkit.exceptions.TOMLKitError - If the data is invalid """ - - if self.document is None: - # Checking for None here since a TOMLDocument can include a - # 'body' property but still be falsy without a 'value' property - self._read() # Empty document prepare the initial structure. # self.document is a nested dict, we need to check each layer and add new tables, otherwise duplicated key # in parent layer will override the whole child layer @@ -144,20 +141,12 @@ def put_comment(self, comment): comment: str A comment to write to the samconfg file """ - if self.document is None: - self._read() - self.document.add(tomlkit.comment(comment)) + self.document = self.file_manager.put_comment(self.document, comment) def flush(self): """ Write the data back to file - - Raises - ------ - tomlkit.exceptions.TOMLKitError - If the data is invalid - """ self._write() @@ -167,7 +156,7 @@ def sanity_check(self): """ try: self._read() - except tomlkit.exceptions.TOMLKitError: + except SamConfigFileReadException: return False else: return True @@ -196,13 +185,10 @@ def config_dir(template_file_path=None): def _read(self): if not self.document: try: - txt = self.filepath.read_text() - self.document = tomlkit.loads(txt) - self._version_sanity_check(self._version()) - except OSError: - self.document = tomlkit.document() - - if self.document.body: + self.document = self.file_manager.read(self.filepath) + except FileParseException as e: + raise SamConfigFileReadException(e) from e + if self.document: self._version_sanity_check(self._version()) return self.document @@ -213,12 +199,9 @@ def _write(self): self._ensure_exists() current_version = self._version() if self._version() else SAM_CONFIG_VERSION - try: - self.document.add(VERSION_KEY, current_version) - except tomlkit.exceptions.KeyAlreadyPresent: - # NOTE(TheSriram): Do not attempt to re-write an existing version - pass - self.filepath.write_text(tomlkit.dumps(self.document)) + self.document.update({VERSION_KEY: current_version}) + + self.file_manager.write(self.document, self.filepath) def _version(self): return self.document.get(VERSION_KEY, None) @@ -261,6 +244,40 @@ def _deduplicate_global_parameters(self, cmd_name_key, section, key, env=DEFAULT # Only keep the global parameter del self.document[env][cmd_name_key][section][key] + @staticmethod + def get_default_file(config_dir: str) -> str: + """Return a defaultly-named config file, if it exists, otherwise the current default. + + Parameters + ---------- + config_dir: str + The name of the directory where the config file is/will be stored. + + Returns + ------- + str + The name of the config file found, if it exists. In the case that it does not exist, the default config + file name is returned instead. + """ + config_files_found = 0 + config_file = DEFAULT_CONFIG_FILE_NAME + + for extension in reversed(list(FILE_MANAGER_MAPPER.keys())): + filename = DEFAULT_CONFIG_FILE + extension + if Path(config_dir, filename).exists(): + config_files_found += 1 + config_file = filename + + if config_files_found == 0: # Config file doesn't exist (yet!) + LOG.debug("No config file found in this directory.") + elif config_files_found > 1: # Multiple config files; let user know which is used + LOG.info( + f"More than one samconfig file found; using {config_file}." + f" To use another config file, please specify it using the '--config-file' flag." + ) + + return config_file + @staticmethod def _version_sanity_check(version: Any) -> None: if not isinstance(version, float): diff --git a/samcli/lib/remote_invoke/lambda_invoke_executors.py b/samcli/lib/remote_invoke/lambda_invoke_executors.py index 323aeceba3..911bdb0a96 100644 --- a/samcli/lib/remote_invoke/lambda_invoke_executors.py +++ b/samcli/lib/remote_invoke/lambda_invoke_executors.py @@ -6,11 +6,12 @@ import logging from abc import ABC, abstractmethod from json import JSONDecodeError -from typing import Any, cast +from typing import cast from botocore.eventstream import EventStream from botocore.exceptions import ClientError, ParamValidationError from botocore.response import StreamingBody +from mypy_boto3_lambda.client import LambdaClient from samcli.lib.remote_invoke.exceptions import ( ErrorBotoApiCallException, @@ -46,11 +47,11 @@ class AbstractLambdaInvokeExecutor(BotoActionExecutor, ABC): For Payload parameter, if a file location provided, the file handle will be passed as Payload object """ - _lambda_client: Any + _lambda_client: LambdaClient _function_name: str _remote_output_format: RemoteInvokeOutputFormat - def __init__(self, lambda_client: Any, function_name: str, remote_output_format: RemoteInvokeOutputFormat): + def __init__(self, lambda_client: LambdaClient, function_name: str, remote_output_format: RemoteInvokeOutputFormat): self._lambda_client = lambda_client self._function_name = function_name self._remote_output_format = remote_output_format @@ -60,7 +61,10 @@ def validate_action_parameters(self, parameters: dict) -> None: """ Validates the input boto parameters and prepares the parameters for calling the API. - :param parameters: Boto parameters provided as input + Parameters + ---------- + parameters: dict + Boto parameters provided as input """ for parameter_key, parameter_value in parameters.items(): if parameter_key == FUNCTION_NAME: @@ -82,7 +86,7 @@ def _execute_boto_call(self, boto_client_method) -> dict: except ParamValidationError as param_val_ex: raise InvalidResourceBotoParameterException( f"Invalid parameter key provided." - f" {str(param_val_ex).replace('{FUNCTION_NAME}, ', '').replace('{PAYLOAD}, ', '')}" + f" {str(param_val_ex).replace(f'{FUNCTION_NAME}, ', '').replace(f'{PAYLOAD}, ', '')}" ) from param_val_ex except ClientError as client_ex: if boto_utils.get_client_error_code(client_ex) == "ValidationException": @@ -215,7 +219,7 @@ def map(self, remote_invoke_input: RemoteInvokeResponse) -> RemoteInvokeResponse return remote_invoke_input -def _is_function_invoke_mode_response_stream(lambda_client: Any, function_name: str): +def _is_function_invoke_mode_response_stream(lambda_client: LambdaClient, function_name: str): """ Returns True if given function has RESPONSE_STREAM as InvokeMode, False otherwise """ diff --git a/samcli/lib/remote_invoke/remote_invoke_executor_factory.py b/samcli/lib/remote_invoke/remote_invoke_executor_factory.py index 19bf7ff106..14c93bb7bc 100644 --- a/samcli/lib/remote_invoke/remote_invoke_executor_factory.py +++ b/samcli/lib/remote_invoke/remote_invoke_executor_factory.py @@ -20,10 +20,12 @@ RemoteInvokeResponse, ResponseObjectToJsonStringMapper, ) -from samcli.lib.utils.cloudformation import CloudFormationResourceSummary -from samcli.lib.utils.resources import ( - AWS_LAMBDA_FUNCTION, +from samcli.lib.remote_invoke.stepfunctions_invoke_executors import ( + SfnDescribeExecutionResponseConverter, + StepFunctionsStartExecutionExecutor, ) +from samcli.lib.utils.cloudformation import CloudFormationResourceSummary +from samcli.lib.utils.resources import AWS_LAMBDA_FUNCTION LOG = logging.getLogger(__name__) @@ -85,11 +87,23 @@ def _create_lambda_boto_executor( """Creates a remote invoke executor for Lambda resource type based on the boto action being called. - :param cfn_resource_summary: Information about the Lambda resource + Parameters + ---------- + cfn_resource_summary: CloudFormationResourceSummary + Information about the Lambda resource + remote_invoke_output_format: RemoteInvokeOutputFormat + Response output format that will be used for remote invoke execution + response_consumer: RemoteInvokeConsumer[RemoteInvokeResponse] + Consumer instance which can process RemoteInvokeResponse events + log_consumer: RemoteInvokeConsumer[RemoteInvokeLogOutput] + Consumer instance which can process RemoteInvokeLogOutput events - :return: Returns the created remote invoke Executor + Returns + ------- + RemoteInvokeExecutor + Returns the Executor created for Lambda """ - LOG.info(f"Invoking Lambda Function {cfn_resource_summary.logical_resource_id}") + LOG.info("Invoking Lambda Function %s", cfn_resource_summary.logical_resource_id) lambda_client = self._boto_client_provider("lambda") mappers = [] if _is_function_invoke_mode_response_stream(lambda_client, cfn_resource_summary.physical_resource_id): @@ -127,6 +141,50 @@ def _create_lambda_boto_executor( log_consumer=log_consumer, ) + def _create_stepfunctions_boto_executor( + self, + cfn_resource_summary: CloudFormationResourceSummary, + remote_invoke_output_format: RemoteInvokeOutputFormat, + response_consumer: RemoteInvokeConsumer[RemoteInvokeResponse], + log_consumer: RemoteInvokeConsumer[RemoteInvokeLogOutput], + ) -> RemoteInvokeExecutor: + """Creates a remote invoke executor for Step Functions resource type based on + the boto action being called. + + Parameters + ---------- + cfn_resource_summary: CloudFormationResourceSummary + Information about the Step Function resource + remote_invoke_output_format: RemoteInvokeOutputFormat + Response output format that will be used for remote invoke execution + response_consumer: RemoteInvokeConsumer[RemoteInvokeResponse] + Consumer instance which can process RemoteInvokeResponse events + log_consumer: RemoteInvokeConsumer[RemoteInvokeLogOutput] + Consumer instance which can process RemoteInvokeLogOutput events + + Returns + ------- + RemoteInvokeExecutor + Returns the Executor created for Step Functions + """ + LOG.info("Invoking Step Function %s", cfn_resource_summary.logical_resource_id) + sfn_client = self._boto_client_provider("stepfunctions") + mappers = [] + if remote_invoke_output_format == RemoteInvokeOutputFormat.JSON: + mappers = [ + SfnDescribeExecutionResponseConverter(), + ResponseObjectToJsonStringMapper(), + ] + return RemoteInvokeExecutor( + request_mappers=[DefaultConvertToJSON()], + response_mappers=mappers, + boto_action_executor=StepFunctionsStartExecutionExecutor( + sfn_client, cfn_resource_summary.physical_resource_id, remote_invoke_output_format + ), + response_consumer=response_consumer, + log_consumer=log_consumer, + ) + # mapping definition for each supported resource type REMOTE_INVOKE_EXECUTOR_MAPPING: Dict[ str, @@ -140,6 +198,4 @@ def _create_lambda_boto_executor( ], RemoteInvokeExecutor, ], - ] = { - AWS_LAMBDA_FUNCTION: _create_lambda_boto_executor, - } + ] = {AWS_LAMBDA_FUNCTION: _create_lambda_boto_executor} diff --git a/samcli/lib/remote_invoke/stepfunctions_invoke_executors.py b/samcli/lib/remote_invoke/stepfunctions_invoke_executors.py new file mode 100644 index 0000000000..df8c9b2b4e --- /dev/null +++ b/samcli/lib/remote_invoke/stepfunctions_invoke_executors.py @@ -0,0 +1,154 @@ +""" +Remote invoke executor implementation for Step Functions +""" +import logging +import time +from datetime import datetime +from typing import cast + +from botocore.exceptions import ClientError, ParamValidationError +from mypy_boto3_stepfunctions import SFNClient + +from samcli.lib.remote_invoke.exceptions import ( + ErrorBotoApiCallException, + InvalideBotoResponseException, + InvalidResourceBotoParameterException, +) +from samcli.lib.remote_invoke.remote_invoke_executors import ( + BotoActionExecutor, + RemoteInvokeIterableResponseType, + RemoteInvokeLogOutput, + RemoteInvokeOutputFormat, + RemoteInvokeRequestResponseMapper, + RemoteInvokeResponse, +) + +LOG = logging.getLogger(__name__) +STATE_MACHINE_ARN = "stateMachineArn" +INPUT = "input" +RUNNING = "RUNNING" +SFN_EXECUTION_WAIT_TIME = 2 + + +class StepFunctionsStartExecutionExecutor(BotoActionExecutor): + """ + Calls "start_execution" method of "Step Functions" service with given input. + If a file location provided, the file handle will be passed as input object. + Calls "describe_execution" method after the executions starts to get more + execution details. + """ + + _stepfunctions_client: SFNClient + _state_machine_arn: str + _remote_output_format: RemoteInvokeOutputFormat + request_parameters: dict + + def __init__( + self, stepfunctions_client: SFNClient, physical_id: str, remote_output_format: RemoteInvokeOutputFormat + ): + self._stepfunctions_client = stepfunctions_client + self._remote_output_format = remote_output_format + self._state_machine_arn = physical_id + self.request_parameters = {} + + def validate_action_parameters(self, parameters: dict) -> None: + """ + Validates the input boto parameters and prepares the parameters for calling the API. + + Parameters + ---------- + parameters: dict + Boto parameters provided as input + """ + for parameter_key, parameter_value in parameters.items(): + if parameter_key == "stateMachineArn": + LOG.warning("stateMachineArn is defined using the value provided for resource_id argument.") + elif parameter_key == "input": + LOG.warning("input is defined using the value provided for either --event or --event-file options.") + else: + self.request_parameters[parameter_key] = parameter_value + + if not self.request_parameters.get("name"): + current_datetime = datetime.now().strftime("%Y%m%dT%H%M%S") + self.request_parameters["name"] = f"sam_remote_invoke_{current_datetime}" + + def _execute_action(self, payload: str) -> RemoteInvokeIterableResponseType: + """ + Calls "start_execution" method to start the execution and waits + for the execution to complete using the "describe_execution" method + + Parameters + ---------- + payload: str + The input which is passed to the execution + + Yields + ------ + RemoteInvokeIterableResponseType + Response that is consumed by remote invoke consumers after execution + """ + self.request_parameters[INPUT] = payload + self.request_parameters[STATE_MACHINE_ARN] = self._state_machine_arn + LOG.debug( + "Calling stepfunctions_client.start_execution with name:%s, input:%s, stateMachineArn:%s", + self.request_parameters["name"], + payload, + self._state_machine_arn, + ) + try: + start_execution_response = self._stepfunctions_client.start_execution(**self.request_parameters) + execution_arn = start_execution_response["executionArn"] + + execution_status = RUNNING + while execution_status == RUNNING: + describe_execution_response = cast( + dict, self._stepfunctions_client.describe_execution(executionArn=execution_arn) + ) + execution_status = describe_execution_response["status"] + LOG.debug("ExecutionArn: %s, status: %s", execution_arn, execution_status) + # Sleep to avoid throttling the API for longer executions + time.sleep(SFN_EXECUTION_WAIT_TIME) + + if self._remote_output_format == RemoteInvokeOutputFormat.JSON: + yield RemoteInvokeResponse(describe_execution_response) + if self._remote_output_format == RemoteInvokeOutputFormat.TEXT: + output_data = describe_execution_response.get("output", "") + error_data = describe_execution_response.get("error", "") + if output_data: + yield RemoteInvokeResponse(output_data) + return + if error_data: + error_cause = describe_execution_response.get("cause", "") + yield RemoteInvokeLogOutput( + f"The execution failed due to the error: {error_data} and cause: {error_cause}" + ) + return + except ParamValidationError as param_val_ex: + raise InvalidResourceBotoParameterException( + f"Invalid parameter key provided." + f" {str(param_val_ex).replace(f'{STATE_MACHINE_ARN}, ', '').replace(f'{INPUT}, ', '')}" + ) + except ClientError as client_ex: + raise ErrorBotoApiCallException(client_ex) from client_ex + + +class SfnDescribeExecutionResponseConverter(RemoteInvokeRequestResponseMapper[RemoteInvokeResponse]): + """ + This class helps to convert response from Step Function service. + This class converts any datetime objects in the response into strings + """ + + def map(self, remote_invoke_input: RemoteInvokeResponse) -> RemoteInvokeResponse: + LOG.debug("Mapping Step Function execution response to string object") + if not isinstance(remote_invoke_input.response, dict): + raise InvalideBotoResponseException( + "Invalid response type received from Step Functions service, expecting dict" + ) + + start_date_field = remote_invoke_input.response.get("startDate") + stop_date_field = remote_invoke_input.response.get("stopDate") + if start_date_field: + remote_invoke_input.response["startDate"] = start_date_field.strftime("%Y-%m-%d %H:%M:%S.%f%z") + if stop_date_field: + remote_invoke_input.response["stopDate"] = stop_date_field.strftime("%Y-%m-%d %H:%M:%S.%f%z") + return remote_invoke_input diff --git a/samcli/lib/telemetry/event.py b/samcli/lib/telemetry/event.py index 2d819e37bf..2c35748951 100644 --- a/samcli/lib/telemetry/event.py +++ b/samcli/lib/telemetry/event.py @@ -11,6 +11,7 @@ from samcli.cli.context import Context from samcli.lib.build.workflows import ALL_CONFIGS +from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER from samcli.lib.telemetry.telemetry import Telemetry from samcli.local.common.runtime_template import INIT_RUNTIMES @@ -26,6 +27,7 @@ class EventName(Enum): SYNC_FLOW_START = "SyncFlowStart" SYNC_FLOW_END = "SyncFlowEnd" BUILD_WORKFLOW_USED = "BuildWorkflowUsed" + CONFIG_FILE_EXTENSION = "SamConfigFileExtension" class UsedFeature(Enum): @@ -69,6 +71,7 @@ class EventType: EventName.SYNC_FLOW_START: _SYNC_FLOWS, EventName.SYNC_FLOW_END: _SYNC_FLOWS, EventName.BUILD_WORKFLOW_USED: _WORKFLOWS, + EventName.CONFIG_FILE_EXTENSION: list(FILE_MANAGER_MAPPER.keys()), } @staticmethod diff --git a/tests/end_to_end/end_to_end_base.py b/tests/end_to_end/end_to_end_base.py index 926a325e86..97ad1cf3c1 100644 --- a/tests/end_to_end/end_to_end_base.py +++ b/tests/end_to_end/end_to_end_base.py @@ -1,3 +1,5 @@ +from pathlib import Path + import os from typing import List @@ -25,9 +27,11 @@ class EndToEndBase(InitIntegBase, StackOutputsIntegBase, DeleteIntegBase, SyncIn def setUp(self): super().setUp() + e2e_dir = Path(__file__).resolve().parent self.stacks = [] self.config_file_dir = GlobalConfig().config_dir self._create_config_dir() + self.e2e_test_data_path = Path(e2e_dir, "testdata") def _create_config_dir(self): # Init tests will lock the config dir, ensure it exists before obtaining a lock diff --git a/tests/end_to_end/test_runtimes_e2e.py b/tests/end_to_end/test_runtimes_e2e.py index 9955c28341..b5d7346161 100644 --- a/tests/end_to_end/test_runtimes_e2e.py +++ b/tests/end_to_end/test_runtimes_e2e.py @@ -175,7 +175,7 @@ def test_integration(self): event = '{"hello": "world"}' stack_name = self._method_to_stack_name(self.id()) with EndToEndTestContext(self.app_name) as e2e_context: - project_path = str(Path("testdata") / "esbuild-datadog-integration") + project_path = str(self.e2e_test_data_path / "esbuild-datadog-integration") os.mkdir(e2e_context.project_directory) copy_tree(project_path, e2e_context.project_directory) self.template_path = e2e_context.template_path diff --git a/tests/end_to_end/testdata/esbuild-datadog-integration/package.json b/tests/end_to_end/testdata/esbuild-datadog-integration/package.json new file mode 100644 index 0000000000..8a10aa75a1 --- /dev/null +++ b/tests/end_to_end/testdata/esbuild-datadog-integration/package.json @@ -0,0 +1,12 @@ +{ + "name": "npmdeps", + "version": "1.0.0", + "description": "", + "keywords": [], + "author": "", + "license": "APACHE2.0", + "main": "main.js", + "dependencies": { + "esbuild": "^0.14.14" + } +} \ No newline at end of file diff --git a/tests/end_to_end/testdata/esbuild-datadog-integration/template.yaml b/tests/end_to_end/testdata/esbuild-datadog-integration/template.yaml index 3341557f3e..8f055237df 100644 --- a/tests/end_to_end/testdata/esbuild-datadog-integration/template.yaml +++ b/tests/end_to_end/testdata/esbuild-datadog-integration/template.yaml @@ -16,6 +16,7 @@ Resources: Properties: Handler: /opt/nodejs/node_modules/datadog-lambda-js/handler.handler Runtime: nodejs18.x + Timeout: 15 Environment: Variables: DD_LAMBDA_HANDLER: main.lambdaHandler diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 13ba14f310..2dab556843 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -83,6 +83,7 @@ def get_command_list( beta_features=None, build_in_source=None, mount_with=None, + config_file=None, ): command_list = [self.cmd, "build"] @@ -146,6 +147,9 @@ def get_command_list( if build_in_source is not None: command_list += ["--build-in-source"] if build_in_source else ["--no-build-in-source"] + if config_file is not None: + command_list += ["--config-file", config_file] + return command_list def verify_docker_container_cleanedup(self, runtime): diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 2b35d7c6e9..b45d2a2418 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Set from unittest import skipIf +from uuid import uuid4 import jmespath import docker @@ -49,6 +50,39 @@ SKIP_SAR_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY +@skipIf(SKIP_DOCKER_TESTS, SKIP_DOCKER_MESSAGE) +class TestBuildingImageTypeLambdaDockerFileFailures(BuildIntegBase): + template = "template_image.yaml" + + def test_with_invalid_dockerfile_location(self): + overrides = { + "Runtime": "3.10", + "Handler": "handler", + "DockerFile": "ThisDockerfileDoesNotExist", + "Tag": uuid4().hex, + } + cmdlist = self.get_command_list(parameter_overrides=overrides) + command_result = run_command(cmdlist, cwd=self.working_dir) + + # confirm build failed + self.assertEqual(command_result.process.returncode, 1) + self.assertIn("Cannot locate specified Dockerfile", command_result.stderr.decode()) + + def test_with_invalid_dockerfile_definition(self): + overrides = { + "Runtime": "3.10", + "Handler": "handler", + "DockerFile": "InvalidDockerfile", + "Tag": uuid4().hex, + } + cmdlist = self.get_command_list(parameter_overrides=overrides) + command_result = run_command(cmdlist, cwd=self.working_dir) + + # confirm build failed + self.assertEqual(command_result.process.returncode, 1) + self.assertIn("COPY requires at least two arguments", command_result.stderr.decode()) + + @skipIf( # Hits public ECR pull limitation, move it to canary tests (not RUN_BY_CANARY and not CI_OVERRIDE), diff --git a/tests/integration/buildcmd/test_build_samconfig.py b/tests/integration/buildcmd/test_build_samconfig.py new file mode 100644 index 0000000000..3df5052599 --- /dev/null +++ b/tests/integration/buildcmd/test_build_samconfig.py @@ -0,0 +1,114 @@ +import os +from pathlib import Path +from parameterized import parameterized, parameterized_class + +from tests.integration.buildcmd.build_integ_base import BuildIntegBase +from tests.testing_utils import run_command + + +configs = { + ".toml": "samconfig/samconfig.toml", + ".yaml": "samconfig/samconfig.yaml", + ".yml": "samconfig/samconfig.yml", + ".json": "samconfig/samconfig.json", +} + + +class TestSamConfigWithBuild(BuildIntegBase): + @parameterized.expand( + [ + (".toml"), + (".yaml"), + # (".json"), + ] + ) + def test_samconfig_works_with_extension(self, extension): + cmdlist = self.get_command_list(config_file=configs[extension]) + + command_result = run_command(cmdlist, cwd=self.working_dir) + stdout = str(command_result[1]) + stderr = str(command_result[2]) + + self.assertEqual(command_result.process.returncode, 0, "Build should succeed") + self.assertIn( + f"Built Artifacts : {extension}", + stdout, + f"Build template should use build_dir from samconfig{extension}", + ) + self.assertIn("Starting Build use cache", stderr, f"'cache'=true should be set in samconfig{extension}") + + @parameterized.expand( + [ + (".toml"), + (".yaml"), + # (".json"), + ] + ) + def test_samconfig_parameters_are_overridden(self, extension): + overrides = {"Runtime": "python3.8"} + overridden_build_dir = f"override_{extension}" + + cmdlist = self.get_command_list( + config_file=configs[extension], parameter_overrides=overrides, build_dir=overridden_build_dir + ) + + command_result = run_command(cmdlist, cwd=self.working_dir) + stdout = str(command_result[1]) + stderr = str(command_result[2]) + + self.assertEqual(command_result.process.returncode, 0, "Build should succeed") + self.assertNotIn( + f"Built Artifacts : {extension}", + stdout, + f"Build template should not use build_dir from samconfig{extension}", + ) + self.assertIn( + f"Built Artifacts : {overridden_build_dir}", stdout, f"Build template should use overridden build_dir" + ) + self.assertIn("Starting Build use cache", stderr, f"'cache'=true should be set in samconfig{extension}") + self.assertNotIn("python3.9", stderr, f"parameter_overrides runtime should not read from samconfig{extension}") + self.assertIn(overrides["Runtime"], stderr, "parameter_overrides should use overridden runtime") + self.assertNotIn("SomeURI", stderr, f"parameter_overrides should not read ANY values from samconfig{extension}") + + +@parameterized_class( + [ # Ordered by expected priority + {"extensions": [".toml", ".yaml", ".yml"]}, + {"extensions": [".yaml", ".yml"]}, + ] +) +class TestSamConfigExtensionHierarchy(BuildIntegBase): + def setUp(self): + super().setUp() + new_template_location = Path(self.working_dir, "template.yaml") + new_template_location.write_text(Path(self.template_path).read_text()) + for extension in self.extensions: + config_contents = Path(self.test_data_path, configs[extension]).read_text() + new_path = Path(self.working_dir, f"samconfig{extension}") + new_path.write_text(config_contents) + self.assertTrue(new_path.exists(), f"File samconfig{extension} should have been created in cwd") + + def tearDown(self): + for extension in self.extensions: + config_path = Path(self.working_dir, f"samconfig{extension}") + os.remove(config_path) + super().tearDown() + + def test_samconfig_pulls_correct_file_if_multiple(self): + self.template_path = str(Path(self.working_dir, "template.yaml")) + cmdlist = self.get_command_list(debug=True) + command_result = run_command(cmdlist, cwd=self.working_dir) + stdout = str(command_result[1]) + + self.assertEqual(command_result.process.returncode, 0, "Build should succeed") + self.assertIn( + f" {self.extensions[0]}", + stdout, + f"samconfig{self.extensions[0]} should take priority in current test group", + ) + for other_extension in self.extensions[1:]: + self.assertNotIn( + f" {other_extension}", + stdout, + f"samconfig{other_extension} should not be read over another, higher priority extension", + ) diff --git a/tests/integration/deploy/deploy_integ_base.py b/tests/integration/deploy/deploy_integ_base.py index f48eae9d61..4db4e5ff6a 100644 --- a/tests/integration/deploy/deploy_integ_base.py +++ b/tests/integration/deploy/deploy_integ_base.py @@ -2,11 +2,13 @@ import tempfile from pathlib import Path from enum import Enum, auto +from typing import List, Optional import boto3 from botocore.config import Config from samcli.lib.bootstrap.bootstrap import SAM_CLI_STACK_NAME +from samcli.lib.config.samconfig import SamConfig from tests.integration.package.package_integ_base import PackageIntegBase from tests.testing_utils import get_sam_command, run_command, run_command_with_input @@ -212,3 +214,26 @@ def get_minimal_build_command_list(template_file=None, build_dir=None): command_list = command_list + ["--build-dir", str(build_dir)] return command_list + + def _assert_deploy_samconfig_parameters( + self, + config: SamConfig, + stack_name: str = SAM_CLI_STACK_NAME, + resolve_s3: bool = True, + region: str = "us-east-1", + capabilities: str = "CAPABILITY_IAM", + confirm_changeset: Optional[bool] = None, + parameter_overrides: Optional[str] = None, + ): + params = config.document["default"]["deploy"]["parameters"] + + self.assertEqual(params["stack_name"], stack_name) + self.assertEqual(params["resolve_s3"], resolve_s3) + self.assertEqual(params["region"], region) + self.assertEqual(params["capabilities"], capabilities) + + if confirm_changeset is not None: + self.assertEqual(params["confirm_changeset"], confirm_changeset) + + if parameter_overrides is not None: + self.assertEqual(params["parameter_overrides"], parameter_overrides) diff --git a/tests/integration/deploy/test_deploy_command.py b/tests/integration/deploy/test_deploy_command.py index 131d3969ab..a7ece04a5c 100644 --- a/tests/integration/deploy/test_deploy_command.py +++ b/tests/integration/deploy/test_deploy_command.py @@ -10,7 +10,7 @@ from parameterized import parameterized from samcli.lib.bootstrap.bootstrap import SAM_CLI_STACK_NAME -from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME +from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, SamConfig from tests.integration.deploy.deploy_integ_base import DeployIntegBase from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, UpdatableSARTemplate @@ -613,6 +613,13 @@ def test_deploy_guided_zip(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + deploy_config_params = config.document["default"]["deploy"]["parameters"] + self.assertEqual(deploy_config_params["stack_name"], stack_name) + self.assertTrue(deploy_config_params["resolve_s3"]) + self.assertEqual(deploy_config_params["region"], "us-east-1") + self.assertEqual(deploy_config_params["capabilities"], "CAPABILITY_IAM") # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -627,7 +634,7 @@ def test_deploy_guided_image_auto(self, template_file): deploy_command_list = self.get_deploy_command_list(template_file=template_path, guided=True) deploy_process_execute = self.run_command_with_input( - deploy_command_list, f"{stack_name}\n\n\n\n\ny\n\n\ny\n\n\n\n".encode() + deploy_command_list, f"{stack_name}\n\n\n\n\ny\n\n\n\n\n\n\n".encode() ) # Deploy should succeed with a managed stack @@ -638,6 +645,10 @@ def test_deploy_guided_image_auto(self, template_file): self._assert_companion_stack(self.cfn_client, companion_stack_name) self._assert_companion_stack_content(self.ecr_client, companion_stack_name) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters(config, stack_name=stack_name) + # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -669,6 +680,9 @@ def test_deploy_guided_image_specify(self, template_file, does_ask_for_authoriza self.fail("Companion stack was created. This should not happen with specifying image repos.") self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters(config, stack_name=stack_name) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -690,6 +704,11 @@ def test_deploy_guided_set_parameter(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, stack_name=stack_name, parameter_overrides='Parameter="SuppliedParameter"' + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -710,6 +729,14 @@ def test_deploy_guided_set_capabilities(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, + stack_name=stack_name, + capabilities="CAPABILITY_IAM CAPABILITY_NAMED_IAM", + parameter_overrides='Parameter="SuppliedParameter"', + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -731,6 +758,11 @@ def test_deploy_guided_capabilities_default(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, stack_name=stack_name, parameter_overrides='Parameter="SuppliedParameter"' + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -752,6 +784,11 @@ def test_deploy_guided_set_confirm_changeset(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, stack_name=stack_name, confirm_changeset=True, parameter_overrides='Parameter="SuppliedParameter"' + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -789,7 +826,7 @@ def test_deploy_with_invalid_config(self, template_file, config_file): deploy_process_execute = self.run_command(deploy_command_list) self.assertEqual(deploy_process_execute.process.returncode, 1) - self.assertIn("Error reading configuration: Unexpected character", str(deploy_process_execute.stderr)) + self.assertIn("SamConfigFileReadException: Unexpected character", str(deploy_process_execute.stderr)) @parameterized.expand([("aws-serverless-function.yaml", "samconfig-tags-list.toml")]) def test_deploy_with_valid_config_tags_list(self, template_file, config_file): diff --git a/tests/integration/telemetry/test_experimental_metric.py b/tests/integration/telemetry/test_experimental_metric.py index 977e65053f..702ceb4a5f 100644 --- a/tests/integration/telemetry/test_experimental_metric.py +++ b/tests/integration/telemetry/test_experimental_metric.py @@ -211,8 +211,11 @@ def test_must_send_not_experimental_metrics_if_not_experimental(self): self.assertEqual(process.returncode, 2, "Command should fail") all_requests = server.get_all_requests() - self.assertEqual(1, len(all_requests), "Command run metric must be sent") - request = all_requests[0] + self.assertEqual(2, len(all_requests), "Command run and event metrics must be sent") + # NOTE: Since requests happen asynchronously, we cannot guarantee whether the + # commandRun metric will be first or second, so we sort for consistency. + all_requests.sort(key=lambda x: list(x["data"]["metrics"][0].keys())[0]) + request = all_requests[0] # "commandRun" comes before "events" self.assertIn("Content-Type", request["headers"]) self.assertEqual(request["headers"]["Content-Type"], "application/json") diff --git a/tests/integration/telemetry/test_installed_metric.py b/tests/integration/telemetry/test_installed_metric.py index e17459828c..725a2d0ff5 100644 --- a/tests/integration/telemetry/test_installed_metric.py +++ b/tests/integration/telemetry/test_installed_metric.py @@ -24,7 +24,9 @@ def test_send_installed_metric_on_first_run(self): self.assertIn(EXPECTED_TELEMETRY_PROMPT, stderrdata.decode()) all_requests = server.get_all_requests() - self.assertEqual(2, len(all_requests), "There should be exactly two metrics request") + self.assertEqual( + 3, len(all_requests), "There should be exactly three metrics request" + ) # 3 = 2 expected + events # First one is usually the installed metric requests = filter_installed_metric_requests(all_requests) diff --git a/tests/integration/telemetry/test_telemetry_contract.py b/tests/integration/telemetry/test_telemetry_contract.py index 08b3585b99..a3e383bb5d 100644 --- a/tests/integration/telemetry/test_telemetry_contract.py +++ b/tests/integration/telemetry/test_telemetry_contract.py @@ -28,7 +28,9 @@ def test_must_not_send_metrics_if_disabled_using_envvar(self): self.assertEqual(process.returncode, 0, "Command should successfully complete") all_requests = server.get_all_requests() - self.assertEqual(1, len(all_requests), "Command run metric should be sent") + self.assertEqual( + 2, len(all_requests), "Command run and event metrics should be sent" + ) # 2 = cmd_run + events def test_must_send_metrics_if_enabled_via_envvar(self): """ @@ -52,7 +54,7 @@ def test_must_send_metrics_if_enabled_via_envvar(self): self.assertEqual(process.returncode, 0, "Command should successfully complete") all_requests = server.get_all_requests() - self.assertEqual(1, len(all_requests), "Command run metric must be sent") + self.assertEqual(2, len(all_requests), "Command run and event metrics must be sent") # cmd_run + events def test_must_not_crash_when_offline(self): """ diff --git a/tests/integration/testdata/buildcmd/PythonImage/InvalidDockerfile b/tests/integration/testdata/buildcmd/PythonImage/InvalidDockerfile new file mode 100644 index 0000000000..04599ba872 --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonImage/InvalidDockerfile @@ -0,0 +1,16 @@ +ARG BASE_RUNTIME + +FROM public.ecr.aws/lambda/python:$BASE_RUNTIME + +ARG FUNCTION_DIR="/var/task" + +RUN mkdir -p $FUNCTION_DIR + +# invalid line below +COPY main.py + +COPY __init__.py $FUNCTION_DIR +COPY requirements.txt $FUNCTION_DIR + +RUN python -m pip install -r $FUNCTION_DIR/requirements.txt -t $FUNCTION_DIR + diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.json b/tests/integration/testdata/buildcmd/samconfig/samconfig.json new file mode 100644 index 0000000000..6fa18e1c11 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.json @@ -0,0 +1,12 @@ +{ + "version": 0.1, + "default": { + "build": { + "parameters": { + "build_dir": ".json", + "cached": true, + "parameter_overrides": "Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler" + } + } + } +} \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.toml b/tests/integration/testdata/buildcmd/samconfig/samconfig.toml new file mode 100644 index 0000000000..23a769ff5e --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.toml @@ -0,0 +1,5 @@ +version = 0.1 +[default.build.parameters] +build_dir = ".toml" +cached = true +parameter_overrides = "Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler" \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.yaml b/tests/integration/testdata/buildcmd/samconfig/samconfig.yaml new file mode 100644 index 0000000000..63af206238 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.yaml @@ -0,0 +1,7 @@ +version: 0.1 +default: + build: + parameters: + build_dir: .yaml + cached: true + parameter_overrides: Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.yml b/tests/integration/testdata/buildcmd/samconfig/samconfig.yml new file mode 100644 index 0000000000..4af8baa434 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.yml @@ -0,0 +1,7 @@ +version: 0.1 +default: + build: + parameters: + build_dir: .yml + cached: true + parameter_overrides: Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/template.yaml b/tests/integration/testdata/buildcmd/samconfig/template.yaml new file mode 100644 index 0000000000..6944799912 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/template.yaml @@ -0,0 +1,37 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameters: + Runtime: + Type: String + CodeUri: + Type: String + Handler: + Type: String + +Resources: + + Function: + Type: AWS::Serverless::Function + Properties: + Handler: !Ref Handler + Runtime: !Ref Runtime + CodeUri: !Ref CodeUri + Timeout: 600 + + + OtherRelativePathResource: + Type: AWS::ApiGateway::RestApi + Properties: + BodyS3Location: SomeRelativePath + + GlueResource: + Type: AWS::Glue::Job + Properties: + Command: + ScriptLocation: SomeRelativePath + + ExampleNestedStack: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: https://s3.amazonaws.com/examplebucket/exampletemplate.yml diff --git a/tests/unit/cli/test_cli_config_file.py b/tests/unit/cli/test_cli_config_file.py index 606e0a004e..19bfcfa011 100644 --- a/tests/unit/cli/test_cli_config_file.py +++ b/tests/unit/cli/test_cli_config_file.py @@ -5,8 +5,11 @@ from unittest import TestCase, skipIf from unittest.mock import MagicMock, patch +import tomlkit + from samcli.commands.exceptions import ConfigException -from samcli.cli.cli_config_file import TomlProvider, configuration_option, configuration_callback, get_ctx_defaults +from samcli.cli.cli_config_file import ConfigProvider, configuration_option, configuration_callback, get_ctx_defaults +from samcli.lib.config.exceptions import SamConfigFileReadException, SamConfigVersionException from samcli.lib.config.samconfig import DEFAULT_ENV from tests.testing_utils import IS_WINDOWS @@ -21,9 +24,9 @@ def __init__(self, info_name, parent, params=None, command=None, default_map=Non self.default_map = default_map -class TestTomlProvider(TestCase): +class TestConfigProvider(TestCase): def setUp(self): - self.toml_provider = TomlProvider() + self.config_provider = ConfigProvider() self.config_env = "config_env" self.parameters = "parameters" self.cmd_name = "topic" @@ -33,29 +36,30 @@ def test_toml_valid_with_section(self): config_path = Path(config_dir, "samconfig.toml") config_path.write_text("version=0.1\n[config_env.topic.parameters]\nword='clarity'\n") self.assertEqual( - TomlProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]), {"word": "clarity"} + ConfigProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]), {"word": "clarity"} ) def test_toml_valid_with_no_version(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("[config_env.topic.parameters]\nword='clarity'\n") - with self.assertRaises(ConfigException): - TomlProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigVersionException): + ConfigProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) def test_toml_valid_with_invalid_version(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("version='abc'\n[config_env.topic.parameters]\nword='clarity'\n") - with self.assertRaises(ConfigException): - TomlProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigVersionException): + ConfigProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) def test_toml_invalid_empty_dict(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("[topic]\nword=clarity\n") - self.assertEqual(self.toml_provider(config_dir, self.config_env, [self.cmd_name]), {}) + with self.assertRaises(SamConfigFileReadException): + self.config_provider(config_path, self.config_env, [self.cmd_name]) def test_toml_invalid_file_name(self): config_dir = tempfile.gettempdir() @@ -63,16 +67,16 @@ def test_toml_invalid_file_name(self): config_path.write_text("version=0.1\n[config_env.topic.parameters]\nword='clarity'\n") config_path_invalid = Path(config_dir, "samconfig.toml") - with self.assertRaises(ConfigException): - self.toml_provider(config_path_invalid, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigFileReadException): + self.config_provider(config_path_invalid, self.config_env, [self.cmd_name]) def test_toml_invalid_syntax(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("version=0.1\n[config_env.topic.parameters]\nword=_clarity'\n") - with self.assertRaises(ConfigException): - self.toml_provider(config_path, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigFileReadException): + self.config_provider(config_path, self.config_env, [self.cmd_name]) class TestCliConfiguration(TestCase): @@ -121,6 +125,7 @@ def test_callback_with_invalid_config_file(self): self.ctx.parent = mock_context3 self.ctx.info_name = "test_info" self.ctx.params = {"config_file": "invalid_config_file"} + self.ctx._parameter_source.__get__ = "COMMANDLINE" setattr(self.ctx, "samconfig_dir", None) with self.assertRaises(ConfigException): configuration_callback( @@ -197,8 +202,8 @@ def test_callback_with_config_file_from_pipe(self): self.assertNotIn(self.value, self.saved_callback.call_args[0]) def test_configuration_option(self): - toml_provider = TomlProvider() - click_option = configuration_option(provider=toml_provider) + config_provider = ConfigProvider() + click_option = configuration_option(provider=config_provider) clc = click_option(self.Dummy()) self.assertEqual(clc.__click_params__[0].is_eager, True) self.assertEqual( @@ -207,7 +212,7 @@ def test_configuration_option(self): ) self.assertEqual(clc.__click_params__[0].hidden, True) self.assertEqual(clc.__click_params__[0].expose_value, False) - self.assertEqual(clc.__click_params__[0].callback.args, (None, None, None, toml_provider)) + self.assertEqual(clc.__click_params__[0].callback.args, (None, None, None, config_provider)) def test_get_ctx_defaults_non_nested(self): provider = MagicMock() diff --git a/tests/unit/commands/delete/test_delete_context.py b/tests/unit/commands/delete/test_delete_context.py index daa72c187b..92e2aa2c6a 100644 --- a/tests/unit/commands/delete/test_delete_context.py +++ b/tests/unit/commands/delete/test_delete_context.py @@ -7,7 +7,7 @@ from samcli.commands.delete.delete_context import DeleteContext from samcli.lib.package.artifact_exporter import Template -from samcli.cli.cli_config_file import TomlProvider +from samcli.cli.cli_config_file import ConfigProvider from samcli.lib.delete.cfn_utils import CfnUtils from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.package.ecr_uploader import ECRUploader @@ -58,7 +58,7 @@ def test_delete_context_enter(self, get_boto_client_provider_mock): self.assertEqual(delete_context.init_clients.call_count, 1) @patch.object( - TomlProvider, + ConfigProvider, "__call__", MagicMock( return_value=( @@ -123,7 +123,7 @@ def test_delete_no_user_input( self.assertEqual(expected_prompt_calls, patched_prompt.call_args_list) @patch.object( - TomlProvider, + ConfigProvider, "__call__", MagicMock( return_value=( @@ -506,7 +506,7 @@ def test_s3_option_flag(self): self.assertEqual(delete_context.s3_prefix, "s3_prefix") @patch.object( - TomlProvider, + ConfigProvider, "__call__", MagicMock( return_value=( diff --git a/tests/unit/commands/local/lib/swagger/test_parser.py b/tests/unit/commands/local/lib/swagger/test_parser.py index 84ce8899de..d854ca595e 100644 --- a/tests/unit/commands/local/lib/swagger/test_parser.py +++ b/tests/unit/commands/local/lib/swagger/test_parser.py @@ -1022,3 +1022,21 @@ def test_invalid_identity_source_throws_exception(self): with self.assertRaises(InvalidSecurityDefinition): parser._get_lambda_identity_sources(Mock(), "request", Route.API, properties, auth_properties) + + +class TestGetDocumentVersion(TestCase): + @parameterized.expand( + [ + ({"swagger": "2.0"}, "2.0"), + ({"swagger": 2.0}, "2.0"), + ({"openapi": "3.0"}, "3.0"), + ({"openapi": 3.0}, "3.0"), + ({"not valid": 3.0}, ""), + ({}, ""), + ] + ) + def test_get_document_version(self, swagger_doc, expected_output): + parser = SwaggerParser(Mock(), swagger_doc) + output = parser._get_document_version() + + self.assertEqual(output, expected_output) diff --git a/tests/unit/commands/remote/test_remote_invoke_context.py b/tests/unit/commands/remote/test_remote_invoke_context.py index e01d9de5fb..efae5b68b8 100644 --- a/tests/unit/commands/remote/test_remote_invoke_context.py +++ b/tests/unit/commands/remote/test_remote_invoke_context.py @@ -7,7 +7,7 @@ AmbiguousResourceForRemoteInvoke, NoResourceFoundForRemoteInvoke, UnsupportedServiceForRemoteInvoke, - NoExecutorFoundForRemoteInvoke, + ResourceNotSupportedForRemoteInvoke, InvalidStackNameProvidedForRemoteInvoke, ) from samcli.commands.remote.remote_invoke_context import RemoteInvokeContext, SUPPORTED_SERVICES @@ -118,7 +118,7 @@ def test_running_without_resource_summary_should_raise_exception(self, patched_g def test_running_with_unsupported_resource_should_raise_exception(self, patched_get_resource_summary): patched_get_resource_summary.return_value = Mock(resource_type="UnSupportedResource") with self._get_remote_invoke_context() as remote_invoke_context: - with self.assertRaises(NoExecutorFoundForRemoteInvoke): + with self.assertRaises(ResourceNotSupportedForRemoteInvoke): remote_invoke_context.run(Mock()) @patch("samcli.commands.remote.remote_invoke_context.RemoteInvokeExecutorFactory") @@ -126,6 +126,7 @@ def test_running_with_unsupported_resource_should_raise_exception(self, patched_ def test_running_should_execute_remote_invoke_executor_instance( self, patched_get_resource_summary, patched_remote_invoke_executor_factory ): + patched_get_resource_summary.return_value = Mock(resource_type=SUPPORTED_SERVICES["lambda"]) mocked_remote_invoke_executor_factory = Mock() patched_remote_invoke_executor_factory.return_value = mocked_remote_invoke_executor_factory mocked_remote_invoke_executor = Mock() diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 675f22a4bf..b2e0822c78 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -8,7 +8,6 @@ import tempfile from pathlib import Path from contextlib import contextmanager -from samcli.lib.config.samconfig import SamConfig, DEFAULT_ENV from click.testing import CliRunner @@ -16,6 +15,7 @@ from unittest.mock import patch, ANY import logging +from samcli.lib.config.samconfig import SamConfig, DEFAULT_ENV from samcli.lib.utils.packagetype import ZIP, IMAGE LOG = logging.getLogger() diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index 408ba8bb35..c2712a5247 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -1510,7 +1510,7 @@ def test_docker_build_raises_DockerBuildFailed_when_error_in_buildlog_stream(sel "DockerBuildArgs": {"a": "b"}, } - self.docker_client_mock.api.build.return_value = [{"error": "Function building failed"}] + self.docker_client_mock.images.build.return_value = (Mock(), [{"error": "Function building failed"}]) self.builder._build_lambda_image("Name", metadata, X86_64) @@ -1530,7 +1530,7 @@ def test_dockerfile_not_in_dockercontext(self): "Bad Request", response=response_mock, explanation="Cannot locate specified Dockerfile" ) self.builder._stream_lambda_image_build_logs = error_mock - self.docker_client_mock.api.build.return_value = [] + self.docker_client_mock.images.build.return_value = (Mock(), []) self.builder._build_lambda_image("Name", metadata, X86_64) @@ -1545,7 +1545,7 @@ def test_error_rerasises(self): error_mock = Mock() error_mock.side_effect = docker.errors.APIError("Bad Request", explanation="Some explanation") self.builder._stream_lambda_image_build_logs = error_mock - self.docker_client_mock.api.build.return_value = [] + self.docker_client_mock.images.build.return_value = (Mock(), []) self.builder._build_lambda_image("Name", metadata, X86_64) @@ -1557,7 +1557,7 @@ def test_can_build_image_function(self): "DockerBuildArgs": {"a": "b"}, } - self.docker_client_mock.api.build.return_value = [] + self.docker_client_mock.images.build.return_value = (Mock(), []) result = self.builder._build_lambda_image("Name", metadata, X86_64) @@ -1598,7 +1598,7 @@ def test_build_image_function_with_empty_metadata_raises_Docker_Build_Failed_Exc def test_can_build_image_function_without_tag(self): metadata = {"Dockerfile": "Dockerfile", "DockerContext": "context", "DockerBuildArgs": {"a": "b"}} - self.docker_client_mock.api.build.return_value = [] + self.docker_client_mock.images.build.return_value = (Mock(), []) result = self.builder._build_lambda_image("Name", metadata, X86_64) self.assertEqual(result, "name:latest") @@ -1613,19 +1613,18 @@ def test_can_build_image_function_under_debug(self, mock_os): "DockerBuildArgs": {"a": "b"}, } - self.docker_client_mock.api.build.return_value = [] + self.docker_client_mock.images.build.return_value = (Mock, []) result = self.builder._build_lambda_image("Name", metadata, X86_64) self.assertEqual(result, "name:Tag-debug") self.assertEqual( - self.docker_client_mock.api.build.call_args, + self.docker_client_mock.images.build.call_args, # NOTE (sriram-mv): path set to ANY to handle platform differences. call( path=ANY, dockerfile="Dockerfile", tag="name:Tag-debug", buildargs={"a": "b", "SAM_BUILD_MODE": "debug"}, - decode=True, platform="linux/amd64", rm=True, ), @@ -1642,24 +1641,31 @@ def test_can_build_image_function_under_debug_with_target(self, mock_os): "DockerBuildTarget": "stage", } - self.docker_client_mock.api.build.return_value = [] + self.docker_client_mock.images.build.return_value = (Mock(), []) result = self.builder._build_lambda_image("Name", metadata, X86_64) self.assertEqual(result, "name:Tag-debug") self.assertEqual( - self.docker_client_mock.api.build.call_args, + self.docker_client_mock.images.build.call_args, call( path=ANY, dockerfile="Dockerfile", tag="name:Tag-debug", buildargs={"a": "b", "SAM_BUILD_MODE": "debug"}, - decode=True, target="stage", platform="linux/amd64", rm=True, ), ) + def test_can_raise_build_error(self): + self.docker_client_mock.images.build.side_effect = docker.errors.BuildError( + reason="Missing Dockerfile", build_log="Build failed" + ) + + with self.assertRaises(DockerBuildFailed): + self.builder._build_lambda_image("Name", {}, X86_64) + class TestApplicationBuilder_build_function(TestCase): def setUp(self): diff --git a/tests/unit/lib/remote_invoke/test_remote_invoke_executor_factory.py b/tests/unit/lib/remote_invoke/test_remote_invoke_executor_factory.py index 57b5e7988c..8f76be8303 100644 --- a/tests/unit/lib/remote_invoke/test_remote_invoke_executor_factory.py +++ b/tests/unit/lib/remote_invoke/test_remote_invoke_executor_factory.py @@ -4,9 +4,7 @@ from parameterized import parameterized -from samcli.lib.remote_invoke.remote_invoke_executor_factory import ( - RemoteInvokeExecutorFactory, -) +from samcli.lib.remote_invoke.remote_invoke_executor_factory import RemoteInvokeExecutorFactory, AWS_LAMBDA_FUNCTION from samcli.lib.remote_invoke.remote_invoke_executors import RemoteInvokeOutputFormat @@ -15,6 +13,12 @@ def setUp(self) -> None: self.boto_client_provider_mock = Mock() self.remote_invoke_executor_factory = RemoteInvokeExecutorFactory(self.boto_client_provider_mock) + def test_supported_resource_executors(self): + supported_executors = self.remote_invoke_executor_factory.REMOTE_INVOKE_EXECUTOR_MAPPING + self.assertEqual(1, len(supported_executors)) + expected_executors = {AWS_LAMBDA_FUNCTION} + self.assertEqual(expected_executors, set(supported_executors.keys())) + @patch( "samcli.lib.remote_invoke.remote_invoke_executor_factory.RemoteInvokeExecutorFactory.REMOTE_INVOKE_EXECUTOR_MAPPING" ) @@ -132,3 +136,56 @@ def test_create_lambda_test_executor( response_consumer=given_response_consumer, log_consumer=given_log_consumer, ) + + @parameterized.expand(itertools.product([RemoteInvokeOutputFormat.JSON, RemoteInvokeOutputFormat.TEXT])) + @patch("samcli.lib.remote_invoke.remote_invoke_executor_factory.StepFunctionsStartExecutionExecutor") + @patch("samcli.lib.remote_invoke.remote_invoke_executor_factory.SfnDescribeExecutionResponseConverter") + @patch("samcli.lib.remote_invoke.remote_invoke_executor_factory.DefaultConvertToJSON") + @patch("samcli.lib.remote_invoke.remote_invoke_executor_factory.ResponseObjectToJsonStringMapper") + @patch("samcli.lib.remote_invoke.remote_invoke_executor_factory.RemoteInvokeExecutor") + def test_create_stepfunctions_test_executor( + self, + remote_invoke_output_format, + patched_remote_invoke_executor, + patched_object_to_json_converter, + patched_convert_to_default_json, + patched_response_converter, + patched_stepfunctions_invoke_executor, + ): + given_physical_resource_id = "physical_resource_id" + given_cfn_resource_summary = Mock(physical_resource_id=given_physical_resource_id) + + given_stepfunctions_client = Mock() + self.boto_client_provider_mock.return_value = given_stepfunctions_client + + given_remote_invoke_executor = Mock() + patched_remote_invoke_executor.return_value = given_remote_invoke_executor + + given_response_consumer = Mock() + given_log_consumer = Mock() + stepfunctions_executor = self.remote_invoke_executor_factory._create_stepfunctions_boto_executor( + given_cfn_resource_summary, remote_invoke_output_format, given_response_consumer, given_log_consumer + ) + + self.assertEqual(stepfunctions_executor, given_remote_invoke_executor) + self.boto_client_provider_mock.assert_called_with("stepfunctions") + patched_convert_to_default_json.assert_called_once() + + expected_mappers = [] + if remote_invoke_output_format == RemoteInvokeOutputFormat.JSON: + patched_object_to_json_converter.assert_called_once() + patched_response_converter.assert_called_once() + patched_stepfunctions_invoke_executor.assert_called_with( + given_stepfunctions_client, given_physical_resource_id, remote_invoke_output_format + ) + expected_mappers = [ + patched_response_converter(), + patched_object_to_json_converter(), + ] + patched_remote_invoke_executor.assert_called_with( + request_mappers=[patched_convert_to_default_json()], + response_mappers=expected_mappers, + boto_action_executor=patched_stepfunctions_invoke_executor(), + response_consumer=given_response_consumer, + log_consumer=given_log_consumer, + ) diff --git a/tests/unit/lib/remote_invoke/test_stepfunctions_invoke_executors.py b/tests/unit/lib/remote_invoke/test_stepfunctions_invoke_executors.py new file mode 100644 index 0000000000..adac0f00e8 --- /dev/null +++ b/tests/unit/lib/remote_invoke/test_stepfunctions_invoke_executors.py @@ -0,0 +1,165 @@ +from unittest import TestCase +from unittest.mock import patch, Mock + +from parameterized import parameterized, parameterized_class +from samcli.lib.remote_invoke.stepfunctions_invoke_executors import ( + SfnDescribeExecutionResponseConverter, + RemoteInvokeOutputFormat, + InvalideBotoResponseException, + StepFunctionsStartExecutionExecutor, + ParamValidationError, + InvalidResourceBotoParameterException, + ErrorBotoApiCallException, + ClientError, + RemoteInvokeLogOutput, +) +from samcli.lib.remote_invoke.remote_invoke_executors import RemoteInvokeExecutionInfo, RemoteInvokeResponse +from datetime import datetime + + +@parameterized_class( + "output", + [[RemoteInvokeOutputFormat.TEXT], [RemoteInvokeOutputFormat.JSON]], +) +class TestStepFunctionsStartExecutionExecutor(TestCase): + output: RemoteInvokeOutputFormat + + def setUp(self) -> None: + self.stepfunctions_client = Mock() + self.state_machine_arn = Mock() + self.stepfunctions_invoke_executor = StepFunctionsStartExecutionExecutor( + self.stepfunctions_client, self.state_machine_arn, self.output + ) + + @patch("samcli.lib.remote_invoke.stepfunctions_invoke_executors.time") + def test_execute_action_successful(self, patched_time): + patched_time.sleep = Mock() + mock_exec_name = "mock_execution_name" + mock_exec_arn = "MockArn" + given_input = '{"input_key": "value"}' + mock_response = { + "executionArn": mock_exec_arn, + "status": "SUCCEEDED", + "output": '{"output_key": "mock_output"}', + } + self.stepfunctions_client.start_execution.return_value = {"executionArn": mock_exec_arn} + self.stepfunctions_client.describe_execution.side_effect = [ + {"executionArn": mock_exec_arn, "status": "RUNNING"}, + mock_response, + ] + self.stepfunctions_invoke_executor.validate_action_parameters({"name": mock_exec_name}) + result = self.stepfunctions_invoke_executor._execute_action(given_input) + + if self.output == RemoteInvokeOutputFormat.JSON: + self.assertEqual(list(result), [RemoteInvokeResponse(mock_response)]) + else: + self.assertEqual(list(result), [RemoteInvokeResponse(mock_response["output"])]) + + self.stepfunctions_client.start_execution.assert_called_with( + stateMachineArn=self.state_machine_arn, input=given_input, name=mock_exec_name + ) + self.stepfunctions_client.describe_execution.assert_called() + + @patch("samcli.lib.remote_invoke.stepfunctions_invoke_executors.time") + def test_execute_action_not_successful(self, patched_time): + patched_time.sleep = Mock() + mock_exec_name = "mock_execution_name" + mock_exec_arn = "MockArn" + mock_error = "MockError" + mock_cause = "Execution failed due to mock error" + given_input = '{"input_key": "value"}' + mock_response = {"executionArn": mock_exec_arn, "status": "FAILED", "error": mock_error, "cause": mock_cause} + self.stepfunctions_client.start_execution.return_value = {"executionArn": mock_exec_arn} + self.stepfunctions_client.describe_execution.side_effect = [ + {"executionArn": mock_exec_arn, "status": "RUNNING"}, + mock_response, + ] + self.stepfunctions_invoke_executor.validate_action_parameters({"name": mock_exec_name}) + result = self.stepfunctions_invoke_executor._execute_action(given_input) + + expected_response = f"The execution failed due to the error: {mock_error} and cause: {mock_cause}" + if self.output == RemoteInvokeOutputFormat.JSON: + self.assertEqual(list(result), [RemoteInvokeResponse(mock_response)]) + else: + self.assertEqual(list(result), [RemoteInvokeLogOutput(expected_response)]) + + @parameterized.expand( + [ + ({}, {"name": "sam_remote_invoke_20230710T072625"}), + ({"name": "custom_execution_name"}, {"name": "custom_execution_name"}), + ( + {"traceHeader": "Mock X-Ray trace header"}, + {"traceHeader": "Mock X-Ray trace header", "name": "sam_remote_invoke_20230710T072625"}, + ), + ( + {"stateMachineArn": "ParameterProvidedArn", "input": "ParameterProvidedInput"}, + {"name": "sam_remote_invoke_20230710T072625"}, + ), + ( + {"invalidParameterKey": "invalidParameterValue"}, + {"invalidParameterKey": "invalidParameterValue", "name": "sam_remote_invoke_20230710T072625"}, + ), + ] + ) + @patch("samcli.lib.remote_invoke.stepfunctions_invoke_executors.datetime") + def test_validate_action_parameters(self, parameters, expected_boto_parameters, patched_datetime): + patched_datetime.now.return_value = datetime(2023, 7, 10, 7, 26, 25) + self.stepfunctions_invoke_executor.validate_action_parameters(parameters) + self.assertEqual(self.stepfunctions_invoke_executor.request_parameters, expected_boto_parameters) + + def test_execute_action_invalid_parameter_key_throws_parameter_validation_exception(self): + given_input = "input" + error = ParamValidationError(report="Invalid parameters") + self.stepfunctions_client.start_execution.side_effect = error + with self.assertRaises(InvalidResourceBotoParameterException): + self.stepfunctions_invoke_executor.validate_action_parameters({}) + for _ in self.stepfunctions_invoke_executor._execute_action(given_input): + pass + + def test_execute_action_throws_client_error_exception(self): + given_input = "input" + error = ClientError(error_response={"Error": {"Code": "MockException"}}, operation_name="invoke") + self.stepfunctions_client.start_execution.side_effect = error + with self.assertRaises(ErrorBotoApiCallException): + self.stepfunctions_invoke_executor.validate_action_parameters({}) + for _ in self.stepfunctions_invoke_executor._execute_action(given_input): + pass + + +class TestSfnDescribeExecutionResponseConverter(TestCase): + def setUp(self) -> None: + self.sfn_response_converter = SfnDescribeExecutionResponseConverter() + + def test_stepfunctions_response_conversion(self): + output_format = RemoteInvokeOutputFormat.JSON + given_output_string = "output string" + execution_date = datetime(2022, 12, 25, 00, 00, 00) + given_execution_result = { + "output": given_output_string, + "startDate": execution_date, + "stopDate": execution_date, + } + remote_invoke_execution_info = RemoteInvokeExecutionInfo(None, None, {}, output_format) + remote_invoke_execution_info.response = given_execution_result + + expected_result = { + "output": given_output_string, + "startDate": "2022-12-25 00:00:00.000000", + "stopDate": "2022-12-25 00:00:00.000000", + } + + result = self.sfn_response_converter.map(remote_invoke_execution_info) + + self.assertEqual(result.response, expected_result) + + def test_stepfunctions_invalid_response_exception(self): + output_format = RemoteInvokeOutputFormat.JSON + given_output_response = Mock() + given_output_string = "output string" + given_output_response.read().decode.return_value = given_output_string + given_test_result = [given_output_response] + remote_invoke_execution_info = RemoteInvokeExecutionInfo(None, None, {}, output_format) + remote_invoke_execution_info.response = given_test_result + + with self.assertRaises(InvalideBotoResponseException): + self.sfn_response_converter.map(remote_invoke_execution_info) diff --git a/tests/unit/lib/samconfig/test_file_manager.py b/tests/unit/lib/samconfig/test_file_manager.py new file mode 100644 index 0000000000..18df66474c --- /dev/null +++ b/tests/unit/lib/samconfig/test_file_manager.py @@ -0,0 +1,277 @@ +import json +from pathlib import Path +import tempfile +from unittest import TestCase, skip + +import tomlkit +from ruamel.yaml import YAML + +from samcli.lib.config.exceptions import FileParseException +from samcli.lib.config.file_manager import COMMENT_KEY, JsonFileManager, TomlFileManager, YamlFileManager + + +class TestTomlFileManager(TestCase): + def test_read_toml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text( + "version=0.1\n[config_env.topic1.parameters]\nword='clarity'\nmultiword=['thing 1', 'thing 2']" + ) + config_doc = TomlFileManager.read(config_path) + self.assertEqual( + config_doc, + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + ) + + def test_read_toml_invalid_toml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text("fake='not real'\nimproper toml file\n") + with self.assertRaises(FileParseException): + TomlFileManager.read(config_path) + + def test_read_toml_file_path_not_valid(self): + config_dir = "path/that/doesnt/exist" + config_path = Path(config_dir, "samconfig.toml") + config_doc = TomlFileManager.read(config_path) + self.assertEqual(config_doc, tomlkit.document()) + + def test_write_toml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + toml = { + "version": 0.1, + "config_env": {"topic2": {"parameters": {"word": "clarity"}}}, + COMMENT_KEY: "This is a comment", + } + + TomlFileManager.write(toml, config_path) + + txt = config_path.read_text() + self.assertIn("version = 0.1", txt) + self.assertIn("[config_env.topic2.parameters]", txt) + self.assertIn('word = "clarity"', txt) + self.assertIn("# This is a comment", txt) + self.assertNotIn(COMMENT_KEY, txt) + + def test_dont_write_toml_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text("nothing to see here\n") + toml = {} + + TomlFileManager.write(toml, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_toml_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.toml") + with self.assertRaises(FileNotFoundError): + TomlFileManager.write({"key": "some value"}, config_path) + + def test_write_toml_file(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + toml = tomlkit.parse('# This is a comment\nversion = 0.1\n[config_env.topic2.parameters]\nword = "clarity"\n') + + TomlFileManager.write(toml, config_path) + + txt = config_path.read_text() + self.assertIn("version = 0.1", txt) + self.assertIn("[config_env.topic2.parameters]", txt) + self.assertIn('word = "clarity"', txt) + self.assertIn("# This is a comment", txt) + + def test_dont_write_toml_file_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text("nothing to see here\n") + toml = tomlkit.document() + + TomlFileManager.write(toml, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_toml_file_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.toml") + with self.assertRaises(FileNotFoundError): + TomlFileManager.write(tomlkit.parse('key = "some value"'), config_path) + + def test_toml_put_comment(self): + toml_doc = tomlkit.loads('version = 0.1\n[config_env.topic2.parameters]\nword = "clarity"\n') + + toml_doc = TomlFileManager.put_comment(toml_doc, "This is a comment") + + txt = tomlkit.dumps(toml_doc) + self.assertIn("# This is a comment", txt) + + +class TestYamlFileManager(TestCase): + + yaml = YAML() + + def test_read_yaml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + config_path.write_text( + "version: 0.1\nconfig_env:\n topic1:\n parameters:\n word: clarity\n multiword: [thing 1, thing 2]" + ) + + config_doc = YamlFileManager.read(config_path) + + self.assertEqual( + config_doc, + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + ) + + def test_read_yaml_invalid_yaml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + config_path.write_text("fake: not real\nthisYaml isn't correct") + + with self.assertRaises(FileParseException): + YamlFileManager.read(config_path) + + def test_read_yaml_file_path_not_valid(self): + config_dir = "path/that/doesnt/exist" + config_path = Path(config_dir, "samconfig.yaml") + + config_doc = YamlFileManager.read(config_path) + + self.assertEqual(config_doc, {}) + + def test_write_yaml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + yaml = { + "version": 0.1, + "config_env": {"topic2": {"parameters": {"word": "clarity"}}}, + COMMENT_KEY: "This is a comment", + } + + YamlFileManager.write(yaml, config_path) + + txt = config_path.read_text() + self.assertIn("version: 0.1", txt) + self.assertIn("config_env:\n topic2:\n parameters:\n", txt) + self.assertIn("word: clarity", txt) + self.assertIn("# This is a comment", txt) + self.assertNotIn(COMMENT_KEY, txt) + + def test_dont_write_yaml_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + config_path.write_text("nothing to see here\n") + yaml = {} + + YamlFileManager.write(yaml, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_yaml_file_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.yaml") + + with self.assertRaises(FileNotFoundError): + YamlFileManager.write(self.yaml.load("key: some value"), config_path) + + def test_yaml_put_comment(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + yaml_doc = self.yaml.load("version: 0.1\nconfig_env:\n topic2:\n parameters:\n word: clarity\n") + + yaml_doc = YamlFileManager.put_comment(yaml_doc, "This is a comment") + + self.yaml.dump(yaml_doc, config_path) + txt = config_path.read_text() + self.assertIn("# This is a comment", txt) + + +@skip("JSON config support disabled") +class TestJsonFileManager(TestCase): + def test_read_json(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + config_path.write_text( + json.dumps( + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + indent=JsonFileManager.INDENT_SIZE, + ) + ) + + config_doc = JsonFileManager.read(config_path) + + self.assertEqual( + config_doc, + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + ) + + def test_read_json_invalid_json(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + config_path.write_text("{\n" + ' "bad_file": "very bad"\n' + ' "improperly": "formatted"\n' + "}\n") + + with self.assertRaises(FileParseException): + JsonFileManager.read(config_path) + + def test_read_json_file_path_not_valid(self): + config_dir = "path/that/doesnt/exist" + config_path = Path(config_dir, "samconfig.json") + + config_doc = JsonFileManager.read(config_path) + + self.assertEqual(config_doc, {}) + + def test_write_json(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + json_doc = { + "version": 0.1, + "config_env": {"topic2": {"parameters": {"word": "clarity"}}}, + COMMENT_KEY: "This is a comment", + } + + JsonFileManager.write(json_doc, config_path) + + txt = config_path.read_text() + self.assertIn('"version": 0.1', txt) + self.assertIn('"config_env": {', txt) + self.assertIn('"topic2": {', txt) + self.assertIn('"parameters": {', txt) + self.assertIn('"word": "clarity"', txt) + self.assertIn(f'"{COMMENT_KEY}": "This is a comment"', txt) + + def test_dont_write_json_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + config_path.write_text("nothing to see here\n") + json_doc = {} + + JsonFileManager.write(json_doc, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_json_file_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.json") + + with self.assertRaises(FileNotFoundError): + JsonFileManager.write({"key": "value"}, config_path) + + def test_json_put_comment(self): + json_doc = {"version": 0.1, "config_env": {"topic1": {"parameters": {"word": "clarity"}}}} + + json_doc = JsonFileManager.put_comment(json_doc, "This is a comment") + + txt = json.dumps(json_doc) + self.assertIn(f'"{COMMENT_KEY}": "This is a comment"', txt) diff --git a/tests/unit/lib/samconfig/test_samconfig.py b/tests/unit/lib/samconfig/test_samconfig.py index 7a86e6f97d..c58f0709a6 100644 --- a/tests/unit/lib/samconfig/test_samconfig.py +++ b/tests/unit/lib/samconfig/test_samconfig.py @@ -1,10 +1,21 @@ import os from pathlib import Path +from unittest.mock import patch +from parameterized import parameterized +import tempfile from unittest import TestCase -from samcli.lib.config.exceptions import SamConfigVersionException -from samcli.lib.config.samconfig import SamConfig, DEFAULT_CONFIG_FILE_NAME, DEFAULT_GLOBAL_CMDNAME, DEFAULT_ENV +from samcli.lib.config.exceptions import SamConfigFileReadException, SamConfigVersionException +from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER, JsonFileManager, TomlFileManager, YamlFileManager +from samcli.lib.config.samconfig import ( + DEFAULT_CONFIG_FILE, + SamConfig, + DEFAULT_CONFIG_FILE_NAME, + DEFAULT_GLOBAL_CMDNAME, + DEFAULT_ENV, +) from samcli.lib.config.version import VERSION_KEY, SAM_CONFIG_VERSION +from samcli.lib.telemetry.event import Event from samcli.lib.utils import osutils @@ -182,27 +193,27 @@ def test_check_sanity(self): def test_check_version_non_supported_type(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) - self.samconfig.document.add(VERSION_KEY, "aadeff") + self.samconfig.document.pop(VERSION_KEY) + self.samconfig.document.update({VERSION_KEY: "aadeff"}) with self.assertRaises(SamConfigVersionException): self.samconfig.sanity_check() def test_check_version_no_version_exists(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) + self.samconfig.document.pop(VERSION_KEY) with self.assertRaises(SamConfigVersionException): self.samconfig.sanity_check() def test_check_version_float(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) - self.samconfig.document.add(VERSION_KEY, 0.2) + self.samconfig.document.pop(VERSION_KEY) + self.samconfig.document.update({VERSION_KEY: 0.2}) self.samconfig.sanity_check() def test_write_config_file_non_standard_version(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) - self.samconfig.document.add(VERSION_KEY, 0.2) + self.samconfig.document.pop(VERSION_KEY) + self.samconfig.document.update({VERSION_KEY: 0.2}) self.samconfig.put(cmd_names=["local", "start", "api"], section="parameters", key="skip_pull_image", value=True) self.samconfig.sanity_check() self.assertEqual(self.samconfig.document.get(VERSION_KEY), 0.2) @@ -210,7 +221,7 @@ def test_write_config_file_non_standard_version(self): def test_write_config_file_will_create_the_file_if_not_exist(self): with osutils.mkdir_temp(ignore_errors=True) as tempdir: non_existing_dir = os.path.join(tempdir, "non-existing-dir") - non_existing_file = "non-existing-file" + non_existing_file = "non-existing-file.toml" samconfig = SamConfig(config_dir=non_existing_dir, filename=non_existing_file) self.assertFalse(samconfig.exists()) @@ -221,3 +232,84 @@ def test_write_config_file_will_create_the_file_if_not_exist(self): samconfig.put(cmd_names=["any", "command"], section="any-section", key="any-key", value="any-value") samconfig.flush() self.assertTrue(samconfig.exists()) + + def test_passed_filename_used(self): + config_path = Path(self.config_dir, "myconfigfile.toml") + + self.assertFalse(config_path.exists()) + + self.samconfig = SamConfig(self.config_dir, filename="myconfigfile.toml") + self.samconfig.put( # put some config options so it creates the file + cmd_names=["any", "command"], section="section", key="key", value="value" + ) + self.samconfig.flush() + + self.assertTrue(config_path.exists()) + self.assertFalse(Path(self.config_dir, DEFAULT_CONFIG_FILE_NAME).exists()) + + def test_config_uses_default_if_none_provided(self): + self.samconfig = SamConfig(self.config_dir) + self.samconfig.put( # put some config options so it creates the file + cmd_names=["any", "command"], section="section", key="key", value="value" + ) + self.samconfig.flush() + + self.assertTrue(Path(self.config_dir, DEFAULT_CONFIG_FILE_NAME).exists()) + + def test_config_priority(self): + config_files = [] + extensions_in_priority = list(FILE_MANAGER_MAPPER.keys()) # priority by order in dict + for extension in extensions_in_priority: + filename = DEFAULT_CONFIG_FILE + extension + config = SamConfig(self.config_dir, filename=filename) + config.put( # put some config options so it creates the file + cmd_names=["any", "command"], section="section", key="key", value="value" + ) + config.flush() + config_files.append(config) + + while extensions_in_priority: + config = SamConfig(self.config_dir) + next_priority = extensions_in_priority.pop(0) + self.assertEqual(config.filepath, Path(self.config_dir, DEFAULT_CONFIG_FILE + next_priority)) + os.remove(config.path()) + + +class TestSamConfigFileManager(TestCase): + def test_file_manager_not_declared(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig") + + with self.assertRaises(SamConfigFileReadException): + SamConfig(config_path, filename="samconfig") + + def test_file_manager_unsupported(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.jpeg") + + with self.assertRaises(SamConfigFileReadException): + SamConfig(config_path, filename="samconfig.jpeg") + + @parameterized.expand( + [ + ("samconfig.toml", TomlFileManager, ".toml"), + ("samconfig.yaml", YamlFileManager, ".yaml"), + ("samconfig.yml", YamlFileManager, ".yml"), + # ("samconfig.json", JsonFileManager, ".json"), + ] + ) + @patch("samcli.lib.telemetry.event.EventTracker.track_event") + def test_file_manager(self, filename, expected_file_manager, expected_extension, track_mock): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, filename) + tracked_events = [] + + def mock_tracker(name, value): # when track_event is called, just append the Event to our list + tracked_events.append(Event(name, value)) + + track_mock.side_effect = mock_tracker + + samconfig = SamConfig(config_path, filename=filename) + + self.assertIs(samconfig.file_manager, expected_file_manager) + self.assertIn(Event("SamConfigFileExtension", expected_extension), tracked_events)