Merge branch 'main' into stable

.github/workflows/main.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11","3.12"]
     defaults:
       run:
         working-directory: .
@@ -33,13 +33,11 @@ jobs:
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
           pip install -e .
+      - name: Ruff Format Check
+        run: ruff format --check .
+      - name: Ruff Check
+        run: ruff check --output-format=github .
       - name: Mypy Check
         run: mypy ./seedfarmer
-      - name: Flake8 Check
-        run: flake8 ./seedfarmer
-      - name: Black Check
-        run: black --check ./seedfarmer
-      - name: Isort Check
-        run: isort --check ./seedfarmer
       - name: PyTest
         run: cp test/unit-test/mock_data/seedfarmer.yaml . && pytest

CHANGELOG.md

@@ -5,11 +5,25 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
 
 ## Unreleased
 
+### New
+- adding support for S3 to store bundles
+- adding seedfarmer version tag to toolchain and deployment roles
+- removing python 3.7 support
+- adding python 3.12 support 
+
+### Changes
+- adding local path of manifests that fail to load to the actual final string printed
+  - this is already being done, but moving closer to last line of output
+### Fixes
+
+## v3.5.1 (2024-05-21)
+
 ### New
 
 ### Changes
 
 ### Fixes
+- updating requests library  `requests==2.32.0`
 
 
 ## v3.5.1 (2024-05-21)

VERSION

@@ -1 +1,2 @@
 3.5.1
+

docs/requirements-docs.txt

@@ -12,15 +12,15 @@ babel==2.12.1
     # via sphinx
 certifi==2023.7.22
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   requests
 charset-normalizer==3.1.0
     # via requests
 click==8.1.7
     # via sphinx-click
 docutils==0.18.1
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   myst-parser
     #   sphinx
     #   sphinx-click
@@ -29,14 +29,12 @@ idna==3.7
     # via requests
 imagesize==1.4.1
     # via sphinx
-importlib-metadata==7.0.1
-    # via sphinx
 jinja2==3.1.4
     # via
     #   myst-parser
     #   sphinx
     #   sphinx-autoapi
-lazy-object-proxy==1.9.0
+lazy-object-proxy==1.10.0
     # via astroid
 markdown-it-py==2.2.0
     # via
@@ -49,17 +47,17 @@ mdit-py-plugins==0.3.5
 mdurl==0.1.2
     # via markdown-it-py
 myst-parser==1.0.0
-    # via -r requirements-docs.in
+    # via -r docs/requirements-docs.in
 packaging==23.1
     # via sphinx
 pygments==2.15.1
     # via sphinx
 pyyaml==5.4
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   myst-parser
     #   sphinx-autoapi
-requests==2.31.0
+requests==2.32.0
     # via sphinx
 snowballstemmer==2.2.0
     # via sphinx
@@ -71,52 +69,48 @@ sphinx==6.2.1
     #   sphinx-rtd-theme
     #   sphinxcontrib-jquery
 sphinx-autoapi==2.1.0
-    # via -r requirements-docs.in
+    # via -r docs/requirements-docs.in
 sphinx-click==5.1.0
-    # via -r requirements-docs.in
+    # via -r docs/requirements-docs.in
 sphinx-rtd-theme==1.2.1
-    # via -r requirements-docs.in
+    # via -r docs/requirements-docs.in
 sphinxcontrib-applehelp==1.0.4
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx
 sphinxcontrib-devhelp==1.0.2
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx
 sphinxcontrib-htmlhelp==2.0.1
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx
 sphinxcontrib-jquery==4.1
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx-rtd-theme
 sphinxcontrib-jsmath==1.0.1
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx
 sphinxcontrib-qthelp==1.0.3
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   sphinx
 typing-extensions==4.5.0
-    # via
-    #   -r requirements-docs.in
-    #   astroid
+    # via -r docs/requirements-docs.in
 unidecode==1.3.6
     # via sphinx-autoapi
 urllib3==1.26.18
     # via
-    #   -r requirements-docs.in
+    #   -r docs/requirements-docs.in
     #   requests
 wheel==0.38.1
-    # via -r requirements-docs.in
+    # via -r docs/requirements-docs.in
 wrapt==1.15.0
     # via astroid
-zipp==3.17.0
-    # via importlib-metadata

docs/source/architecture.md

@@ -13,7 +13,8 @@ module code and the AWS Cloud via AWS CodeSeeder.
 2. **seedfarmer** via `toolchain role` reads/writes deployment metadata with AWS Systems Manager
 3. **seedfarmer** `toolchain role` assumes `deployment role` in all `target acccount` to fetch module metadata
 4. **seedfarmer** via `deployment role` in `target acccount` initiates module deployment ([see below](method_processing))
-5. **seedfarmer** via `deployment role` assumes `module deployment role` to complete module deployment
+5. **seedfarmer** via `deployment role` interacts with S3 for bundle references
+6. **seedfarmer** via `deployment role` assumes `module deployment role` to complete module deployment
 
 
 (method_processing)=

docs/source/conf.py

@@ -33,7 +33,13 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ["sphinx.ext.autosectionlabel", "sphinx.ext.napoleon", "myst_parser", "autoapi.extension","sphinx_click.ext"]
+extensions = [
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.napoleon",
+    "myst_parser",
+    "autoapi.extension",
+    "sphinx_click.ext",
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]

docs/source/upgrades.md

@@ -14,4 +14,44 @@ If you are currently using an older version of `seed-farmer`, to upgrade you mus
 seedfarmer apply <manifest-path> --update-seedkit
 ```
 
-Please see the [CLI references](./cli_commands.rst) for more details
+Please see the [CLI references](./cli_commands.rst) for more details
+
+
+## Upgrading to 4.0.0  
+This is a **BREAKING CHANGE !!!**
+
+`seed-farmer` 4.0.0 introduces the support of S3 to persist bundles of sucessfully deployed modules.  This is meant to help when destroying modules that may have changed code over time.  For example, if a developer has deployed a module from a local path, then moves that module, `seed-farmer` will still be able to destroy that module as the bundle used to deploy is stored in S3.  The change is backward-compatible, but the permissions ARE BREAKING from an older version, so you MUST perform the following steps.  
+
+To upgrade:
+1. Identify ALL target accounts where modules are deployed and DELETE the `seedfarmer-<project>-deployment-role` Cloudformation stack in these accounts (IAM is a global service, not regional but the Cloudformation template is regional)
+2. Update your version of `seed-farmer` via
+    ```code
+    pip install --upgrade seed-farmer==4.0.0
+     ```
+3. Redeploy the `seedfarmer-<project>-deployment-role` in EVERY ACCOUNT
+    ``` code
+    seedfarmer bootstrap target -p <project> -t <toolchain-account-id>
+    ``` 
+4. Update the project policy on the next deployment (only once) *See Below*
+    ```code 
+    seedfarmer apply <manifest-path> --update-project-policy
+    ```  
+
+**NOTE:** If you are using your own `projectpolicy.yaml` (not using the one provided by `seed-farmer` and have defined it in your manifests) you can update your project policy with the following prior to running ***Step 4*** :
+
+```yaml
+          - Action:
+            - s3:Delete*
+            - s3:Put*
+            - s3:Get*
+            - s3:Create*
+            - s3:List*
+            Effect: Allow
+            Resource:
+            - Fn::Sub: "arn:${AWS::Partition}:s3:::seedfarmer-${ProjectName}*"
+            - Fn::Sub: "arn:${AWS::Partition}:s3:::seedfarmer-${ProjectName}*/*" 
+```
+      
+  ** *please update ${ProjectName} accordingly -- this is for you to manage*
+
+Your existing deployment is unaffected after this change, and `seed-farmer` will continue to destroy as it previously did UNTIL the module you are looking to destroy has been sucessfully deployed with this version.  In other words, your modules WILL NOT benefit from the persisted bundle feature UNTIL they are deployed successfully with this new `seed-farmer` version.  In that regard, `seed-farmer` will continue to delete modules they way it always has (is backward compatible).

pyproject.toml

@@ -1,37 +1,34 @@
-[tool.black]
+[tool.ruff]
+exclude = [
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".ruff_cache",
+    ".tox",
+    ".venv",
+    ".env",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "codeseeder.out",
+    "seedfarmer.gitmodules"
+]
 line-length = 120
-target-version = ["py37", "py38", "py39", "py310", "py311"]
-exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | \.env
-  | _build
-  | buck-out
-  | build
-  | dist
-  | codeseeder.out
-  | seedfarmer.gitmodules
-)/
-'''
+target-version = "py38"
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I"]
+ignore = []
+fixable = ["ALL"]
 
-[tool.isort]
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-ensure_newline_before_comments = true
-line_length = 120
-src_paths = ["seedfarmer"]
-py_version = 37
-skip_gitignore = false
+[tool.ruff.lint.per-file-ignores]
+"docs/*" = ["E402"]
+"test/unit-test/mock_data/*" = ["E501"]
 
 [tool.mypy]
-python_version = "3.7"
+python_version = "3.8"
 strict = true
 ignore_missing_imports = true
 disallow_untyped_decorators = false

requirements-dev.in

@@ -1,9 +1,6 @@
 awscli~=1.31.13
-black~=24.3.0
 certifi~=2023.7.22
 check-manifest~=0.48
-flake8~=5.0.4
-isort~=5.10.1
 mypy~=0.961
 myst-parser~=0.18.0
 pip-tools~=6.14.0
@@ -13,6 +10,7 @@ pytest~=7.2.0
 pytest-cov~=4.0.0
 pytest-mock~=3.10.0
 pytest-ordering~=0.6
+ruff~=0.4.4
 twine~=4.0.1
 types-PyYAML~=6.0.8
 types-setuptools~=57.4.17