MLO-61: Use ".apolo" directory and apolo-* files to configure apolo-f…

…low (#705)

* MLO-61: Use ".apolo" directory and apolo-* files to configure apolo-flow

* add changelog

* bump deps

* fix auto-merge

* bump more deps

* downgrade ssh-agent

.github/workflows/ci.yaml

@@ -27,7 +27,6 @@ jobs:
         uses: actions/checkout@v4
       - name: metadata
         id: metadata
-        if: github.actor == 'dependabot[bot]'
         uses: dependabot/fetch-metadata@v2
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yaml

@@ -14,12 +14,12 @@ jobs:
       PYTHONIOENCODING: utf-8
     steps:
       - name: Checkout commit
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Install python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/update-apolo-flow-actions.yml

@@ -1,33 +1,33 @@
 name: Update apolo-flow action tags
 on:
   schedule:
-  - cron: 0 0 * * *
+    - cron: 0 0 * * *
 jobs:
   auto-update:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.11
-    - name: Install dependencies
-      run: make setup
-    - id: run_updater
-      name: Run update actions script
-      run: python \{\{cookiecutter.flow_dir\}\}/update_actions.py ".neuro/*.y*ml"
-    - id: generate_token
-      uses: tibdex/github-app-token@v1
-      with:
-        app_id: ${{ secrets.BOT_APP_ID }}
-        private_key: ${{ secrets.BOT_PRIVATE_KEY }}
-    - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v4
-      with:
-        token: ${{ steps.generate_token.outputs.token }}
-        branch: update/actions-tags
-        title: Auto-update actions tags
-        commit-message: Auto-update neuro-flow actions tags
-        body: |
-          Updated apolo-flow action version tags in the following
-          files: ${{ steps.run_updater.outputs.updated_files }}
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: Install dependencies
+        run: make setup
+      - id: run_updater
+        name: Run update actions script
+        run: python \{\{cookiecutter.flow_dir\}\}/update_actions.py ".apolo/*.y*ml"
+      - id: generate_token
+        uses: tibdex/github-app-token@v2
+        with:
+          app_id: ${{ secrets.BOT_APP_ID }}
+          private_key: ${{ secrets.BOT_PRIVATE_KEY }}
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ steps.generate_token.outputs.token }}
+          branch: update/actions-tags
+          title: Auto-update actions tags
+          commit-message: Auto-update apolo-flow actions tags
+          body: |
+            Updated apolo-flow action version tags in the following
+            files: ${{ steps.run_updater.outputs.updated_files }}

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: "v4.6.0"
+    rev: "v5.0.0"
     hooks:
       - id: check-merge-conflict
   - repo: https://github.com/asottile/yesqa
@@ -12,13 +12,13 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: "24.8.0"
+    rev: "24.10.0"
     hooks:
       - id: black
         language_version: python3 # Should be a command that runs python3.6+
         args: ["--line-length", "88"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: "v4.6.0"
+    rev: "v5.0.0"
     hooks:
       - id: check-case-conflict
       - id: check-yaml
@@ -32,13 +32,13 @@ repos:
       - id: debug-statements
   # Another entry is required to apply file-contents-sorter to another file
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: "v4.6.0"
+    rev: "v5.0.0"
     hooks:
       - id: file-contents-sorter
         files: |
           .gitignore
   - repo: https://github.com/asottile/pyupgrade
-    rev: "v3.16.0"
+    rev: "v3.19.1"
     hooks:
       - id: pyupgrade
         args: ["--py36-plus"]

CHANGELOG.D/705.misc

@@ -0,0 +1 @@
+Use ".apolo" directory instead of ".neuro" and apolo* files istead of neuro* to configure apolo-flow.

Makefile

@@ -29,7 +29,7 @@ format:
 test:
 	 export TMP_DIR=$$(mktemp -d) && \
 	   cookiecutter --no-input --config-file ./tests/cookiecutter.yaml --output-dir $$TMP_DIR . && \
-	   ls -d "$$TMP_DIR/test flow/.neuro/"
+	   ls -d "$$TMP_DIR/test flow/.apolo/"
 	 pytest -v -n auto tests/unit
 	 pytest -v -n auto tests/e2e
 	 @echo -e "OK\n"

hooks/post_gen_project.py

@@ -31,14 +31,14 @@ async def get_project_name() -> str:
                     PROJECT_NAME = line.split()[2]
                     break
 if PROJECT_NAME:
-    proj_file = Path("./.neuro/project.yml")
+    proj_file = Path("./.apolo/project.yml")
     content = proj_file.read_text()
     content = content.replace(
         "# project_name: {project_name}", f"project_name: {PROJECT_NAME}"
     )
     proj_file.write_text("".join(content))
 else:
-    live_file = Path("./.neuro/live.yml")
+    live_file = Path("./.apolo/live.yml")
     content = live_file.read_text()
     content = content.replace("/$[[ project.project_name ]]/", "")
     live_file.write_text("".join(content))
@@ -47,8 +47,8 @@ async def get_project_name() -> str:
 
 # >>> Optionally clearing comments
 COMMENTS_STRUCTURE = {
-    "./.neuro/live.yml": r"(\s*#(?! yaml-language-server).*)",
-    "./.neuro/project.yml": r"(\s*#(?! yaml-language-server).*)",
+    "./.apolo/live.yml": r"(\s*#(?! yaml-language-server).*)",
+    "./.apolo/project.yml": r"(\s*#(?! yaml-language-server).*)",
 }
 PRESERVE_HINTS_VARIANS = {
     "yes": True,

requirements/dev.txt

@@ -1,12 +1,12 @@
 -r base.txt
 git+https://github.com/andriihomiak/pytest-cookies.git@fix/quotes_in_user_config # pytest-cookies==0.6.1
-mypy==1.12.1
+mypy==1.14.1
 pipx==1.7.1
-pre-commit==3.8.0
-PyGithub==1.55  # Bump also {{cookiecutter.flow_dir}}/.github/workflows/update-apolo-flow-actions.yml:15
-pytest==8.3.2
+pre-commit==4.0.1
+PyGithub==2.5.0  # Bump also {{cookiecutter.flow_dir}}/.github/workflows/update-apolo-flow-actions.yml:15
+pytest==8.3.4
 pytest-asyncio==0.24.0
 pytest-xdist==3.6.1
 PyYAML==6.0.2
 towncrier==24.8.0
-types-PyYAML==6.0.12.20240808
+types-PyYAML==6.0.12.20241230

tests/e2e/test_live.py

@@ -7,7 +7,7 @@
 
 # Also check whether comments and their removal does not break something
 @pytest.mark.parametrize("preserve_comments", ["yes", "no"])
-def test_neuro_flow_live(cookies: Cookies, preserve_comments: str) -> None:
+def test_apolo_flow_live(cookies: Cookies, preserve_comments: str) -> None:
     result = cookies.bake(
         extra_context={
             "flow_dir": "test-flow",

tests/unit/test_bake_project.py

@@ -85,8 +85,8 @@ def test_flow_config_with_comments(cookies: Cookies, preserve_comments: str) ->
     assert result.exit_code == 0
     comment_regex = re.compile(r"(\s*#(?! yaml-language-server).*)")
     with inside_dir(str(result.project_path)):
-        live_file_content = Path(".neuro/live.yml").read_text()
-        project_file_content = Path(".neuro/project.yml").read_text()
+        live_file_content = Path(".apolo/live.yml").read_text()
+        project_file_content = Path(".apolo/project.yml").read_text()
 
         l_com_exists = any(
             [
@@ -101,11 +101,11 @@ def test_flow_config_with_comments(cookies: Cookies, preserve_comments: str) ->
             ]
         )
         if preserve_comments == "yes":
-            assert l_com_exists, ".neuro/live.yml file does not contain comments"
-            assert p_com_exists, ".neuro/project.yml file does not contain comments"
+            assert l_com_exists, ".apolo/live.yml file does not contain comments"
+            assert p_com_exists, ".apolo/project.yml file does not contain comments"
         elif preserve_comments == "no":
-            assert not l_com_exists, ".neuro/live.yml file contains comments"
-            assert not p_com_exists, ".neuro/project.yml file contains comments"
+            assert not l_com_exists, ".apolo/live.yml file contains comments"
+            assert not p_com_exists, ".apolo/project.yml file contains comments"
         else:
             raise RuntimeError(
                 f"invalid value '{preserve_comments}' for 'preserve_comments' arg. "
@@ -134,7 +134,7 @@ def test_flow_name(cookies: Cookies) -> None:
     result = cookies.bake()
 
     assert result.exit_code == 0
-    project_yml = result.project_path / ".neuro" / "project.yml"
+    project_yml = result.project_path / ".apolo" / "project.yml"
     project_yml_content = yaml.safe_load(project_yml.read_text())
 
     assert "id" in project_yml_content

{{cookiecutter.flow_dir}}/.apolo.toml

@@ -0,0 +1,3 @@
+[storage]
+cp-exclude = [".git", ".apolo"]
+cp-exclude-from-files=[".apoloignore", ".dockerignore"]

{{cookiecutter.flow_dir}}/.neuro/live.yml → {{cookiecutter.flow_dir}}/.apolo/live.yml

@@ -54,7 +54,7 @@ images:
     ## Required. Image reference, can be of two types:
     ##  - Platform-hosted image - its reference should start with the 'image:' prefix. `apolo-flow build <img_id>` will work in this case.
     ##  - Image hosted on DockerHub - without the 'image:' prefix. In this case, `apolo-flow build <img_id>` will not work.
-    ## Check ./neuro/project.yaml to configure the $[[ flow.project_id ]] part.
+    ## Check .apolo/project.yaml to configure the $[[ flow.project_id ]] part.
     ## During job execution, the '$[[ flow.project_id ]]' part will be replaced with its string value by the Apolo-Flow engine.
     ## Hint: You can use the embedded `hash_files()` function to generate a built image's tag based on its content.
     ## Example:
@@ -64,7 +64,7 @@ images:
     ## An optional Docker file path used for building images, `Dockerfile` by default. The path should be relative to the context's root.
     context: $[[ flow.workspace ]]/
     ## Optional. The Docker context used to build an image, specified as a local path relative to the project's root folder.
-    ## The project's root folder is the folder that contains the '.neuro' directory,
+    ## The project's root folder is the folder that contains the '.apolo' directory,
     ##  its path can be referenced via $[[ flow.workspace ]]/.
     # build_preset: cpu-small
     ## Optional. Preset name used to build the Docker image.

{{cookiecutter.flow_dir}}/.neuro/project.yml → {{cookiecutter.flow_dir}}/.apolo/project.yml

@@ -3,6 +3,6 @@
 ## Check our full reference documentation at https://docs.apolo.us/apolo-flow-reference/workflow-syntax/project-configuration-syntax
 id: {{ cookiecutter.flow_id }}
 ## Could be referenced as $[[ project.id ]] or $[[ flow.project_id ]] contexts.
-## Default value: name of the folder containing `.neuro`
+## Default value: name of the folder containing `.apolo`
 # project_name: {project_name}
 ## Optional platform project name that owns the flow, referenced via $[[ project.project_name ]]

{{cookiecutter.flow_dir}}/.github/workflows/update-flow-action-versions.yml

@@ -12,17 +12,17 @@ jobs:
         with:
           python-version: 3.8
       - name: Install dependencies
-        run: pip install -U "PyGithub==1.55"
+        run: pip install -U "PyGithub==2.5.0"
       - id: run_updater
         name: Run update actions script
-        run: python update_actions.py ".neuro/*.y*ml"
+        run: python update_actions.py ".apolo/*.y*ml"
       - id: generate_token
-        uses: tibdex/github-app-token@v1
+        uses: tibdex/github-app-token@v2
         with:
           app_id: ${{ secrets.BOT_APP_ID }}
           private_key: ${{ secrets.BOT_PRIVATE_KEY }}
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3.10.1
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ steps.generate_token.outputs.token }}
           branch: update/actions-tags

{{cookiecutter.flow_dir}}/HELP.md

@@ -113,7 +113,7 @@ On a local machine, run `apolo-flow mkvolumes`. This command creates storage fol
 
 After the storage folders have been created, run `apolo-flow upload data` from the local machine as well. This command pushes local files stored in `./data` into the `storage:{{ cookiecutter.flow_id }}/data` volume mounted to your development environment's `/project/data`.
 
-You can upload (or download) every folder for which the `local` parameter is specified in the [live.yml file](./.neuro/live.yml).
+You can upload (or download) every folder for which the `local` parameter is specified in the [live.yml file](./.apolo/live.yml).
 
 ### Uploading data from Google Cloud Storage to a job