Add expansion feature (#37)

* Add expansion feature

Read dotenv file and expand based on env and system env

* Add --dotenv cli option and expand_yaml

1. `--dotenv`: an optional cli option for overriding the dotenv option in
yaml file
2. move the expansion codes to expansion.py and handle it in `expand_yaml`

* Back to original even it is too long

To make diff easy.

* Fix LGTM alerts

* Remove _str_to_python_value and remove {}

* Add test: test_expansion

* Add Makefile and test action

* Update Makefile to test all tests

* Move dotenv=None to init_cluster definition

It is more general and backward compatible.

* Move pytest and deepdiff to dev

.github/workflows/test.yml

@@ -0,0 +1,20 @@
+name: test
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: install dependencies
+        run: python3 -m pip install --user --upgrade poetry
+      - name: install
+        run: poetry install
+      - name: Run tests
+        run: make test

Makefile

@@ -0,0 +1,2 @@
+test:
+	@(cd pystarport/tests && poetry run pytest)

pyproject.toml

@@ -22,8 +22,11 @@ supervisor = "^4.2.1"
 docker = "^4.3.1"
 bech32 = "^1.1.0"
 multitail2 = "^1.5.2"
+python-dotenv = "^0.19.2"
 
 [tool.poetry.dev-dependencies]
+pytest = "^6.2.5"
+deepdiff = "^5.6.0"
 
 [tool.poetry.scripts]
 pystarport = "pystarport.cli:main"

pystarport/cli.py

@@ -15,14 +15,15 @@ def init(
     data,
     config,
     base_port,
+    dotenv,
     *args,
     **kwargs,
 ):
     interact(
         f"rm -r {data}; mkdir {data}",
         ignore_error=True,
     )
-    return init_cluster(data, config, base_port, *args, **kwargs)
+    return init_cluster(data, config, base_port, dotenv, *args, **kwargs)
 
 
 def start(data, quiet):
@@ -42,8 +43,8 @@ def start(data, quiet):
         tailer.join()
 
 
-def serve(data, config, base_port, cmd, quiet):
-    init(data, config, base_port, cmd=cmd)
+def serve(data, config, base_port, dotenv, cmd, quiet):
+    init(data, config, base_port, dotenv, cmd=cmd)
     start(data, quiet)
 
 
@@ -59,6 +60,7 @@ def init(
         data: str = "./data",
         config: str = "./config.yaml",
         base_port: int = 26650,
+        dotenv: str = None,
         image: str = IMAGE,
         gen_compose_file: bool = False,
         cmd: str = CHAIN,
@@ -70,11 +72,12 @@ def init(
         :param config: path to the configuration file
         :param base_port: the base port to use, the service ports of different nodes
         are calculated based on this
+        :param dotenv: path to .env file
         :param image: the image used in the generated docker-compose.yml
         :param gen_compose_file: generate a docker-compose.yml
         :param cmd: path to chain binary
         """
-        init(Path(data), config, base_port, image, self.cmd, gen_compose_file)
+        init(Path(data), config, base_port, dotenv, image, self.cmd, gen_compose_file)
 
     def start(self, data: str = "./data", quiet: bool = False):
         """
@@ -99,6 +102,7 @@ def serve(
         data: str = "./data",
         config: str = "./config.yaml",
         base_port: int = 26650,
+        dotenv: str = None,
         quiet: bool = False,
     ):
         """
@@ -108,10 +112,11 @@ def serve(
         :param config: path to the configuration file
         :param base_port: the base port to use, the service ports of different nodes
         are calculated based on this
+        :param dotenv: path to .env file
         :param cmd: path to chain binary
         :param quiet: don't print logs of subprocesses
         """
-        serve(Path(data), config, base_port, self.cmd, quiet)
+        serve(Path(data), config, base_port, dotenv, self.cmd, quiet)
 
     def supervisorctl(self, *args, data: str = "./data"):
         from supervisor.supervisorctl import main

pystarport/cluster.py

@@ -24,6 +24,7 @@
 from . import ports
 from .app import CHAIN, IMAGE, SUPERVISOR_CONFIG_FILE
 from .cosmoscli import ChainCommand, CosmosCLI, ModuleAccount, module_address
+from .expansion import expand_yaml
 from .ledger import ZEMU_BUTTON_PORT, ZEMU_HOST
 from .utils import format_doc_string, interact, write_ini
 
@@ -902,9 +903,9 @@ def relayer_chain_config(data_dir, chain, relayer_chains_config):
 
 
 def init_cluster(
-    data_dir, config_path, base_port, image=IMAGE, cmd=None, gen_compose_file=False
+    data_dir, config_path, base_port, dotenv=None, image=IMAGE, cmd=None, gen_compose_file=False
 ):
-    config = yaml.safe_load(open(config_path))
+    config = expand_yaml(config_path, dotenv)
 
     relayer_config = config.pop("relayer", {})
     for chain_id, cfg in config.items():