Add expansion feature

Expand .env if specified in yaml configuration

pystarport/cluster.py

@@ -18,12 +18,14 @@
 import tomlkit
 import yaml
 from dateutil.parser import isoparse
+from dotenv import dotenv_values, load_dotenv
 from supervisor import xmlrpc
 from supervisor.compat import xmlrpclib
 
 from . import ports
 from .app import CHAIN, IMAGE, SUPERVISOR_CONFIG_FILE
 from .cosmoscli import ChainCommand, CosmosCLI, ModuleAccount, module_address
+from .expansion import expand_posix_vars
 from .ledger import ZEMU_BUTTON_PORT, ZEMU_HOST
 from .utils import format_doc_string, interact, write_ini
 
@@ -689,10 +691,7 @@ def create_account(cli, account, use_ledger=False):
             acct = cli.create_account_ledger(account["name"])
         elif account.get("address"):
             # if address field exists, will use account with that address directly
-            acct = {
-                "name": account.get("name"),
-                "address": account.get("address")
-            }
+            acct = {"name": account.get("name"), "address": account.get("address")}
         else:
             mnemonic = account.get("mnemonic")
             acct = cli.create_account(account["name"], mnemonic=mnemonic)
@@ -884,28 +883,47 @@ def relayer_chain_config(data_dir, chain, relayer_chains_config):
     rpc_port = ports.rpc_port(cfg["validators"][0]["base_port"])
     grpc_port = ports.grpc_port(cfg["validators"][0]["base_port"])
 
-    chain_config = next((i for i in relayer_chains_config if i["id"] == chain["chain_id"]), {})
+    chain_config = next(
+        (i for i in relayer_chains_config if i["id"] == chain["chain_id"]), {}
+    )
 
-    return jsonmerge.merge({
-        "key_name": "relayer",
-        "id": chain["chain_id"],
-        "rpc_addr": f"http://localhost:{rpc_port}",
-        "grpc_addr": f"http://localhost:{grpc_port}",
-        "websocket_addr": f"ws://localhost:{rpc_port}/websocket",
-        "rpc_timeout": "10s",
-        "account_prefix": chain.get("account-prefix", "cro"),
-        "store_prefix": "ibc",
-        "max_gas": 300000,
-        "gas_price": {"price": 0, "denom": "basecro"},
-        "trusting_period": "336h",
-    },chain_config)
+    return jsonmerge.merge(
+        {
+            "key_name": "relayer",
+            "id": chain["chain_id"],
+            "rpc_addr": f"http://localhost:{rpc_port}",
+            "grpc_addr": f"http://localhost:{grpc_port}",
+            "websocket_addr": f"ws://localhost:{rpc_port}/websocket",
+            "rpc_timeout": "10s",
+            "account_prefix": chain.get("account-prefix", "cro"),
+            "store_prefix": "ibc",
+            "max_gas": 300000,
+            "gas_price": {"price": 0, "denom": "basecro"},
+            "trusting_period": "336h",
+        },
+        chain_config,
+    )
 
 
 def init_cluster(
     data_dir, config_path, base_port, image=IMAGE, cmd=None, gen_compose_file=False
 ):
     config = yaml.safe_load(open(config_path))
 
+    if "dotenv" in config:
+        dotenv = config.pop("dotenv", {})
+        if not isinstance(dotenv, str):
+            raise ValueError(f"Invalid value passed to dotenv: {dotenv}")
+        config_vars = load_system_envvars()
+        env_path = Path(config_path).parent.joinpath(dotenv)
+        if not env_path.is_file():
+            raise ValueError(
+                f"Dotenv specified in config but not found at path: {env_path}"
+            )
+        config_vars.update(dotenv_values(dotenv_path=env_path))  # type: ignore
+        load_dotenv(dotenv_path=env_path)
+        config = expand_posix_vars(config, config_vars)
+
     relayer_config = config.pop("relayer", {})
     for chain_id, cfg in config.items():
         cfg["path"] = str(config_path)
@@ -935,10 +953,11 @@ def init_cluster(
                             "log_level": "info",
                         },
                         "chains": [
-                            relayer_chain_config(data_dir, chain, relayer_chains_config) for chain in chains
+                            relayer_chain_config(data_dir, chain, relayer_chains_config)
+                            for chain in chains
                         ],
                     },
-                    relayer_config
+                    relayer_config,
                 )
             )
         )
@@ -1101,3 +1120,8 @@ def format_value(v, ctx):
     supervisord.wait()
     t.stop()
     t.join()
+
+
+def load_system_envvars():
+    config_vars = dict(os.environ)
+    return config_vars

pystarport/expansion.py

@@ -0,0 +1,58 @@
+import re
+from typing import Any, Mapping, Optional, Text
+
+from dotenv.variables import parse_variables
+
+
+def expand_posix_vars(obj: Any, variables: Mapping[Text, Optional[Any]]) -> Any:
+    """expand_posix_vars recursively expands POSIX values in an object.
+
+    Args:
+        obj (any): object in which to interpolate variables.
+        variables (dict): dictionary that maps variable names to their value
+    """
+    if isinstance(obj, (dict,)):
+        for key, val in obj.items():
+            obj[key] = expand_posix_vars(val, variables)
+    elif isinstance(obj, (list,)):
+        for index in range(len(obj)):
+            obj[index] = expand_posix_vars(obj[index], variables)
+    elif isinstance(obj, (str,)):
+        obj = _str_to_python_value(_expand(obj, variables))
+    return obj
+
+
+def _expand(value, variables={}):
+    """_expand does POSIX-style variable expansion
+
+    This is adapted from python-dotenv, specifically here:
+
+    https://github.com/theskumar/python-dotenv/commit/17dba65244c1d4d10f591fe37c924bd2c6fd1cfc
+
+    We need this layer here so we can explicitly pass in variables;
+    python-dotenv assumes you want to use os.environ.
+    """
+
+    if not isinstance(value, (str,)):
+        return value
+    atoms = parse_variables(value)
+    return "".join([str(atom.resolve(variables)) for atom in atoms])
+
+
+INT_REGEX = re.compile(r"^[-+]?[0-9]+$")
+
+
+def _str_to_python_value(val):
+    """_str_to_python_value infers the data type from a string.
+
+    This could eventually use PyYAML's parsing logic.
+    """
+    if not isinstance(val, (str,)):
+        return val
+    elif val == "true" or val == "True" or val == "on":
+        return True
+    elif val == "false" or val == "False" or val == "off":
+        return False
+    # elif INT_REGEX.match(val):
+    #     return int(val)
+    return val