diff --git a/.gitignore b/.gitignore index b4b613fa82..3972825143 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ dump.rdb log/ results/ +result tags terraform.tfstate* test/builder-api/built/* diff --git a/components/hab/src/cli.rs b/components/hab/src/cli.rs index 73e13e556f..017a1f6a36 100644 --- a/components/hab/src/cli.rs +++ b/components/hab/src/cli.rs @@ -580,6 +580,19 @@ pub fn get() -> App<'static, 'static> { (@arg SCAFFOLDING: --scaffolding -s +takes_value "Specify explicit Scaffolding for your app (ex: node, ruby)") ) + (@subcommand render => + (about: "Renders plan config files") + (aliases: &["r", "re", "ren", "rend", "rende"]) + (@arg TEMPLATE_PATH: +required {file_exists} "Path to config to render") + (@arg DEFAULT_TOML: -d --("default-toml") +takes_value default_value("./default.toml") "Path to default.toml") + (@arg USER_TOML: -u --("user-toml") +takes_value "Path to user.toml, defaults to none") + (@arg MOCK_DATA: -m --("mock-data") +takes_value "Path to json file with mock data for template, defaults to none") + (@arg PRINT: -p --("print") "Prints config to STDOUT") + (@arg RENDER_DIR: -r --("render-dir") +takes_value default_value("./results") "Path to render templates") + (@arg NO_RENDER: -n --("no-render") "Don't write anything to disk, ignores --render-dir") + (@arg QUIET: -q --("no-verbose") --quiet + "Don't print any helper messages. When used with `--print` will only print config file") + ) ) (@subcommand ring => (about: "Commands relating to Habitat rings") diff --git a/components/hab/src/command/plan/mod.rs b/components/hab/src/command/plan/mod.rs index 66d2764efe..4aad7f1dfc 100644 --- a/components/hab/src/command/plan/mod.rs +++ b/components/hab/src/command/plan/mod.rs @@ -13,3 +13,4 @@ // limitations under the License. pub mod init; +pub mod render; diff --git a/components/hab/src/command/plan/render.rs b/components/hab/src/command/plan/render.rs new file mode 100644 index 0000000000..e66c7fd147 --- /dev/null +++ b/components/hab/src/command/plan/render.rs @@ -0,0 +1,170 @@ +// Copyright (c) 2019 Chef Software Inc. and/or applicable contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde_json::{self, + json, + Value as Json}; +use std::{fs::{create_dir_all, + read_to_string, + File}, + io::Write, + path::Path}; +use toml::Value; + +use crate::{common::{templating::TemplateRenderer, + ui::{Status, + UIWriter, + UI}}, + error::Result}; + +pub fn start(ui: &mut UI, + template_path: &Path, + default_toml_path: &Path, + user_toml_path: Option<&Path>, + mock_data_path: Option<&Path>, + print: bool, + render: bool, + render_dir: &Path, + quiet: bool) + -> Result<()> { + // Strip the file name out of our passed template + let file_name = Path::new(template_path.file_name().expect("valid template file")); + + if !quiet { + ui.begin(format!("Rendering: {} into: {} as: {}", + template_path.display(), + render_dir.display(), + file_name.display()))?; + ui.br()?; + } + + // read our template from file + let template = read_to_string(&template_path)?; + + // create a "data" json struct + let mut data = json!({}); + + if !quiet { + // import default.toml values, convert to JSON + ui.begin(format!("Importing default.toml: {}", &default_toml_path.display()))?; + } + + // we should always have a default.toml, would be nice to "autodiscover" based on package name, + // for now assume we're working in the plan dir if --default-toml not passed + let default_toml = read_to_string(&default_toml_path)?; + + // merge default into data struct + merge(&mut data, toml_to_json(&default_toml)?); + + // import default.toml values, convert to JSON + let user_toml = match user_toml_path { + Some(path) => { + if !quiet { + // print helper message, maybe only print if '--verbose'? how? + ui.begin(format!("Importing user.toml: {}", path.display()))?; + } + read_to_string(path)? + } + None => String::new(), + }; + // merge default into data struct + merge(&mut data, toml_to_json(&user_toml)?); + + // read mock data if provided + let mock_data = match mock_data_path { + Some(path) => { + if !quiet { + // print helper message, maybe only print if '--verbose'? how? + ui.begin(format!("Importing override file: {}", path.display()))?; + } + read_to_string(path)? + } + // return an empty json block if '--mock-data' isn't defined. + // this allows us to merge an empty JSON block + None => "{}".to_string(), + }; + // merge mock data into data + merge(&mut data, serde_json::from_str(&mock_data)?); + + // create a template renderer + let mut renderer = TemplateRenderer::new(); + // register our template + renderer.register_template_string(&template, &template) + .expect("Could not register template content"); + // render our JSON override in our template. + let rendered_template = renderer.render(&template, &data)?; + + if print { + if !quiet { + ui.br()?; + ui.warn(format!("###======== Rendered template: {}", + &template_path.display()))?; + } + + println!("{}", rendered_template); + + if !quiet { + ui.warn(format!("========### End rendered template: {}", + &template_path.display()))?; + } + } + + if render { + // Render our template file + create_with_template(ui, &render_dir, &file_name, &rendered_template, quiet)?; + } + + if !quiet { + ui.br()?; + } + Ok(()) +} + +fn toml_to_json(cfg: &str) -> Result { + let toml_value = cfg.parse::()?; + let toml_string = serde_json::to_string(&toml_value)?; + let json = serde_json::from_str(&format!(r#"{{ "cfg": {} }}"#, &toml_string))?; + Ok(json) +} + +// merge two Json structs +fn merge(a: &mut Json, b: Json) { + if let Json::Object(a_map) = a { + if let Json::Object(b_map) = b { + for (k, v) in b_map { + merge(a_map.entry(k).or_insert(Json::Null), v); + } + return; + } + } + *a = b; +} + +fn create_with_template(ui: &mut UI, + render_dir: &std::path::Path, + file_name: &std::path::Path, + template: &str, + quiet: bool) + -> Result<()> { + let path = Path::new(&render_dir).join(&file_name); + if !quiet { + ui.status(Status::Creating, format!("file: {}", path.display()))?; + } + + create_dir_all(render_dir)?; + + // Write file to disk + File::create(path).and_then(|mut file| file.write(template.as_bytes()))?; + Ok(()) +} diff --git a/components/hab/src/error.rs b/components/hab/src/error.rs index 7af1cb5265..43cdce1f10 100644 --- a/components/hab/src/error.rs +++ b/components/hab/src/error.rs @@ -62,6 +62,7 @@ pub enum Error { JobGroupPromoteOrDemote(api_client::Error, bool /* promote */), JobGroupCancel(api_client::Error), JobGroupPromoteOrDemoteUnprocessable(bool /* promote */), + JsonErr(serde_json::Error), NameLookup, NetErr(net::NetErr), PackageArchiveMalformed(String), @@ -156,6 +157,7 @@ impl fmt::Display for Error { if promote { "promote" } else { "demote" }, e) } + Error::JsonErr(ref e) => e.to_string(), Error::JobGroupCancel(ref e) => format!("Failed to cancel job group: {:?}", e), Error::NameLookup => "Error resolving a name or IP address".to_string(), Error::NetErr(ref e) => e.to_string(), @@ -238,6 +240,7 @@ impl error::Error for Error { Error::ProvidesError(_) => { "Can't find a package that provides the given search parameter" } + Error::JsonErr(ref err) => err.description(), Error::RemoteSupResolutionError(_, ref err) => err.description(), Error::RootRequired => { "Root or administrator permissions required to complete operation" @@ -300,3 +303,7 @@ impl From for Error { impl From for Error { fn from(err: net::NetErr) -> Self { Error::NetErr(err) } } + +impl From for Error { + fn from(err: serde_json::Error) -> Self { Error::JsonErr(err) } +} diff --git a/components/hab/src/lib.rs b/components/hab/src/lib.rs index ed75b517e4..d46aa5c57b 100644 --- a/components/hab/src/lib.rs +++ b/components/hab/src/lib.rs @@ -36,6 +36,8 @@ extern crate log; #[macro_use] extern crate serde_derive; +extern crate serde_json; + #[cfg(windows)] extern crate widestring; #[cfg(windows)] diff --git a/components/hab/src/main.rs b/components/hab/src/main.rs index 2b6c6065b3..bf3301c442 100644 --- a/components/hab/src/main.rs +++ b/components/hab/src/main.rs @@ -280,6 +280,7 @@ fn start(ui: &mut UI) -> Result<()> { ("plan", Some(matches)) => { match matches.subcommand() { ("init", Some(m)) => sub_plan_init(ui, m)?, + ("render", Some(m)) => sub_plan_render(ui, m)?, _ => unreachable!(), } } @@ -702,6 +703,32 @@ fn sub_plan_init(ui: &mut UI, m: &ArgMatches<'_>) -> Result<()> { name) } +fn sub_plan_render(ui: &mut UI, m: &ArgMatches<'_>) -> Result<()> { + let template_path = Path::new(m.value_of("TEMPLATE_PATH").unwrap()); + + let default_toml_path = Path::new(m.value_of("DEFAULT_TOML").unwrap()); + + let user_toml_path = m.value_of("USER_TOML").map(Path::new); + + let mock_data_path = m.value_of("MOCK_DATA").map(Path::new); + + let print = m.is_present("PRINT"); + let render = !m.is_present("NO_RENDER"); + let quiet = m.is_present("QUIET"); + + let render_dir = Path::new(m.value_of("RENDER_DIR").unwrap()); + + command::plan::render::start(ui, + template_path, + default_toml_path, + user_toml_path, + mock_data_path, + print, + render, + render_dir, + quiet) +} + fn sub_pkg_install(ui: &mut UI, m: &ArgMatches<'_>) -> Result<()> { let url = bldr_url_from_matches(&m)?; let channel = channel_from_matches_or_default(m); diff --git a/test/fixtures/render/README.md b/test/fixtures/render/README.md new file mode 100644 index 0000000000..e9a587fe18 --- /dev/null +++ b/test/fixtures/render/README.md @@ -0,0 +1,80 @@ +# About + +These files are for testing `hab plan render` command. + +see `hab plan render --help` for full usage instructions. + +# Usage + +Try: + + +``` +cargo run -p hab plan render ./test/fixtures/render/consul/config/consul_config.json \ + --default-toml ./test/fixtures/render/consul/default.toml \ + --user-toml ./test/fixtures/render/consul/user.toml \ + --mock-data ./test/fixtures/render/consul/override.json \ + --render-dir ~/result/config \ + --print +``` + +``` +cargo run -p hab plan render ./test/fixtures/render/consul/config/consul_config.json \ + --default-toml ./test/fixtures/render/consul/default.toml \ + --render-dir ~/result/config \ + --print +``` + +or + +``` +cargo run -p hab plan render ./test/fixtures/render/consul/hooks/run \ + --default-toml ./test/fixtures/render/consul/default.toml \ + --user-toml ./test/fixtures/render/consul/user.toml \ + --mock-data ./test/fixtures/render/consul/override.json \ + --render-dir ~/result/hooks \ + --print +``` + +# Example output + +* `consul/config/basic_config.json` render: + +``` +{ + "datacenter": "IN_OVERRIDE_JSON", + "data_dir": "IN_DEFAULT_TOML", + "log_level": "IN_USER_TOML", + "bind_addr": "9.9.9.9", + "client_addr": "9.9.9.9", + "server": true, + "retry_join": [ + ], + "ports": { + "dns": 6666, + "http": 6667, + "https": 6668, + "serf_lan": 8888, + "serf_wan": 8302, + "server": 9999 + } +} +``` + +* `consul/hook/run` render: + +``` +#!/bin/sh + +exec 2>&1 + +SERVERMODE=true +export CONSUL_UI_LEGACY=false + +CONSUL_OPTS="-dev" +if [ "$SERVERMODE" = true ]; then + CONSUL_OPTS=" -ui -server -bootstrap-expect 3 -config-file=/basic_config.json" +fi + +exec consul agent ${CONSUL_OPTS} +``` diff --git a/test/fixtures/render/consul/config/consul_config.json b/test/fixtures/render/consul/config/consul_config.json new file mode 100644 index 0000000000..fdb1985ead --- /dev/null +++ b/test/fixtures/render/consul/config/consul_config.json @@ -0,0 +1,21 @@ +{ + "datacenter": "{{cfg.server.datacenter}}", + "data_dir": "{{cfg.server.data-dir}}", + "log_level": "{{cfg.server.loglevel}}", + "bind_addr": "{{sys.ip}}", + "client_addr": "{{sys.ip}}", + "server": {{cfg.server.mode}}, + "retry_join": [ + {{#eachAlive svc.members as |member| ~}} + "{{member.sys.ip}}" {{~#unless @last}},{{/unless}} + {{/eachAlive ~}} + ], + "ports": { + "dns": {{cfg.ports.dns}}, + "http": {{cfg.ports.http}}, + "https": {{cfg.ports.https}}, + "serf_lan": {{cfg.ports.serf_lan}}, + "serf_wan": {{cfg.ports.serf_wan}}, + "server": {{cfg.ports.server}} + } +} diff --git a/test/fixtures/render/consul/consul-config.inspec.rb b/test/fixtures/render/consul/consul-config.inspec.rb new file mode 100644 index 0000000000..a98374a620 --- /dev/null +++ b/test/fixtures/render/consul/consul-config.inspec.rb @@ -0,0 +1,13 @@ +describe json('result/consul_config.json') do + its('datacenter') { should eq 'IN_OVERRIDE_JSON' } + its('data_dir') { should eq 'IN_DEFAULT_TOML' } + its('log_level') { should eq 'IN_USER_TOML' } + its('bind_addr') { should eq '9.9.9.9' } + its('server') { should eq true } + + its(['retry_join', 0]) { should eq '1.1.1.1' } + its(['retry_join', 1]) { should eq '2.2.2.2' } + its(['retry_join', 2]) { should eq '3.3.3.3' } + its(['ports','dns']) { should eq 6666 } + its(['ports','server']) { should eq 9999 } +end diff --git a/test/fixtures/render/consul/default.toml b/test/fixtures/render/consul/default.toml new file mode 100644 index 0000000000..5b1c806317 --- /dev/null +++ b/test/fixtures/render/consul/default.toml @@ -0,0 +1,31 @@ +# If you would like the web gui on the agent +website = true + +# The options for consul are available here +# https://www.consul.io/docs/agent/options.html +[bootstrap] +expect = "3" + +[server] +data-dir = "IN_DEFAULT_TOML" +datacenter = "dc1" +loglevel = "INFO" +# Revert back to the Legacy UI +legacy_ui = false +# switch this to false you want to start in DEVMODE +# https://www.consul.io/docs/guides/bootstrapping.html +mode = true + +[ports] +# The DNS server, -1 to disable +dns = 8600 +# The HTTP API, -1 to disable +http = 8500 +# The HTTPS API, -1 to disable +https = -1 +# The Serf LAN port +serf_lan = 8301 +# The Serf WAN port +serf_wan = 8302 +# Server RPC address +server = 8300 diff --git a/test/fixtures/render/consul/hooks/run b/test/fixtures/render/consul/hooks/run new file mode 100644 index 0000000000..f880abc2bb --- /dev/null +++ b/test/fixtures/render/consul/hooks/run @@ -0,0 +1,13 @@ +#!/bin/sh + +exec 2>&1 + +SERVERMODE={{cfg.server.mode}} +export CONSUL_UI_LEGACY={{cfg.server.legacy_ui}} + +CONSUL_OPTS="-dev" +if [ "$SERVERMODE" = true ]; then + CONSUL_OPTS="{{~#if cfg.website}} -ui {{~/if}} -server -bootstrap-expect {{cfg.bootstrap.expect}} -config-file={{pkg.svc_config_path}}/basic_config.json" +fi + +exec consul agent ${CONSUL_OPTS} diff --git a/test/fixtures/render/consul/override.json b/test/fixtures/render/consul/override.json new file mode 100644 index 0000000000..df76993495 --- /dev/null +++ b/test/fixtures/render/consul/override.json @@ -0,0 +1,26 @@ +{ + "cfg": { + "website": false, + "server": { + "datacenter": "IN_OVERRIDE_JSON" + }, + "ports": { + "dns": 6666, + "http": 6667, + "https": 6668 + } + }, + "sys": { + "ip": "9.9.9.9" + }, + "svc": { + "members": [ + { "alive": true, "sys": { "ip": "1.1.1.1" }}, + { "alive": true, "sys": { "ip": "2.2.2.2" }}, + { "alive": true, "sys": { "ip": "3.3.3.3" }} + ] + }, + "pkg": { + "svc_config_path": "/home/foo/bar" + } +} diff --git a/test/fixtures/render/consul/user.toml b/test/fixtures/render/consul/user.toml new file mode 100644 index 0000000000..24c91d0014 --- /dev/null +++ b/test/fixtures/render/consul/user.toml @@ -0,0 +1,6 @@ +[server] +loglevel = "IN_USER_TOML" + +[ports] +serf_lan = 8888 +server = 9999 diff --git a/test/fixtures/render/error/consul_config.json b/test/fixtures/render/error/consul_config.json new file mode 100644 index 0000000000..71042ad7eb --- /dev/null +++ b/test/fixtures/render/error/consul_config.json @@ -0,0 +1,21 @@ +{ + "datacenter": "{{cfg.server.datacenter", + "data_dir": "{{cfg.server.data-dir}}", + "log_level": "{{cfg.server.loglevel}}", + "bind_addr": "{{sys.ip}}", + "client_addr": "{{sys.ip}}", + "server": {{cfg.server.mode}}, + "retry_join": [ + {{#eachAlive svc.members as |member| ~}} + "{{member.sys.ip}}" {{~#unless @last}},{{/unless}} + {{/eachAlive ~}} + ], + "ports": { + "dns": {{cfg.ports.dns}}, + "http": {{cfg.ports.http}}, + "https": {{cfg.ports.https}}, + "serf_lan": {{cfg.ports.serf_lan}}, + "serf_wan": {{cfg.ports.serf_wan}}, + "server": {{cfg.ports.server}} + } +} diff --git a/test/fixtures/render/error/default.toml b/test/fixtures/render/error/default.toml new file mode 100644 index 0000000000..19e50fa74b --- /dev/null +++ b/test/fixtures/render/error/default.toml @@ -0,0 +1,31 @@ +# If you would like the web gui on the agent +website = true + +# The options for consul are available here +# https://www.consul.io/docs/agent/options.html +[[bootstrap] +expect = "3" + +[server] +data-dir = "IN_DEFAULT_TOML" +datacenter = "dc1" +loglevel = "INFO" +# Revert back to the Legacy UI +legacy_ui = false +# switch this to false you want to start in DEVMODE +# https://www.consul.io/docs/guides/bootstrapping.html +mode = true + +[ports] +# The DNS server, -1 to disable +dns = 8600 +# The HTTP API, -1 to disable +http = 8500 +# The HTTPS API, -1 to disable +https = -1 +# The Serf LAN port +serf_lan = 8301 +# The Serf WAN port +serf_wan = 8302 +# Server RPC address +server = 8300 diff --git a/test/fixtures/render/error/override.json b/test/fixtures/render/error/override.json new file mode 100644 index 0000000000..512e04a361 --- /dev/null +++ b/test/fixtures/render/error/override.json @@ -0,0 +1,26 @@ +{ + "cfg": { + "website": false + "server": { + "datacenter": "IN_OVERRIDE_JSON" + }, + "ports": { + "dns": 6666, + "http": 6667, + "https": 6668 + } + }, + "sys": { + "ip": "9.9.9.9" + }, + "svc": { + "members": [ + { "alive": true, "sys": { "ip": "1.1.1.1" }}, + { "alive": true, "sys": { "ip": "2.2.2.2" }}, + { "alive": true, "sys": { "ip": "3.3.3.3" }} + ] + }, + "pkg": { + "svc_config_path": "/home/foo/bar" + } +} diff --git a/test/shellcheck.sh b/test/shellcheck.sh index 8910c3ccb2..0b3295f88c 100755 --- a/test/shellcheck.sh +++ b/test/shellcheck.sh @@ -32,6 +32,8 @@ find . -type f \ -and \! -path "./test/integration/test_helper/bats-assert/*" \ -and \! -path "./test/integration/test_helper/bats-file/*" \ -and \! -path "./test/integration/test_helper/bats-support/*" \ + -and \! -path "./test/fixtures/render/consul/hooks/run" \ + -and \! -path "./test/fixtures/render/error/*" \ -print \ | xargs shellcheck --external-sources --exclude=1090,1091,1117,2148,2034