Merge pull request #6114 from qubitrenegade/qbr/add-plan-render-command

Attempt to implement template debugging tool

.gitignore

@@ -12,6 +12,7 @@
 dump.rdb
 log/
 results/
+result
 tags
 terraform.tfstate*
 test/builder-api/built/*

components/hab/src/cli.rs

@@ -580,6 +580,19 @@ pub fn get() -> App<'static, 'static> {
                 (@arg SCAFFOLDING: --scaffolding -s +takes_value
                     "Specify explicit Scaffolding for your app (ex: node, ruby)")
             )
+            (@subcommand render =>
+                (about: "Renders plan config files")
+                (aliases: &["r", "re", "ren", "rend", "rende"])
+                (@arg TEMPLATE_PATH: +required {file_exists} "Path to config to render")
+                (@arg DEFAULT_TOML: -d --("default-toml") +takes_value default_value("./default.toml") "Path to default.toml")
+                (@arg USER_TOML: -u --("user-toml") +takes_value "Path to user.toml, defaults to none")
+                (@arg MOCK_DATA: -m --("mock-data") +takes_value "Path to json file with mock data for template, defaults to none")
+                (@arg PRINT: -p --("print") "Prints config to STDOUT")
+                (@arg RENDER_DIR: -r --("render-dir") +takes_value default_value("./results") "Path to render templates")
+                (@arg NO_RENDER: -n --("no-render") "Don't write anything to disk, ignores --render-dir")
+                (@arg QUIET: -q --("no-verbose") --quiet
+                    "Don't print any helper messages.  When used with `--print` will only print config file")
+            )
         )
         (@subcommand ring =>
             (about: "Commands relating to Habitat rings")

components/hab/src/command/plan/mod.rs

@@ -13,3 +13,4 @@
 // limitations under the License.
 
 pub mod init;
+pub mod render;

components/hab/src/command/plan/render.rs

@@ -0,0 +1,170 @@
+// Copyright (c) 2019 Chef Software Inc. and/or applicable contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde_json::{self,
+                 json,
+                 Value as Json};
+use std::{fs::{create_dir_all,
+               read_to_string,
+               File},
+          io::Write,
+          path::Path};
+use toml::Value;
+
+use crate::{common::{templating::TemplateRenderer,
+                     ui::{Status,
+                          UIWriter,
+                          UI}},
+            error::Result};
+
+pub fn start(ui: &mut UI,
+             template_path: &Path,
+             default_toml_path: &Path,
+             user_toml_path: Option<&Path>,
+             mock_data_path: Option<&Path>,
+             print: bool,
+             render: bool,
+             render_dir: &Path,
+             quiet: bool)
+             -> Result<()> {
+    // Strip the file name out of our passed template
+    let file_name = Path::new(template_path.file_name().expect("valid template file"));
+
+    if !quiet {
+        ui.begin(format!("Rendering: {} into: {} as: {}",
+                         template_path.display(),
+                         render_dir.display(),
+                         file_name.display()))?;
+        ui.br()?;
+    }
+
+    // read our template from file
+    let template = read_to_string(&template_path)?;
+
+    // create a "data" json struct
+    let mut data = json!({});
+
+    if !quiet {
+        // import default.toml values, convert to JSON
+        ui.begin(format!("Importing default.toml: {}", &default_toml_path.display()))?;
+    }
+
+    // we should always have a default.toml, would be nice to "autodiscover" based on package name,
+    // for now assume we're working in the plan dir if --default-toml not passed
+    let default_toml = read_to_string(&default_toml_path)?;
+
+    // merge default into data struct
+    merge(&mut data, toml_to_json(&default_toml)?);
+
+    // import default.toml values, convert to JSON
+    let user_toml = match user_toml_path {
+        Some(path) => {
+            if !quiet {
+                // print helper message, maybe only print if '--verbose'? how?
+                ui.begin(format!("Importing user.toml: {}", path.display()))?;
+            }
+            read_to_string(path)?
+        }
+        None => String::new(),
+    };
+    // merge default into data struct
+    merge(&mut data, toml_to_json(&user_toml)?);
+
+    // read mock data if provided
+    let mock_data = match mock_data_path {
+        Some(path) => {
+            if !quiet {
+                // print helper message, maybe only print if '--verbose'? how?
+                ui.begin(format!("Importing override file: {}", path.display()))?;
+            }
+            read_to_string(path)?
+        }
+        // return an empty json block if '--mock-data' isn't defined.
+        // this allows us to merge an empty JSON block
+        None => "{}".to_string(),
+    };
+    // merge mock data into data
+    merge(&mut data, serde_json::from_str(&mock_data)?);
+
+    // create a template renderer
+    let mut renderer = TemplateRenderer::new();
+    // register our template
+    renderer.register_template_string(&template, &template)
+            .expect("Could not register template content");
+    // render our JSON override in our template.
+    let rendered_template = renderer.render(&template, &data)?;
+
+    if print {
+        if !quiet {
+            ui.br()?;
+            ui.warn(format!("###======== Rendered template: {}",
+                            &template_path.display()))?;
+        }
+
+        println!("{}", rendered_template);
+
+        if !quiet {
+            ui.warn(format!("========### End rendered template: {}",
+                            &template_path.display()))?;
+        }
+    }
+
+    if render {
+        // Render our template file
+        create_with_template(ui, &render_dir, &file_name, &rendered_template, quiet)?;
+    }
+
+    if !quiet {
+        ui.br()?;
+    }
+    Ok(())
+}
+
+fn toml_to_json(cfg: &str) -> Result<Json> {
+    let toml_value = cfg.parse::<Value>()?;
+    let toml_string = serde_json::to_string(&toml_value)?;
+    let json = serde_json::from_str(&format!(r#"{{ "cfg": {} }}"#, &toml_string))?;
+    Ok(json)
+}
+
+// merge two Json structs
+fn merge(a: &mut Json, b: Json) {
+    if let Json::Object(a_map) = a {
+        if let Json::Object(b_map) = b {
+            for (k, v) in b_map {
+                merge(a_map.entry(k).or_insert(Json::Null), v);
+            }
+            return;
+        }
+    }
+    *a = b;
+}
+
+fn create_with_template(ui: &mut UI,
+                        render_dir: &std::path::Path,
+                        file_name: &std::path::Path,
+                        template: &str,
+                        quiet: bool)
+                        -> Result<()> {
+    let path = Path::new(&render_dir).join(&file_name);
+    if !quiet {
+        ui.status(Status::Creating, format!("file: {}", path.display()))?;
+    }
+
+    create_dir_all(render_dir)?;
+
+    // Write file to disk
+    File::create(path).and_then(|mut file| file.write(template.as_bytes()))?;
+    Ok(())
+}

components/hab/src/error.rs

@@ -62,6 +62,7 @@ pub enum Error {
     JobGroupPromoteOrDemote(api_client::Error, bool /* promote */),
     JobGroupCancel(api_client::Error),
     JobGroupPromoteOrDemoteUnprocessable(bool /* promote */),
+    JsonErr(serde_json::Error),
     NameLookup,
     NetErr(net::NetErr),
     PackageArchiveMalformed(String),
@@ -156,6 +157,7 @@ impl fmt::Display for Error {
                         if promote { "promote" } else { "demote" },
                         e)
             }
+            Error::JsonErr(ref e) => e.to_string(),
             Error::JobGroupCancel(ref e) => format!("Failed to cancel job group: {:?}", e),
             Error::NameLookup => "Error resolving a name or IP address".to_string(),
             Error::NetErr(ref e) => e.to_string(),
@@ -238,6 +240,7 @@ impl error::Error for Error {
             Error::ProvidesError(_) => {
                 "Can't find a package that provides the given search parameter"
             }
+            Error::JsonErr(ref err) => err.description(),
             Error::RemoteSupResolutionError(_, ref err) => err.description(),
             Error::RootRequired => {
                 "Root or administrator permissions required to complete operation"
@@ -300,3 +303,7 @@ impl From<SrvClientError> for Error {
 impl From<net::NetErr> for Error {
     fn from(err: net::NetErr) -> Self { Error::NetErr(err) }
 }
+
+impl From<serde_json::Error> for Error {
+    fn from(err: serde_json::Error) -> Self { Error::JsonErr(err) }
+}

components/hab/src/lib.rs

@@ -36,6 +36,8 @@ extern crate log;
 #[macro_use]
 extern crate serde_derive;
 
+extern crate serde_json;
+
 #[cfg(windows)]
 extern crate widestring;
 #[cfg(windows)]

components/hab/src/main.rs

@@ -280,6 +280,7 @@ fn start(ui: &mut UI) -> Result<()> {
         ("plan", Some(matches)) => {
             match matches.subcommand() {
                 ("init", Some(m)) => sub_plan_init(ui, m)?,
+                ("render", Some(m)) => sub_plan_render(ui, m)?,
                 _ => unreachable!(),
             }
         }
@@ -702,6 +703,32 @@ fn sub_plan_init(ui: &mut UI, m: &ArgMatches<'_>) -> Result<()> {
                                name)
 }
 
+fn sub_plan_render(ui: &mut UI, m: &ArgMatches<'_>) -> Result<()> {
+    let template_path = Path::new(m.value_of("TEMPLATE_PATH").unwrap());
+
+    let default_toml_path = Path::new(m.value_of("DEFAULT_TOML").unwrap());
+
+    let user_toml_path = m.value_of("USER_TOML").map(Path::new);
+
+    let mock_data_path = m.value_of("MOCK_DATA").map(Path::new);
+
+    let print = m.is_present("PRINT");
+    let render = !m.is_present("NO_RENDER");
+    let quiet = m.is_present("QUIET");
+
+    let render_dir = Path::new(m.value_of("RENDER_DIR").unwrap());
+
+    command::plan::render::start(ui,
+                                 template_path,
+                                 default_toml_path,
+                                 user_toml_path,
+                                 mock_data_path,
+                                 print,
+                                 render,
+                                 render_dir,
+                                 quiet)
+}
+
 fn sub_pkg_install(ui: &mut UI, m: &ArgMatches<'_>) -> Result<()> {
     let url = bldr_url_from_matches(&m)?;
     let channel = channel_from_matches_or_default(m);

test/fixtures/render/README.md

@@ -0,0 +1,80 @@
+# About
+
+These files are for testing `hab plan render` command.
+
+see `hab plan render --help` for full usage instructions.
+
+# Usage
+
+Try:
+
+
+```
+cargo run -p hab plan render ./test/fixtures/render/consul/config/consul_config.json \
+  --default-toml ./test/fixtures/render/consul/default.toml \
+  --user-toml ./test/fixtures/render/consul/user.toml \
+  --mock-data ./test/fixtures/render/consul/override.json \
+  --render-dir ~/result/config \
+  --print
+```
+
+```
+cargo run -p hab plan render ./test/fixtures/render/consul/config/consul_config.json \
+  --default-toml ./test/fixtures/render/consul/default.toml \
+  --render-dir ~/result/config \
+  --print
+```
+
+or
+
+```
+cargo run -p hab plan render ./test/fixtures/render/consul/hooks/run \
+  --default-toml ./test/fixtures/render/consul/default.toml \
+  --user-toml ./test/fixtures/render/consul/user.toml \
+  --mock-data ./test/fixtures/render/consul/override.json \
+  --render-dir ~/result/hooks \
+  --print
+```
+
+# Example output
+
+* `consul/config/basic_config.json` render:
+
+```
+{
+  "datacenter": "IN_OVERRIDE_JSON",
+  "data_dir": "IN_DEFAULT_TOML",
+  "log_level": "IN_USER_TOML",
+  "bind_addr": "9.9.9.9",
+  "client_addr": "9.9.9.9",
+  "server": true,
+  "retry_join": [
+  ],
+  "ports": {
+    "dns": 6666,
+    "http": 6667,
+    "https": 6668,
+    "serf_lan": 8888,
+    "serf_wan": 8302,
+    "server": 9999
+  }
+}
+```
+
+* `consul/hook/run` render:
+
+```
+#!/bin/sh
+
+exec 2>&1
+
+SERVERMODE=true
+export CONSUL_UI_LEGACY=false
+
+CONSUL_OPTS="-dev"
+if [ "$SERVERMODE" = true ]; then
+  CONSUL_OPTS=" -ui -server -bootstrap-expect 3 -config-file=/basic_config.json"
+fi
+
+exec consul agent ${CONSUL_OPTS}
+```

test/fixtures/render/consul/config/consul_config.json

@@ -0,0 +1,21 @@
+{
+  "datacenter": "{{cfg.server.datacenter}}",
+  "data_dir": "{{cfg.server.data-dir}}",
+  "log_level": "{{cfg.server.loglevel}}",
+  "bind_addr": "{{sys.ip}}",
+  "client_addr": "{{sys.ip}}",
+  "server": {{cfg.server.mode}},
+  "retry_join": [
+  {{#eachAlive svc.members as |member| ~}}
+    "{{member.sys.ip}}" {{~#unless @last}},{{/unless}}
+  {{/eachAlive ~}}
+  ],
+  "ports": {
+    "dns": {{cfg.ports.dns}},
+    "http": {{cfg.ports.http}},
+    "https": {{cfg.ports.https}},
+    "serf_lan": {{cfg.ports.serf_lan}},
+    "serf_wan": {{cfg.ports.serf_wan}},
+    "server": {{cfg.ports.server}}
+  }
+}