Support logging of env by cmd, shell and daemon_cmd

also, fixed the following issues:
* quoting args if they contain $ and ' sings. We should check more chars
in the future
* daemon_cmd didn't support marking args as secret even though this
info was present in typespec
* logs for shell commands now always wrapped in `sh -c "command"`

lib/owl/daemon.ex

@@ -19,8 +19,20 @@ defmodule Owl.Daemon do
   def init(args) do
     command = Keyword.fetch!(args, :command)
     command_args = Keyword.fetch!(args, :args)
+    command_env = Keyword.get(args, :env, [])
     executable = System.find_executable(command)
-    Owl.System.Helpers.log_shell_command(command, command_args)
+    Owl.System.Helpers.log_cmd(command_env, command, command_args)
+
+    command_env =
+      command_env
+      |> Owl.System.Helpers.normalize_env()
+      # https://github.com/elixir-lang/elixir/blob/a64d42f5d3cb6c32752af9d3312897e8cd5bb7ec/lib/elixir/lib/system.ex#L1099
+      |> Enum.map(fn
+        {k, nil} -> {String.to_charlist(k), false}
+        {k, v} -> {String.to_charlist(k), String.to_charlist(v)}
+      end)
+
+    command_args = Owl.System.Helpers.normalize_cmd_args(command_args)
 
     {handle_data_state, handle_data_callback} =
       case Keyword.get(args, :handle_data) do
@@ -35,7 +47,8 @@ defmodule Owl.Daemon do
         :binary,
         :exit_status,
         :stderr_to_stdout,
-        args: command_args
+        args: command_args,
+        env: command_env
       ])
 
     prefix =

lib/owl/system.ex

@@ -16,6 +16,7 @@ defmodule Owl.System do
   * `:ready_check` - a function which checks the content of the messages produced by `command` before writing to `device`.
     If the function is set, then the execution of the `operation` will be blocked until `ready_check` returns `true`.
     By default this check is absent and `operation` is invoked immediately without awaiting any message.
+  * `:env` - a list of tuples containing environment key-value. The behaviour is similar to the option described in `cmd/3`
 
   ## Example
 
@@ -52,10 +53,14 @@ defmodule Owl.System do
       # Forwarding from [::1]:5432 -> 5432
       :ok
   """
-  @spec daemon_cmd(binary(), [binary() | {:secret, binary()}], (() -> result),
+  @spec daemon_cmd(
+          binary(),
+          [binary() | {:secret, binary()} | [binary() | {:secret, binary()}]],
+          (() -> result),
           prefix: Owl.Data.t(),
           device: IO.device(),
-          ready_check: (String.t() -> boolean())
+          ready_check: (String.t() -> boolean()),
+          env: [{binary(), binary() | {:secret, binary()} | nil}]
         ) :: result
         when result: any()
   def daemon_cmd(command, args, operation, options \\ []) when is_function(operation, 0) do
@@ -90,7 +95,7 @@ defmodule Owl.System do
         [
           command: command,
           args: args
-        ] ++ handle_data_opts ++ Keyword.take(options, [:prefix, :device])
+        ] ++ handle_data_opts ++ Keyword.take(options, [:prefix, :device, :env])
       )
 
     Process.link(pid)
@@ -105,64 +110,75 @@ defmodule Owl.System do
   end
 
   @doc """
-  A wrapper around `System.cmd/3` which additionally logs executed `command` and `args`.
+  A wrapper around `System.cmd/3` which additionally logs executed `command`, `args` and `env`.
 
   If URL is found in logged message, then password in it is masked with asterisks.
-  Additionally, it is possible to explicitly mark a whole argument as secret.
+  Additionally, it is possible to explicitly mark env values and arguments as secret for safe logging.
+  See examples for details.
 
   ## Examples
 
       > Owl.System.cmd("echo", ["test"])
       # 10:25:34.252 [debug] $ echo test
       {"test\\n", 0}
 
+      # marking an argument as secret
       > Owl.System.cmd("echo", ["hello", secret: "world"])
       # 10:25:40.516 [debug] $ echo hello ********
       {"hello world\\n", 0}
 
+      # marking a part of an argument as secret
+      > Owl.System.cmd("echo", ["hello", ["--password=", {:secret, "world"}]])
+      # 10:25:40.516 [debug] $ echo hello --password=********
+      {"hello --password=world\\n", 0}
+
+      # marking a env as secret
+      > Owl.System.cmd("echo", ["hello", "world"], env: [{"PASSWORD", {:secret, "mypassword"}}])
+      # 10:25:40.516 [debug] $ PASSWORD=******** sh -c "echo hello world"
+      {"hello world\\n", 0}
+
       > Owl.System.cmd("psql", ["postgresql://postgres:postgres@127.0.0.1:5432", "-tAc", "SELECT 1;"])
       # 10:25:50.947 [debug] $ psql postgresql://postgres:********@127.0.0.1:5432 -tAc 'SELECT 1;'
       {"1\\n", 0}
 
   """
-  @spec cmd(binary(), [binary() | {:secret, binary()}], keyword()) ::
+  @spec cmd(
+          binary(),
+          [binary() | {:secret, binary()} | [binary() | {:secret, binary()}]],
+          keyword()
+        ) ::
           {Collectable.t(), exit_status :: non_neg_integer()}
   def cmd(command, args, opts \\ []) when is_binary(command) and is_list(args) do
-    Owl.System.Helpers.log_shell_command(command, args)
-
-    args =
-      Enum.map(
-        args,
-        fn
-          {:secret, arg} when is_binary(arg) -> arg
-          arg when is_binary(arg) -> arg
-        end
-      )
+    Owl.System.Helpers.log_cmd(Keyword.get(opts, :env, []), command, args)
 
-    System.cmd(command, args, opts)
+    System.cmd(
+      command,
+      Owl.System.Helpers.normalize_cmd_args(args),
+      Owl.System.Helpers.normalize_env_option(opts)
+    )
   end
 
   @doc """
   A wrapper around `System.shell/2` which additionally logs executed `command`.
 
-  Similarly to `cmd/3`, it automatically hides password in found URLs.
+  Similarly to `cmd/3`, it automatically hides password in found URLs and allows manual hiding of env values.
 
   ## Examples
 
       > Owl.System.shell("echo hello world")
-      # 22:36:01.440 [debug] $ echo hello world
+      # 22:36:01.440 [debug] $ sh -c "echo hello world"
       {"hello world\\n", 0}
 
       > Owl.System.shell("echo postgresql://postgres:postgres@127.0.0.1:5432")
-      # 22:36:51.797 [debug] $ echo postgresql://postgres:********@127.0.0.1:5432
+      # 22:36:51.797 [debug] $ sh -c "echo postgresql://postgres:********@127.0.0.1:5432"
       {"postgresql://postgres:postgres@127.0.0.1:5432\\n", 0}
   """
   @spec shell(
           binary(),
           keyword()
         ) :: {Collectable.t(), exit_status :: non_neg_integer()}
   def shell(command, opts \\ []) when is_binary(command) do
-    Owl.System.Helpers.log_shell_command(command)
-    System.shell(command, opts)
+    Owl.System.Helpers.log_shell(Keyword.get(opts, :env, []), command)
+    System.shell(command, Owl.System.Helpers.normalize_env_option(opts))
   end
 end

lib/owl/system/helpers.ex

@@ -4,13 +4,44 @@ defmodule Owl.System.Helpers do
 
   @secret_placeholder "********"
 
-  def log_shell_command(command) do
-    command = sanitize_passwords_in_urls(command)
+  def normalize_env_option(opts) do
+    Keyword.update(opts, :env, [], &normalize_env/1)
+  end
 
-    Logger.debug("$ #{command}")
+  def normalize_cmd_args(args) do
+    Enum.map(
+      args,
+      fn
+        {:secret, arg} when is_binary(arg) ->
+          arg
+
+        arg when is_binary(arg) ->
+          arg
+
+        parts when is_list(parts) ->
+          Enum.map_join(parts, fn
+            part when is_binary(part) -> part
+            {:secret, part} when is_binary(part) -> part
+          end)
+      end
+    )
+  end
+
+  def normalize_env(env) do
+    Enum.map(
+      env,
+      fn
+        {variable, {:secret, value}} -> {variable, value}
+        item -> item
+      end
+    )
   end
 
-  def log_shell_command(command, args) do
+  def log_shell(env, command) do
+    log_command(env, command, :force)
+  end
+
+  def log_cmd(env, command, args) do
     command =
       case args do
         [] ->
@@ -23,31 +54,75 @@ defmodule Owl.System.Helpers do
                 @secret_placeholder
 
               arg ->
-                if String.contains?(arg, [" ", ";"]) do
-                  "'" <> String.replace(arg, "'", "'\\''") <> "'"
-                else
-                  arg
-                end
+                arg
+                |> List.wrap()
+                |> Enum.map_join(fn
+                  {:secret, _arg} -> @secret_placeholder
+                  arg -> arg
+                end)
+                |> maybe_quote_arg()
             end)
 
           "#{command} #{args}"
       end
 
-    log_shell_command(command)
+    log_command(env, command, :auto)
   end
 
-  defp sanitize_passwords_in_urls(text) do
-    Regex.replace(~r/\w+:\/\/[^ ]+/, text, fn value ->
-      uri = URI.parse(value)
+  defp log_command(env, command, shell) do
+    command =
+      if shell == :force or (shell == :auto and not Enum.empty?(env)) do
+        wrap_command_to_shell(command)
+      else
+        command
+      end
 
-      case uri.userinfo do
-        nil ->
-          value
+    command =
+      command
+      |> prepend_env(env)
+      |> sanitize_passwords_in_urls()
 
-        userinfo ->
-          [username, _password] = String.split(userinfo, ":")
-          to_string(%{uri | userinfo: "#{username}:#{@secret_placeholder}"})
-      end
-    end)
+    Logger.debug("$ #{command}")
+  end
+
+  defp wrap_command_to_shell(command) do
+    case :os.type() do
+      {:unix, _} ->
+        command = command |> String.replace("\"", "\\\"") |> String.replace("$", "\\$")
+        "sh -c \"#{command}\""
+
+      {:win32, _osname} ->
+        raise "windows is not supported yet"
+    end
+  end
+
+  defp prepend_env(command, []), do: command
+
+  defp prepend_env(command, env) do
+    env =
+      Enum.map_join(env, " ", fn
+        {variable, {:secret, _value}} ->
+          "#{variable}=#{@secret_placeholder}"
+
+        {variable, nil} ->
+          "#{variable}="
+
+        {variable, value} ->
+          "#{variable}=#{maybe_quote_arg(value)}"
+      end)
+
+    "#{env} #{command}"
+  end
+
+  defp maybe_quote_arg(arg) do
+    if String.contains?(arg, [" ", ";", "$", "'"]) do
+      "'" <> String.replace(arg, "'", "'\\''") <> "'"
+    else
+      arg
+    end
+  end
+
+  defp sanitize_passwords_in_urls(text) do
+    Regex.replace(~r/(\w+:\/\/[^:]+:)(.+?)@/, text, "\\1#{@secret_placeholder}@")
   end
 end

test/owl/system_test.exs

@@ -27,6 +27,22 @@ defmodule Owl.SystemTest do
       assert count_active_children.() == children_number
     end
 
+    test "env and args with :secret extension" do
+      log =
+        capture_log(fn ->
+          Owl.System.daemon_cmd(
+            "sleep",
+            [{:secret, "5"}],
+            fn ->
+              Process.sleep(10)
+            end,
+            env: [{"PASSWORD", {:secret, "PASSWORD"}}, {"USERNAME", nil}]
+          )
+        end)
+
+      assert log =~ "$ PASSWORD=******** USERNAME= sh -c \"sleep ********\"\n"
+    end
+
     test "successful run with :ready_check option" do
       sh_script = """
       sleep 1
@@ -159,11 +175,35 @@ defmodule Owl.SystemTest do
                "SELECT 1;"
              ])
            end) =~ "$ echo postgresql://postgres:********@127.0.0.1:5432 -tAc 'SELECT 1;'\n"
+
+    assert capture_log(fn ->
+             Owl.System.cmd(
+               "echo",
+               ["hello world", ["--password=", {:secret, "mypassword"}]],
+               env: [
+                 {"GREETING", "hello world"},
+                 {"SINGLE_WORD", "single"},
+                 {"PASSWORD", {:secret, "pass"}}
+               ]
+             )
+           end) =~
+             "$ GREETING='hello world' SINGLE_WORD=single PASSWORD=******** sh -c \"echo 'hello world' --password=********\"\n"
   end
 
   test inspect(&Owl.System.shell/2) do
     assert capture_log(fn ->
              Owl.System.shell("echo hello world")
-           end) =~ "$ echo hello world\n"
+           end) =~ "$ sh -c \"echo hello world\"\n"
+
+    assert capture_log(fn ->
+             Owl.System.shell("echo hello world",
+               env: [
+                 {"GREETING", "hello world"},
+                 {"SINGLE_WORD", "single"},
+                 {"PASSWORD", {:secret, "pass"}}
+               ]
+             )
+           end) =~
+             "$ GREETING='hello world' SINGLE_WORD=single PASSWORD=******** sh -c \"echo hello world\"\n"
   end
 end