From d2fb789aa7fe4a5b5dd74ff39c74b57939481d59 Mon Sep 17 00:00:00 2001 From: Yimin Zheng Date: Fri, 13 Sep 2019 06:14:46 -0400 Subject: [PATCH 1/4] Add a `cmd` type remote-state, * delegate state locking and storage to an external command. * call the command with one of 'GET', 'PUT', 'DELETE', 'LOCK', 'UNLOCK'. * pass the content of state between terraform and the command through a file. * pass the content of lock between terraform and the command through a file. * sample scripts are at: https://github.com/bzcnsh/tf_remote_state_cmd_samples --- backend/init/init.go | 2 + backend/remote-state/cmd/backend.go | 73 +++++++++++++++++ backend/remote-state/cmd/client.go | 103 ++++++++++++++++++++++++ backend/remote-state/cmd/client_test.go | 44 ++++++++++ 4 files changed, 222 insertions(+) create mode 100644 backend/remote-state/cmd/backend.go create mode 100644 backend/remote-state/cmd/client.go create mode 100644 backend/remote-state/cmd/client_test.go diff --git a/backend/init/init.go b/backend/init/init.go index 8f70a1f81e74..e6d44b94036d 100644 --- a/backend/init/init.go +++ b/backend/init/init.go @@ -15,6 +15,7 @@ import ( backendRemote "github.com/hashicorp/terraform/backend/remote" backendArtifactory "github.com/hashicorp/terraform/backend/remote-state/artifactory" backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure" + backendCmd "github.com/hashicorp/terraform/backend/remote-state/cmd" backendConsul "github.com/hashicorp/terraform/backend/remote-state/consul" backendEtcdv2 "github.com/hashicorp/terraform/backend/remote-state/etcdv2" backendEtcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3" @@ -67,6 +68,7 @@ func Init(services *disco.Disco) { "pg": func() backend.Backend { return backendPg.New() }, "s3": func() backend.Backend { return backendS3.New() }, "swift": func() backend.Backend { return backendSwift.New() }, + "cmd": func() backend.Backend { return backendCmd.New() }, // Deprecated backends. "azure": func() backend.Backend { diff --git a/backend/remote-state/cmd/backend.go b/backend/remote-state/cmd/backend.go new file mode 100644 index 000000000000..27a9f5197e33 --- /dev/null +++ b/backend/remote-state/cmd/backend.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "context" + + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/state" + "github.com/hashicorp/terraform/state/remote" +) + +func New() backend.Backend { + s := &schema.Backend{ + Schema: map[string]*schema.Schema{ + "base_command": &schema.Schema{ + Type: schema.TypeString, + Required: true, + Description: "base command, to be called with one of PUT, GET, DELETE, LOCK, or UNLOCK as the only argument", + }, + "state_transfer_file": &schema.Schema{ + Type: schema.TypeString, + Required: true, + Description: "path to the file that passes state between terraform and the base_command", + }, + "lock_transfer_file": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: "path to the file that passes lock between terraform and the base_command", + }, + }, + } + + b := &Backend{Backend: s} + b.Backend.ConfigureFunc = b.configure + return b +} + +type Backend struct { + *schema.Backend + client *CmdClient +} + +func (b *Backend) configure(ctx context.Context) error { + data := schema.FromContextBackendConfig(ctx) + + baseCmd := data.Get("base_command").(string) + statesTransferFile := data.Get("state_transfer_file").(string) + lockTransferFile := data.Get("lock_transfer_file").(string) + + b.client = &CmdClient{ + baseCmd: baseCmd, + statesTransferFile: statesTransferFile, + lockTransferFile: lockTransferFile, + } + return nil +} + +func (b *Backend) Workspaces() ([]string, error) { + return nil, backend.ErrWorkspacesNotSupported +} + +func (b *Backend) DeleteWorkspace(string) error { + return backend.ErrWorkspacesNotSupported +} + +func (b *Backend) StateMgr(name string) (state.State, error) { + if name != backend.DefaultStateName { + return nil, backend.ErrWorkspacesNotSupported + } + return &remote.State{ + Client: b.client, + }, nil +} diff --git a/backend/remote-state/cmd/client.go b/backend/remote-state/cmd/client.go new file mode 100644 index 000000000000..bb976b0d9b11 --- /dev/null +++ b/backend/remote-state/cmd/client.go @@ -0,0 +1,103 @@ +package cmd + +import ( + "bytes" + "crypto/md5" + "io/ioutil" + "os" + "os/exec" + + "github.com/hashicorp/terraform/state" + "github.com/hashicorp/terraform/state/remote" +) + +type CmdClient struct { + baseCmd string + statesTransferFile string + lockTransferFile string + lockID string + jsonLockInfo []byte +} + +func (c *CmdClient) execCommand(arg string) error { + args := []string{arg} + cmd := exec.Command(c.baseCmd, args...) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + err := cmd.Run() + return err +} + +func (c *CmdClient) Get() (*remote.Payload, error) { + if err := c.execCommand("GET"); err != nil { + return nil, err + } + + file, err := os.Open(c.statesTransferFile) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer file.Close() + output, err := ioutil.ReadAll(file) + + hash := md5.Sum(output) + payload := &remote.Payload{ + Data: output, + MD5: hash[:md5.Size], + } + + // If there was no data, then return nil + if len(payload.Data) == 0 { + return nil, nil + } + + return payload, nil +} + +func (c *CmdClient) Put(data []byte) error { + err := ioutil.WriteFile(c.statesTransferFile, data, 0644) + if err != nil { + return err + } + err = c.execCommand("PUT") + return err +} + +func (c *CmdClient) Delete() error { + err := c.execCommand("DELETE") + return err +} + +func (c *CmdClient) Unlock(id string) error { + if c.lockTransferFile == "" { + return nil + } + err := c.execCommand("UNLOCK") + return err +} + +func (c *CmdClient) Lock(info *state.LockInfo) (string, error) { + if c.lockTransferFile == "" { + return "", nil + } + c.lockID = "" + + jsonLockInfo := info.Marshal() + err := ioutil.WriteFile(c.lockTransferFile, jsonLockInfo, 0644) + if err != nil { + return "", err + } + + err = c.execCommand("LOCK") + if err != nil { + return "", err + } else { + c.lockID = info.ID + c.jsonLockInfo = jsonLockInfo + return info.ID, nil + } +} diff --git a/backend/remote-state/cmd/client_test.go b/backend/remote-state/cmd/client_test.go new file mode 100644 index 000000000000..c7f6baae42e6 --- /dev/null +++ b/backend/remote-state/cmd/client_test.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "testing" + + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/state/remote" + "github.com/zclconf/go-cty/cty" +) + +func TestCmdClient_impl(t *testing.T) { + var _ remote.Client = new(CmdClient) +} + +func TestCmdFactory(t *testing.T) { + // This test just instantiates the client. Shouldn't make any actual + // requests nor incur any costs. + + config := make(map[string]cty.Value) + + config["base_command"] = cty.StringVal("/usr/bin/base_command") + config["state_transfer_file"] = cty.StringVal("terraform_states_file") + config["lock_transfer_file"] = cty.StringVal("terraform_lock_file") + + b := backend.TestBackendConfig(t, New(), configs.SynthBody("synth", config)) + + state, err := b.StateMgr(backend.DefaultStateName) + if err != nil { + t.Fatalf("Error for valid config: %s", err) + } + + cmdClient := state.(*remote.State).Client.(*CmdClient) + + if cmdClient.baseCmd != "/usr/bin/base_command" { + t.Fatalf("Incorrect base_command was populated") + } + if cmdClient.statesTransferFile != "terraform_states_file" { + t.Fatalf("Incorrect state_transfer_file was populated") + } + if cmdClient.lockTransferFile != "terraform_lock_file" { + t.Fatalf("Incorrect lock_transfer_file was populated") + } +} From 25914b61a12d532a4808989fb4a1ae69d7fc7976 Mon Sep 17 00:00:00 2001 From: Yimin Zheng Date: Tue, 24 Sep 2019 07:19:22 -0400 Subject: [PATCH 2/4] add logging for 'cmd' backend type --- backend/remote-state/cmd/client.go | 38 ++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/backend/remote-state/cmd/client.go b/backend/remote-state/cmd/client.go index bb976b0d9b11..2dd0ba6286d6 100644 --- a/backend/remote-state/cmd/client.go +++ b/backend/remote-state/cmd/client.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/md5" "io/ioutil" + "log" "os" "os/exec" @@ -29,8 +30,23 @@ func (c *CmdClient) execCommand(arg string) error { return err } +func logStart(action string) { + log.Printf("[TRACE] backend/remote-state/artifactory: starting %s operation", action) +} + +func logResult(action string, err *error) { + if *err == nil { + log.Printf("[TRACE] backend/remote-state/artifactory: exiting %s operation with success", action) + } else { + log.Printf("[TRACE] backend/remote-state/artifactory: exiting %s operation with failure", action) + } +} + func (c *CmdClient) Get() (*remote.Payload, error) { - if err := c.execCommand("GET"); err != nil { + var err error + logStart("Get") + defer logResult("Get", &err) + if err = c.execCommand("GET"); err != nil { return nil, err } @@ -59,7 +75,10 @@ func (c *CmdClient) Get() (*remote.Payload, error) { } func (c *CmdClient) Put(data []byte) error { - err := ioutil.WriteFile(c.statesTransferFile, data, 0644) + var err error + logStart("Put") + defer logResult("Put", &err) + err = ioutil.WriteFile(c.statesTransferFile, data, 0644) if err != nil { return err } @@ -68,26 +87,35 @@ func (c *CmdClient) Put(data []byte) error { } func (c *CmdClient) Delete() error { - err := c.execCommand("DELETE") + var err error + logStart("Delete") + defer logResult("Delete", &err) + err = c.execCommand("DELETE") return err } func (c *CmdClient) Unlock(id string) error { + var err error + logStart("Unlock") + defer logResult("Unlock", &err) if c.lockTransferFile == "" { return nil } - err := c.execCommand("UNLOCK") + err = c.execCommand("UNLOCK") return err } func (c *CmdClient) Lock(info *state.LockInfo) (string, error) { + var err error + logStart("Lock") + defer logResult("Lock", &err) if c.lockTransferFile == "" { return "", nil } c.lockID = "" jsonLockInfo := info.Marshal() - err := ioutil.WriteFile(c.lockTransferFile, jsonLockInfo, 0644) + err = ioutil.WriteFile(c.lockTransferFile, jsonLockInfo, 0644) if err != nil { return "", err } From 3aac3cef45978ad94a29e03a91ea6b9c74d95a1b Mon Sep 17 00:00:00 2001 From: Yimin Zheng Date: Tue, 24 Sep 2019 07:45:30 -0400 Subject: [PATCH 3/4] update documentation for 'cmd' backend type --- website/docs/backends/types/cmd.html.md | 55 +++++++++++++++++++++++++ website/layouts/backend-types.erb | 3 ++ 2 files changed, 58 insertions(+) create mode 100644 website/docs/backends/types/cmd.html.md diff --git a/website/docs/backends/types/cmd.html.md b/website/docs/backends/types/cmd.html.md new file mode 100644 index 000000000000..24ac53197da6 --- /dev/null +++ b/website/docs/backends/types/cmd.html.md @@ -0,0 +1,55 @@ +--- +layout: "backend-types" +page_title: "Backend Type: cmd" +sidebar_current: "docs-backends-types-standard-cmd" +description: |- + Terraform can delegate state storage and locking to an external command. +--- + +# cmd + +**Kind: Standard (with optional locking)** + +Delegates the storage and locking of state to an external command. +Uses files to pass the content of state and lock between terraform and the command. +Calls the external command with one of these subcommands: 'GET', 'PUT', 'DELETE', 'LOCK', 'UNLOCK'. +* GET: retrieve the state from storage and save its content to `state_transfer_file` +* PUT: read content of the state from `state_transfer_file` and save the state to storage +* DELETE: delete the state from storage +* LOCK (optional): read content of the lock from `lock_transfer_file` and create a lock with the content +* UNLOCK (optional): remove the lock + +```hcl +terraform { + backend "cmd" { + base_command = "./backend.sh" + state_transfer_file = "state_transfer" + lock_transfer_file = "lock_transfer" + } +} +``` + +## Example Referencing + +```hcl +data "terraform_remote_state" "foo" { + backend = "cmd" + config = { + base_command = "./backend.sh" + state_transfer_file = "state_transfer" + lock_transfer_file = "lock_transfer" + } +} +``` + +## Configuration variables + +The following configuration options / environment variables are supported: + + * `base_command` (Required) - Pass to the external command + * `state_transfer_file` (Required) - Path to the intermediate file for state + * `lock_transfer_file` (Optional) - Path to the intermediate file for lock + +## Sample external command + +[Sample implementation of external command](https://github.com/bzcnsh/tf_remote_state_cmd_samples) \ No newline at end of file diff --git a/website/layouts/backend-types.erb b/website/layouts/backend-types.erb index 315470890f34..7f8e025bb3d4 100644 --- a/website/layouts/backend-types.erb +++ b/website/layouts/backend-types.erb @@ -33,6 +33,9 @@ > azurerm + > + cmd + > consul From 84559d9d89b4cbee58b141702cf1af43dbc5cc5d Mon Sep 17 00:00:00 2001 From: Yimin Zheng Date: Tue, 24 Sep 2019 08:05:34 -0400 Subject: [PATCH 4/4] backend/remote-state/cmd, fix logging --- backend/remote-state/cmd/client.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/remote-state/cmd/client.go b/backend/remote-state/cmd/client.go index 2dd0ba6286d6..0fc62d644ebb 100644 --- a/backend/remote-state/cmd/client.go +++ b/backend/remote-state/cmd/client.go @@ -27,18 +27,19 @@ func (c *CmdClient) execCommand(arg string) error { cmd.Stdout = &out cmd.Stderr = &out err := cmd.Run() + log.Printf("[TRACE] backend/remote-state/cmd execCommand: %s\n%s", arg, out.String()) return err } func logStart(action string) { - log.Printf("[TRACE] backend/remote-state/artifactory: starting %s operation", action) + log.Printf("[TRACE] backend/remote-state/cmd: starting %s operation", action) } func logResult(action string, err *error) { if *err == nil { - log.Printf("[TRACE] backend/remote-state/artifactory: exiting %s operation with success", action) + log.Printf("[TRACE] backend/remote-state/cmd: exiting %s operation with success", action) } else { - log.Printf("[TRACE] backend/remote-state/artifactory: exiting %s operation with failure", action) + log.Printf("[TRACE] backend/remote-state/cmd: exiting %s operation with failure", action) } }