From f60cff9965f8f8926069e47bd43e2c96bd4aff7b Mon Sep 17 00:00:00 2001 From: pengli Date: Fri, 15 May 2020 14:31:20 +0800 Subject: [PATCH] Invalid length of container termination message Tekton use container termination message of kubenetes to pass additional information to the controller(by Tekton Result). Kubenates use `MaxContainerTerminationMessageLength` which is 4096 to limit the read size from disk, the redundant content will be discard. --- cmd/entrypoint/main.go | 4 ++++ docs/tasks.md | 10 +++++----- pkg/termination/write.go | 22 ++++++++++++++++++++++ pkg/termination/write_test.go | 21 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/cmd/entrypoint/main.go b/cmd/entrypoint/main.go index 8799e443471..908a5b59050 100644 --- a/cmd/entrypoint/main.go +++ b/cmd/entrypoint/main.go @@ -27,6 +27,7 @@ import ( "github.com/tektoncd/pipeline/pkg/credentials" "github.com/tektoncd/pipeline/pkg/entrypoint" + "github.com/tektoncd/pipeline/pkg/termination" ) var ( @@ -66,6 +67,9 @@ func main() { case skipError: log.Print("Skipping step because a previous step failed") os.Exit(1) + case termination.MessageLengthError: + log.Print(err.Error()) + os.Exit(1) case *exec.ExitError: // Copied from https://stackoverflow.com/questions/10385551/get-exit-code-go // This works on both Unix and Windows. Although diff --git a/docs/tasks.md b/docs/tasks.md index cb2dcaf5dc7..e5338ed7562 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -423,16 +423,16 @@ spec: The stored results can be used [at the `Task` level](./pipelines.md#configuring-execution-results-at-the-task-level) or [at the `Pipeline` level](./pipelines.md#configuring-execution-results-at-the-pipeline-level). -**Note:** The maximum size of a `Task's` results is limited by the container termination log feature of Kubernetes, +**Note:** The maximum size of a `Task's` results is limited by the container termination message feature of Kubernetes, as results are passed back to the controller via this mechanism. At present, the limit is -["2048 bytes or 80 lines, whichever is smaller."](https://kubernetes.io/docs/tasks/debug-application-cluster/determine-reason-pod-failure/#customizing-the-termination-message). -Results are written to the termination log encoded as JSON objects and Tekton uses those objects +["4096 bytes"](https://github.com/kubernetes/kubernetes/blob/96e13de777a9eb57f87889072b68ac40467209ac/pkg/kubelet/container/runtime.go#L632). +Results are written to the termination message encoded as JSON objects and Tekton uses those objects to pass additional information to the controller. As such, `Task` results are best suited for holding small amounts of data, such as commit SHAs, branch names, ephemeral namespaces, and so on. If your `Task` writes a large number of small results, you can work around this limitation -by writing each result from a separate `Step` so that each `Step` has its own termination log. -However, for results larger than a kilobyte, use a [`Workspace`](#specifying-workspaces) to +by writing each result from a separate `Step` so that each `Step` has its own termination message. +About size limitation, there is validation for it, will raise exception: `Termination message is above max allowed size 4096, caused by large task result`. Since Tekton also uses the termination message for some internal information, so the real available size will less than 4096 bytes. For results larger than a kilobyte, use a [`Workspace`](#specifying-workspaces) to shuttle data between `Tasks` within a `Pipeline`. ### Specifying `Volumes` diff --git a/pkg/termination/write.go b/pkg/termination/write.go index 2d720319f79..114798b2b99 100644 --- a/pkg/termination/write.go +++ b/pkg/termination/write.go @@ -23,6 +23,12 @@ import ( v1alpha1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" ) +const ( + // MaxContainerTerminationMessageLength is the upper bound any one container may write to + // its termination message path. Contents above this length will cause a failure. + MaxContainerTerminationMessageLength = 1024 * 4 +) + // WriteMessage writes the results to the termination message path. func WriteMessage(path string, pro []v1alpha1.PipelineResourceResult) error { // if the file at path exists, concatenate the new values otherwise create it @@ -41,6 +47,11 @@ func WriteMessage(path string, pro []v1alpha1.PipelineResourceResult) error { if err != nil { return err } + + if len(jsonOutput) > MaxContainerTerminationMessageLength { + return aboveMax + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return err @@ -55,3 +66,14 @@ func WriteMessage(path string, pro []v1alpha1.PipelineResourceResult) error { } return nil } + +//MessageLengthError indicate the length of termination message of container is beyond 4096 which is the max length read by kubenates +type MessageLengthError string + +const ( + aboveMax MessageLengthError = "Termination message is above max allowed size 4096, caused by large task result." +) + +func (e MessageLengthError) Error() string { + return string(e) +} diff --git a/pkg/termination/write_test.go b/pkg/termination/write_test.go index a4c83abe09b..acd98c65250 100644 --- a/pkg/termination/write_test.go +++ b/pkg/termination/write_test.go @@ -16,9 +16,11 @@ limitations under the License. package termination import ( + "errors" "io/ioutil" "log" "os" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -66,3 +68,22 @@ func TestExistingFile(t *testing.T) { } } } + +func TestMaxSizeFile(t *testing.T) { + value := strings.Repeat("a", 4096) + tmpFile, err := ioutil.TempFile(os.TempDir(), "tempFile") + if err != nil { + log.Fatal("Cannot create temporary file", err) + } + // Remember to clean up the file afterwards + defer os.Remove(tmpFile.Name()) + + output := []v1alpha1.PipelineResourceResult{{ + Key: "key1", + Value: value, + }} + + if err := WriteMessage(tmpFile.Name(), output); !errors.Is(err, aboveMax) { + t.Fatalf("Expected MessageLengthError, receved: %v", err) + } +}