-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[QT-437] Optimize matrix operations (#77)
This work was spawned by reports of people encountering context deadline errors when trying to list scenarios. We previously had a hard timeout of 5 seconds for listing which we have changed to use scenario level `--timeout` flag going forward. This solved the immediate problem but begged the question of why scenario listing was taking nearly that long at all. When I was investigating this I determined there were several reasons that listing was slow: * We were always decoding and evaluating all scenarios when listing, even though we only needed the reference information. If you're using `enos scenario list` to validate that the configuration for all scenarios is correct, that makes sense, but most people are probably using it to see which scenarios are available and don't necessarily want to validate all of them every time the command is invoked. * When filtering we would always fully decode all scenarios before filtering to keep the scenarios that are desired. * Every time we were comparing a matrix vector we were making a copy of it. The problem compounded exponentially as we added additional vectors and elements to the matrix. Nearly 3/4's of the CPU time we spent when listing scenarios was actually being used here for allocations, garbage collection, and sorting after copy. To improve our situation we do the following: * Add support to the decoder for shallow decoding of scenarios to the reference level. * Add filtering support to the decoder. Now we can optionally shallow decode, then filter, before fully decoding a scenario. * Rewrite our matrix vector implementation. Instead of an array alias the matrix Vector type is a struct which we use for new efficiency gains. We now always pass references to Vectors instead of passing them by value and creating copies. We also lazily keep track of a sorted copy of vectors to allow faster repeat comparisons. This has drastically reduced our allocations and garbage collection. When we combine all of these changes we improved the listing time by at least an order of magnitude. A drawback to this approach is that listing no longer validates that the full configuration in a flight plan, the scenarios and all of their variants. To handle that we introduce a new `validate` sub-command that fully decodes all matched scenarios to ensure that the flight plan is valid. We also introduce the `--profile` hidden flag that will turn on CPU and memory profiling and output the pprof files into the current directory. These profiles were useful in determining the bottleneck of the prior implementation so we'll leave them there for possible future use. * Add support for reference level scenario decoding * Add support for filtering during decoding * Rewrite matrix vector implementation to reduce allocations and GC * Use references to vectors instead of passing by value * Lazily create ordered copies of vectors when comparing * Add `--profile` hidden flag to enable CPU and memory profiling * Add `validate` sub-command for validating configuration of a flight plan. * Bump version Signed-off-by: Ryan Cragun <me@ryan.ec>
- Loading branch information
1 parent
a4ec981
commit 9a833b5
Showing
33 changed files
with
1,878 additions
and
1,236 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package acceptance | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "path/filepath" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "google.golang.org/protobuf/encoding/protojson" | ||
|
|
||
| "github.com/hashicorp/enos/proto/hashicorp/enos/v1/pb" | ||
| ) | ||
|
|
||
| func TestAcc_Cmd_Scenario_Validate(t *testing.T) { | ||
| enos := newAcceptanceRunner(t) | ||
|
|
||
| for _, test := range []struct { | ||
| dir string | ||
| out *pb.ValidateScenariosConfigurationResponse | ||
| fail bool | ||
| }{ | ||
| { | ||
| dir: "scenario_list_pass_0", | ||
| out: &pb.ValidateScenariosConfigurationResponse{}, | ||
| }, | ||
| { | ||
| dir: "scenario_list_fail_malformed", | ||
| fail: true, | ||
| }, | ||
| } { | ||
| t.Run(test.dir, func(t *testing.T) { | ||
| path, err := filepath.Abs(filepath.Join("./scenarios", test.dir)) | ||
| require.NoError(t, err) | ||
| cmd := fmt.Sprintf("scenario validate --chdir %s --format json", path) | ||
| fmt.Println(path) | ||
| out, err := enos.run(context.Background(), cmd) | ||
| if test.fail { | ||
| require.Error(t, err) | ||
| return | ||
| } | ||
|
|
||
| require.NoError(t, err) | ||
| got := &pb.ValidateScenariosConfigurationResponse{} | ||
| require.NoError(t, protojson.Unmarshal(out, got)) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "github.com/spf13/cobra" | ||
|
|
||
| "github.com/hashicorp/enos/internal/diagnostics" | ||
| "github.com/hashicorp/enos/internal/flightplan" | ||
| "github.com/hashicorp/enos/proto/hashicorp/enos/v1/pb" | ||
| ) | ||
|
|
||
| func newScenarioValidateConfigCmd() *cobra.Command { | ||
| return &cobra.Command{ | ||
| Use: "validate [FILTER]", | ||
| Short: "Validate configuration", | ||
| Long: "Validate all scenario and variant configurations", | ||
| RunE: runScenarioValidateCfgCmd, | ||
| ValidArgsFunction: scenarioNameCompletion, | ||
| } | ||
| } | ||
|
|
||
| // runScenarioValidateCfgCmd is the function that validates all flight plan configuration | ||
| func runScenarioValidateCfgCmd(cmd *cobra.Command, args []string) error { | ||
| ctx, cancel := scenarioTimeoutContext() | ||
| defer cancel() | ||
|
|
||
| sf, err := flightplan.ParseScenarioFilter(args) | ||
| if err != nil { | ||
| return ui.ShowScenariosValidateConfig(&pb.ValidateScenariosConfigurationResponse{ | ||
| Diagnostics: diagnostics.FromErr(err), | ||
| }) | ||
| } | ||
|
|
||
| res, err := rootState.enosConnection.Client.ValidateScenariosConfiguration( | ||
| ctx, &pb.ValidateScenariosConfigurationRequest{ | ||
| Workspace: &pb.Workspace{ | ||
| Flightplan: scenarioState.protoFp, | ||
| }, | ||
| Filter: sf.Proto(), | ||
| }, | ||
| ) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return ui.ShowScenariosValidateConfig(res) | ||
| } |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.