-
Notifications
You must be signed in to change notification settings - Fork 43
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add new
inspect-chunks command for more detailed chunks info
Add a new `inspect-chunks` command that can provide additional information about the chunks in an index, including their compressed size (if a local store is provided). One use case is for the clients to download the server's inspect-chunks JSON and feed it into `desync info`, to calculate the precise download size of an update. Signed-off-by: Ludovico de Nittis <ludovico.denittis@collabora.com>
- Loading branch information
Showing
11 changed files
with
2,407 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package desync | ||
|
|
||
| // ChunkAdditionalInfo contains detailed information about a particular chunk. | ||
| // Some of those info, e.g. CompressedSize, are only exact for the store used when | ||
| // generating it. Because other stores could potentially use different compression levels. | ||
| type ChunkAdditionalInfo struct { | ||
| ID ChunkID `json:"id"` | ||
| UncompressedSize uint64 `json:"uncompressed_size"` | ||
| CompressedSize int64 `json:"compressed_size,omitempty"` | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "github.com/folbricht/desync" | ||
| "github.com/spf13/cobra" | ||
| "io" | ||
| "os" | ||
| ) | ||
|
|
||
| type inspectChunksOptions struct { | ||
| cmdStoreOptions | ||
| store string | ||
| } | ||
|
|
||
| func newinspectChunksCommand(ctx context.Context) *cobra.Command { | ||
| var opt inspectChunksOptions | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "inspect-chunks <index> [<output>]", | ||
| Short: "Inspect chunks from an index and an optional local store", | ||
| Long: `Prints a detailed JSON with information about chunks stored in an index file. | ||
| By using the '--store' option to provide a local store, the generated JSON will include, if | ||
| available, the chunks compressed size info from that particular store.`, | ||
| Example: ` desync inspect-chunks file.caibx | ||
| desync inspect-chunks --store /mnt/store file.caibx inspect_result.json`, | ||
| Args: cobra.RangeArgs(1, 2), | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| return runInspectChunks(ctx, opt, args) | ||
| }, | ||
| SilenceUsage: true, | ||
| } | ||
| flags := cmd.Flags() | ||
| flags.StringVarP(&opt.store, "store", "s", "", "local source store") | ||
| addStoreOptions(&opt.cmdStoreOptions, flags) | ||
| return cmd | ||
| } | ||
|
|
||
| func runInspectChunks(ctx context.Context, opt inspectChunksOptions, args []string) error { | ||
| if err := opt.cmdStoreOptions.validate(); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var ( | ||
| outFile io.Writer | ||
| err error | ||
| ) | ||
| if len(args) == 2 { | ||
| outFileName := args[1] | ||
| outFile, err = os.Create(outFileName) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } else { | ||
| outFile = stdout | ||
| } | ||
|
|
||
| // Read the input | ||
| c, err := readCaibxFile(args[0], opt.cmdStoreOptions) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var chunksInfo []desync.ChunkAdditionalInfo | ||
|
|
||
| var s desync.LocalStore | ||
| if opt.store != "" { | ||
| options, err := cfg.GetStoreOptionsFor(opt.store) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // Use compressed chunks, if available, to record their sizes | ||
| options.Uncompressed = false | ||
| s, err = desync.NewLocalStore(opt.store, options) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| for _, chunk := range c.Chunks { | ||
| var size int64 = 0 | ||
| if opt.store != "" { | ||
| size, _ = s.GetChunkSize(chunk.ID) | ||
| } | ||
|
|
||
| chunksInfo = append(chunksInfo, desync.ChunkAdditionalInfo{ | ||
| ID: chunk.ID, | ||
| UncompressedSize: chunk.Size, | ||
| CompressedSize: size, | ||
| }) | ||
| // See if we're meant to stop | ||
| select { | ||
| case <-ctx.Done(): | ||
| return nil | ||
| default: | ||
| } | ||
| } | ||
|
|
||
| return printJSON(outFile, chunksInfo) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "encoding/json" | ||
| "io/ioutil" | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/folbricht/desync" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestInspectChunksCommand(t *testing.T) { | ||
| for _, test := range []struct { | ||
| name string | ||
| args []string | ||
| expectedOutputJSON string | ||
| }{ | ||
| {"inspect the chunks info with a local store", | ||
| []string{"-s", "testdata/blob2.store", "testdata/blob2.caibx"}, | ||
| "testdata/blob2_chunks_info.json", | ||
| }, | ||
| {"run inspect with a seed that doesn't have all the compressed chunks", | ||
| []string{"-s", "testdata/blob2.cache", "testdata/blob2.caibx"}, | ||
| "testdata/blob2_chunks_info_missing.json", | ||
| }, | ||
| {"inspect the chunks info without any stores", | ||
| []string{"testdata/blob2.caibx"}, | ||
| "testdata/blob2_chunks_info_no_store.json", | ||
| }, | ||
| } { | ||
| t.Run(test.name, func(t *testing.T) { | ||
| var exp []desync.ChunkAdditionalInfo | ||
| be, err := os.ReadFile(test.expectedOutputJSON) | ||
| require.NoError(t, err) | ||
| err = json.Unmarshal(be, &exp) | ||
|
|
||
| cmd := newinspectChunksCommand(context.Background()) | ||
| cmd.SetArgs(test.args) | ||
| b := new(bytes.Buffer) | ||
|
|
||
| // Redirect the command's output | ||
| stdout = b | ||
| cmd.SetOutput(ioutil.Discard) | ||
| _, err = cmd.ExecuteC() | ||
| require.NoError(t, err) | ||
|
|
||
| // Decode the output and compare to what's expected | ||
| var got []desync.ChunkAdditionalInfo | ||
| err = json.Unmarshal(b.Bytes(), &got) | ||
| require.NoError(t, err) | ||
| require.Equal(t, exp, got) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.