Skip to content

Commit

Permalink
Support configuring the page size for listing catalog entries (#151)
Browse files Browse the repository at this point in the history 
In cases where listing entries can be expensive, it's useful to be able
to "tune down" the page size when we list catalog entries. This pull
request introduces a `--catalog-entries-api-page-size` flag (and a
`CATALOG_ENTRIES_API_PAGE_SIZE` environment variable) that can be used
to customise the page size for this API call.
  • Loading branch information
@samstarling
samstarling authored Nov 22, 2024
1 parent 39c552a commit ac31266
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 20 deletions.
31 changes: 18 additions & 13 deletions cmd/catalog-importer/cmd/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ import (
)

type SyncOptions struct {
ConfigFile string
APIEndpoint string
APIKey string
Targets []string
SampleLength int
DryRun bool
Prune bool
AllowDeleteAll bool
SourceRepoUrl string
ConfigFile string
APIEndpoint string
APIKey string
Targets []string
SampleLength int
DryRun bool
Prune bool
AllowDeleteAll bool
SourceRepoUrl string
CatalogEntriesAPIPageSize int
}

func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions {
Expand Down Expand Up @@ -59,6 +60,10 @@ func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions {
BoolVar(&opt.Prune)
cmd.Flag("allow-delete-all", "Allow removing all entries from a catalog entry").
BoolVar(&opt.AllowDeleteAll)
cmd.Flag("catalog-entries-api-page-size", "The page size to use when listing catalog entries from the API").
Envar("CATALOG_ENTRIES_API_PAGE_SIZE").
Default("250").
IntVar(&opt.CatalogEntriesAPIPageSize)

return opt
}
Expand Down Expand Up @@ -473,7 +478,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf
logger.Log("msg", "reconciling catalog entries", "output", outputType.TypeName)
catalogType := catalogTypesByOutput[outputType.TypeName]

err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun))
err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("outputs (type_name = '%s'): reconciling catalog entries", outputType.TypeName))
}
Expand Down Expand Up @@ -509,7 +514,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf

OUT("\n ↻ %s (enum)", enumModel.TypeName)
catalogType := catalogTypesByOutput[enumModel.TypeName]
err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun))
err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize)
if err != nil {
return errors.Wrap(err,
fmt.Sprintf("outputs (type_name = '%s'): enum for attribute (id = '%s'): %s: reconciling catalog entries",
Expand All @@ -530,7 +535,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli
}

return reconcile.EntriesClient{
GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
// We're in dry-run and this catalog type is yet to be created. We can't ask the API
// for the entries of a type that doesn't exist, so we return the type we faked from
// the dry-run create and an empty list of entries.
Expand All @@ -545,7 +550,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli
}

// We're just a normal catalog type, use the real client.
return reconcile.GetEntries(ctx, cl, catalogTypeID)
return reconcile.GetEntries(ctx, cl, catalogTypeID, pageSize)
},
Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error {
DIFF(" ", *entry, client.CatalogEntryV2{})
Expand Down
14 changes: 7 additions & 7 deletions reconcile/entries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
)

type EntriesClient struct {
GetEntries func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error)
GetEntries func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error)
Delete func(ctx context.Context, entry *client.CatalogEntryV2) error
Create func(ctx context.Context, payload client.CreateEntryRequestBody) (*client.CatalogEntryV2, error)
Update func(ctx context.Context, entry *client.CatalogEntryV2, payload client.UpdateEntryRequestBody) (*client.CatalogEntryV2, error)
Expand All @@ -26,8 +26,8 @@ type EntriesClient struct {
// actually perform updates.
func EntriesClientFromClient(cl *client.ClientWithResponses) EntriesClient {
return EntriesClient{
GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
return GetEntries(ctx, cl, catalogTypeID)
GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
return GetEntries(ctx, cl, catalogTypeID, pageSize)
},
Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error {
_, err := cl.CatalogV2DestroyEntryWithResponse(ctx, entry.Id)
Expand Down Expand Up @@ -65,7 +65,7 @@ type EntriesProgress struct {
OnUpdateProgress func()
}

func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress) error {
func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress, pageSize int) error {
logger = kitlog.With(logger,
"catalog_type_id", catalogType.Id,
"catalog_type_name", catalogType.TypeName,
Expand All @@ -77,7 +77,7 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output
}

logger.Log("msg", "listing existing entries")
catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id)
catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id, pageSize)
if err != nil {
return errors.Wrap(err, "listing entries")
}
Expand Down Expand Up @@ -307,15 +307,15 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output
}

// GetEntries paginates through all catalog entries for the given type.
func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) {
func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string, pageSize int) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) {
var (
after *string
)

for {
result, err := cl.CatalogV2ListEntriesWithResponse(ctx, &client.CatalogV2ListEntriesParams{
CatalogTypeId: catalogTypeID,
PageSize: lo.ToPtr(int64(250)),
PageSize: lo.ToPtr(int64(pageSize)),
After: after,
})
if err != nil {
Expand Down

0 comments on commit ac31266

Please sign in to comment.