diff --git a/cmd/catalog-importer/cmd/sync.go b/cmd/catalog-importer/cmd/sync.go index 4a36aac..e784953 100644 --- a/cmd/catalog-importer/cmd/sync.go +++ b/cmd/catalog-importer/cmd/sync.go @@ -23,15 +23,16 @@ import ( ) type SyncOptions struct { - ConfigFile string - APIEndpoint string - APIKey string - Targets []string - SampleLength int - DryRun bool - Prune bool - AllowDeleteAll bool - SourceRepoUrl string + ConfigFile string + APIEndpoint string + APIKey string + Targets []string + SampleLength int + DryRun bool + Prune bool + AllowDeleteAll bool + SourceRepoUrl string + CatalogEntriesAPIPageSize int } func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions { @@ -59,6 +60,10 @@ func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions { BoolVar(&opt.Prune) cmd.Flag("allow-delete-all", "Allow removing all entries from a catalog entry"). BoolVar(&opt.AllowDeleteAll) + cmd.Flag("catalog-entries-api-page-size", "The page size to use when listing catalog entries from the API"). + Envar("CATALOG_ENTRIES_API_PAGE_SIZE"). + Default("250"). + IntVar(&opt.CatalogEntriesAPIPageSize) return opt } @@ -473,7 +478,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf logger.Log("msg", "reconciling catalog entries", "output", outputType.TypeName) catalogType := catalogTypesByOutput[outputType.TypeName] - err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun)) + err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize) if err != nil { return errors.Wrap(err, fmt.Sprintf("outputs (type_name = '%s'): reconciling catalog entries", outputType.TypeName)) } @@ -509,7 +514,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf OUT("\n ↻ %s (enum)", enumModel.TypeName) catalogType := catalogTypesByOutput[enumModel.TypeName] - err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun)) + err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize) if err != nil { return errors.Wrap(err, fmt.Sprintf("outputs (type_name = '%s'): enum for attribute (id = '%s'): %s: reconciling catalog entries", @@ -530,7 +535,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli } return reconcile.EntriesClient{ - GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) { + GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) { // We're in dry-run and this catalog type is yet to be created. We can't ask the API // for the entries of a type that doesn't exist, so we return the type we faked from // the dry-run create and an empty list of entries. @@ -545,7 +550,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli } // We're just a normal catalog type, use the real client. - return reconcile.GetEntries(ctx, cl, catalogTypeID) + return reconcile.GetEntries(ctx, cl, catalogTypeID, pageSize) }, Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error { DIFF(" ", *entry, client.CatalogEntryV2{}) diff --git a/reconcile/entries.go b/reconcile/entries.go index 827cbcd..5d905d0 100644 --- a/reconcile/entries.go +++ b/reconcile/entries.go @@ -15,7 +15,7 @@ import ( ) type EntriesClient struct { - GetEntries func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) + GetEntries func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) Delete func(ctx context.Context, entry *client.CatalogEntryV2) error Create func(ctx context.Context, payload client.CreateEntryRequestBody) (*client.CatalogEntryV2, error) Update func(ctx context.Context, entry *client.CatalogEntryV2, payload client.UpdateEntryRequestBody) (*client.CatalogEntryV2, error) @@ -26,8 +26,8 @@ type EntriesClient struct { // actually perform updates. func EntriesClientFromClient(cl *client.ClientWithResponses) EntriesClient { return EntriesClient{ - GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) { - return GetEntries(ctx, cl, catalogTypeID) + GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) { + return GetEntries(ctx, cl, catalogTypeID, pageSize) }, Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error { _, err := cl.CatalogV2DestroyEntryWithResponse(ctx, entry.Id) @@ -65,7 +65,7 @@ type EntriesProgress struct { OnUpdateProgress func() } -func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress) error { +func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress, pageSize int) error { logger = kitlog.With(logger, "catalog_type_id", catalogType.Id, "catalog_type_name", catalogType.TypeName, @@ -77,7 +77,7 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output } logger.Log("msg", "listing existing entries") - catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id) + catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id, pageSize) if err != nil { return errors.Wrap(err, "listing entries") } @@ -307,7 +307,7 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output } // GetEntries paginates through all catalog entries for the given type. -func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) { +func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string, pageSize int) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) { var ( after *string ) @@ -315,7 +315,7 @@ func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogType for { result, err := cl.CatalogV2ListEntriesWithResponse(ctx, &client.CatalogV2ListEntriesParams{ CatalogTypeId: catalogTypeID, - PageSize: lo.ToPtr(int64(250)), + PageSize: lo.ToPtr(int64(pageSize)), After: after, }) if err != nil {