Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support configuring the page size for listing catalog entries #151

Merged
merged 1 commit into from
Nov 22, 2024
+25 −20
Merged

Support configuring the page size for listing catalog entries #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions cmd/catalog-importer/cmd/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ import (
)

type SyncOptions struct {
ConfigFile string
APIEndpoint string
APIKey string
Targets []string
SampleLength int
DryRun bool
Prune bool
AllowDeleteAll bool
SourceRepoUrl string
ConfigFile string
APIEndpoint string
APIKey string
Targets []string
SampleLength int
DryRun bool
Prune bool
AllowDeleteAll bool
SourceRepoUrl string
CatalogEntriesAPIPageSize int
}

func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions {
Expand Down Expand Up @@ -59,6 +60,10 @@ func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions {
BoolVar(&opt.Prune)
cmd.Flag("allow-delete-all", "Allow removing all entries from a catalog entry").
BoolVar(&opt.AllowDeleteAll)
cmd.Flag("catalog-entries-api-page-size", "The page size to use when listing catalog entries from the API").
Envar("CATALOG_ENTRIES_API_PAGE_SIZE").
Default("250").
IntVar(&opt.CatalogEntriesAPIPageSize)

return opt
}
Expand Down Expand Up @@ -473,7 +478,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf
logger.Log("msg", "reconciling catalog entries", "output", outputType.TypeName)
catalogType := catalogTypesByOutput[outputType.TypeName]

err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun))
err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("outputs (type_name = '%s'): reconciling catalog entries", outputType.TypeName))
}
Expand Down Expand Up @@ -509,7 +514,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf

OUT("\n ↻ %s (enum)", enumModel.TypeName)
catalogType := catalogTypesByOutput[enumModel.TypeName]
err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun))
err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize)
if err != nil {
return errors.Wrap(err,
fmt.Sprintf("outputs (type_name = '%s'): enum for attribute (id = '%s'): %s: reconciling catalog entries",
Expand All @@ -530,7 +535,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli
}

return reconcile.EntriesClient{
GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
// We're in dry-run and this catalog type is yet to be created. We can't ask the API
// for the entries of a type that doesn't exist, so we return the type we faked from
// the dry-run create and an empty list of entries.
Expand All @@ -545,7 +550,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli
}

// We're just a normal catalog type, use the real client.
return reconcile.GetEntries(ctx, cl, catalogTypeID)
return reconcile.GetEntries(ctx, cl, catalogTypeID, pageSize)
},
Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error {
DIFF(" ", *entry, client.CatalogEntryV2{})
Expand Down
14 changes: 7 additions & 7 deletions reconcile/entries.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
)

type EntriesClient struct {
GetEntries func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error)
GetEntries func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error)
Delete func(ctx context.Context, entry *client.CatalogEntryV2) error
Create func(ctx context.Context, payload client.CreateEntryRequestBody) (*client.CatalogEntryV2, error)
Update func(ctx context.Context, entry *client.CatalogEntryV2, payload client.UpdateEntryRequestBody) (*client.CatalogEntryV2, error)
Expand All @@ -26,8 +26,8 @@ type EntriesClient struct {
// actually perform updates.
func EntriesClientFromClient(cl *client.ClientWithResponses) EntriesClient {
return EntriesClient{
GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
return GetEntries(ctx, cl, catalogTypeID)
GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
return GetEntries(ctx, cl, catalogTypeID, pageSize)
},
Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error {
_, err := cl.CatalogV2DestroyEntryWithResponse(ctx, entry.Id)
Expand Down Expand Up @@ -65,7 +65,7 @@ type EntriesProgress struct {
OnUpdateProgress func()
}

func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress) error {
func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress, pageSize int) error {
logger = kitlog.With(logger,
"catalog_type_id", catalogType.Id,
"catalog_type_name", catalogType.TypeName,
Expand All @@ -77,7 +77,7 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output
}

logger.Log("msg", "listing existing entries")
catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id)
catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id, pageSize)
if err != nil {
return errors.Wrap(err, "listing entries")
}
Expand Down Expand Up @@ -307,15 +307,15 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output
}

// GetEntries paginates through all catalog entries for the given type.
func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) {
func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string, pageSize int) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) {
var (
after *string
)

for {
result, err := cl.CatalogV2ListEntriesWithResponse(ctx, &client.CatalogV2ListEntriesParams{
CatalogTypeId: catalogTypeID,
PageSize: lo.ToPtr(int64(250)),
PageSize: lo.ToPtr(int64(pageSize)),
After: after,
})
if err != nil {
Expand Down
Loading