From f601c828e2b04ffd8328d30493443f1047a5cc7f Mon Sep 17 00:00:00 2001 From: Integralist Date: Fri, 28 Jul 2023 11:59:20 +0100 Subject: [PATCH 1/3] fix(kvstore/entry): disallow --json flag with --all --- pkg/commands/kvstoreentry/delete.go | 7 ++++++- pkg/commands/kvstoreentry/kvstoreentry_test.go | 4 ++++ pkg/errors/errors.go | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/commands/kvstoreentry/delete.go b/pkg/commands/kvstoreentry/delete.go index 00fd9e3cf..ae70b7f8f 100644 --- a/pkg/commands/kvstoreentry/delete.go +++ b/pkg/commands/kvstoreentry/delete.go @@ -17,7 +17,8 @@ import ( ) // deleteKeysConcurrencyLimit is used to limit the concurrency when deleting ALL keys. -const deleteKeysConcurrencyLimit int = 1000 +// This is effectively the 'thread pool' size. +const deleteKeysConcurrencyLimit int = 100 // DeleteCommand calls the Fastly API to delete an kv store. type DeleteCommand struct { @@ -58,6 +59,10 @@ func (c *DeleteCommand) Exec(in io.Reader, out io.Writer) error { if c.Globals.Verbose() && c.JSONOutput.Enabled { return fsterr.ErrInvalidVerboseJSONCombo } + // TODO: Support --json for bulk deletions. + if c.deleteAll && c.JSONOutput.Enabled { + return fsterr.ErrInvalidDeleteAllJSONKeyCombo + } if c.deleteAll && c.key.WasSet { return fsterr.ErrInvalidDeleteAllKeyCombo } diff --git a/pkg/commands/kvstoreentry/kvstoreentry_test.go b/pkg/commands/kvstoreentry/kvstoreentry_test.go index 37570b82b..090cb5041 100644 --- a/pkg/commands/kvstoreentry/kvstoreentry_test.go +++ b/pkg/commands/kvstoreentry/kvstoreentry_test.go @@ -163,6 +163,10 @@ func TestDeleteCommand(t *testing.T) { Args: testutil.Args(kvstoreentry.RootName + " delete --store-id " + storeID), WantError: "invalid command, neither --all or --key provided", }, + { + Args: testutil.Args(kvstoreentry.RootName + " delete --json --all --store-id " + storeID), + WantError: "invalid flag combination, --all and --json", + }, { Args: testutil.Args(kvstoreentry.RootName + " delete --key a-key --all --store-id " + storeID), WantError: "invalid flag combination, --all and --key", diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index e464bc9ab..ff560e60f 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -107,6 +107,13 @@ var ErrInvalidVerboseJSONCombo = RemediationError{ Remediation: "Use either --verbose or --json, not both.", } +// ErrInvalidDeleteAllJSONKeyCombo means the user provided both a --all and +// --json flag which are mutually exclusive behaviours. +var ErrInvalidDeleteAllJSONKeyCombo = RemediationError{ + Inner: fmt.Errorf("invalid flag combination, --all and --json"), + Remediation: "Use either --all or --json, not both.", +} + // ErrInvalidDeleteAllKeyCombo means the user provided both a --all and --key // flag which are mutually exclusive behaviours. var ErrInvalidDeleteAllKeyCombo = RemediationError{ From bd37790a7060d87428c9da2df2e72a561f47b8ad Mon Sep 17 00:00:00 2001 From: Integralist Date: Fri, 28 Jul 2023 12:12:21 +0100 Subject: [PATCH 2/3] feat(config-store/entry): support deleting all keys --- .../configstoreentry/configstoreentry_test.go | 61 ++++++++- pkg/commands/configstoreentry/delete.go | 125 ++++++++++++++++-- 2 files changed, 176 insertions(+), 10 deletions(-) diff --git a/pkg/commands/configstoreentry/configstoreentry_test.go b/pkg/commands/configstoreentry/configstoreentry_test.go index 1b3506d9b..0d94d2337 100644 --- a/pkg/commands/configstoreentry/configstoreentry_test.go +++ b/pkg/commands/configstoreentry/configstoreentry_test.go @@ -15,6 +15,7 @@ import ( "github.com/fastly/cli/pkg/mock" "github.com/fastly/cli/pkg/testutil" "github.com/fastly/cli/pkg/text" + "github.com/fastly/cli/pkg/threadsafe" ) func TestCreateEntryCommand(t *testing.T) { @@ -97,11 +98,36 @@ func TestDeleteEntryCommand(t *testing.T) { itemKey = "key" ) + now := time.Now() + + testItems := make([]*fastly.ConfigStoreItem, 3) + for i := range testItems { + testItems[i] = &fastly.ConfigStoreItem{ + StoreID: storeID, + Key: fmt.Sprintf("key-%02d", i), + Value: fmt.Sprintf("value %02d", i), + CreatedAt: &now, + UpdatedAt: &now, + } + } + scenarios := []testutil.TestScenario{ { Args: testutil.Args(configstoreentry.RootName + " delete --key a-key"), WantError: "error parsing arguments: required flag --store-id not provided", }, + { + Args: testutil.Args(configstoreentry.RootName + " delete --store-id " + storeID), + WantError: "invalid command, neither --all or --key provided", + }, + { + Args: testutil.Args(configstoreentry.RootName + " delete --json --all --store-id " + storeID), + WantError: "invalid flag combination, --all and --json", + }, + { + Args: testutil.Args(configstoreentry.RootName + " delete --key a-key --all --store-id " + storeID), + WantError: "invalid flag combination, --all and --key", + }, { Args: testutil.Args(fmt.Sprintf("%s delete --store-id %s --key %s", configstoreentry.RootName, storeID, itemKey)), API: mock.API{ @@ -137,20 +163,51 @@ func TestDeleteEntryCommand(t *testing.T) { true, }), }, + { + Args: testutil.Args(fmt.Sprintf("%s delete --store-id %s --all --auto-yes", configstoreentry.RootName, storeID)), + API: mock.API{ + ListConfigStoreItemsFn: func(i *fastly.ListConfigStoreItemsInput) ([]*fastly.ConfigStoreItem, error) { + return testItems, nil + }, + DeleteConfigStoreItemFn: func(i *fastly.DeleteConfigStoreItemInput) error { + return nil + }, + }, + WantOutput: fmt.Sprintf(`Deleting key: key-00 +Deleting key: key-01 +Deleting key: key-02 + +SUCCESS: Deleted all keys from Config Store '%s' +`, storeID), + }, + { + Args: testutil.Args(fmt.Sprintf("%s delete --store-id %s --all --auto-yes", configstoreentry.RootName, storeID)), + API: mock.API{ + ListConfigStoreItemsFn: func(i *fastly.ListConfigStoreItemsInput) ([]*fastly.ConfigStoreItem, error) { + return testItems, nil + }, + DeleteConfigStoreItemFn: func(i *fastly.DeleteConfigStoreItemInput) error { + return errors.New("whoops") + }, + }, + WantError: "failed to delete keys: key-00, key-01, key-02", + }, } for _, testcase := range scenarios { testcase := testcase t.Run(testcase.Name, func(t *testing.T) { - var stdout bytes.Buffer + var stdout threadsafe.Buffer opts := testutil.NewRunOpts(testcase.Args, &stdout) opts.APIClient = mock.APIClient(testcase.API) err := app.Run(opts) + t.Log(stdout.String()) + testutil.AssertErrorContains(t, err, testcase.WantError) - testutil.AssertString(t, testcase.WantOutput, stdout.String()) + testutil.AssertStringContains(t, stdout.String(), testcase.WantOutput) }) } } diff --git a/pkg/commands/configstoreentry/delete.go b/pkg/commands/configstoreentry/delete.go index e2fea7d4b..efe5daa96 100644 --- a/pkg/commands/configstoreentry/delete.go +++ b/pkg/commands/configstoreentry/delete.go @@ -1,7 +1,10 @@ package configstoreentry import ( + "fmt" "io" + "strings" + "sync" "github.com/fastly/go-fastly/v8/fastly" @@ -12,6 +15,14 @@ import ( "github.com/fastly/cli/pkg/text" ) +// deleteKeysConcurrencyLimit is used to limit the concurrency when deleting ALL keys. +// This is effectively the 'thread pool' size. +const deleteKeysConcurrencyLimit int = 100 + +// batchLimit is used to split the list of items into batches. +// The batch size of 100 aligns with the KV Store pagination default limit. +const batchLimit int = 100 + // NewDeleteCommand returns a usable command registered under the parent. func NewDeleteCommand(parent cmd.Registerer, g *global.Data, m manifest.Data) *DeleteCommand { c := DeleteCommand{ @@ -24,17 +35,19 @@ func NewDeleteCommand(parent cmd.Registerer, g *global.Data, m manifest.Data) *D c.CmdClause = parent.Command("delete", "Delete a config store item") // Required. + c.RegisterFlag(cmd.StoreIDFlag(&c.input.StoreID)) // --store-id + + // Optional. + c.CmdClause.Flag("all", "Delete all entries within the store").Short('a').BoolVar(&c.deleteAll) + c.CmdClause.Flag("batch-size", "Key batch processing size (ignored when set without the --all flag)").Short('b').Action(c.batchSize.Set).IntVar(&c.batchSize.Value) + c.CmdClause.Flag("concurrency", "Control thread pool size (ignored when set without the --all flag)").Short('c').Action(c.concurrency.Set).IntVar(&c.concurrency.Value) + c.RegisterFlagBool(c.JSONFlag()) // --json c.RegisterFlag(cmd.StringFlagOpts{ Name: "key", Short: 'k', Description: "Item name", Dst: &c.input.Key, - Required: true, }) - c.RegisterFlag(cmd.StoreIDFlag(&c.input.StoreID)) // --store-id - - // Optional. - c.RegisterFlagBool(c.JSONFlag()) // --json return &c } @@ -44,15 +57,44 @@ type DeleteCommand struct { cmd.Base cmd.JSONOutput - input fastly.DeleteConfigStoreItemInput - manifest manifest.Data + batchSize cmd.OptionalInt + concurrency cmd.OptionalInt + deleteAll bool + input fastly.DeleteConfigStoreItemInput + manifest manifest.Data } // Exec invokes the application logic for the command. -func (c *DeleteCommand) Exec(_ io.Reader, out io.Writer) error { +func (c *DeleteCommand) Exec(in io.Reader, out io.Writer) error { if c.Globals.Verbose() && c.JSONOutput.Enabled { return fsterr.ErrInvalidVerboseJSONCombo } + // TODO: Support --json for bulk deletions. + if c.deleteAll && c.JSONOutput.Enabled { + return fsterr.ErrInvalidDeleteAllJSONKeyCombo + } + if c.deleteAll && c.input.Key != "" { + return fsterr.ErrInvalidDeleteAllKeyCombo + } + if !c.deleteAll && c.input.Key == "" { + return fsterr.ErrMissingDeleteAllKeyCombo + } + + if c.deleteAll { + if !c.Globals.Flags.AutoYes && !c.Globals.Flags.NonInteractive { + text.Warning(out, "This will delete ALL entries from your store!") + text.Break(out) + cont, err := text.AskYesNo(out, "Are you sure you want to continue? [yes/no]: ", in) + if err != nil { + return err + } + if !cont { + return nil + } + text.Break(out) + } + return c.deleteAllKeys(out) + } err := c.Globals.APIClient.DeleteConfigStoreItem(&c.input) if err != nil { @@ -78,3 +120,70 @@ func (c *DeleteCommand) Exec(_ io.Reader, out io.Writer) error { return nil } + +func (c *DeleteCommand) deleteAllKeys(out io.Writer) error { + // NOTE: The Config Store returns ALL items (there is no pagination). + items, err := c.Globals.APIClient.ListConfigStoreItems(&fastly.ListConfigStoreItemsInput{ + StoreID: c.input.StoreID, + }) + if err != nil { + return fmt.Errorf("failed to acquire list of Config Store items: %w", err) + } + + var ( + mu sync.Mutex + wg sync.WaitGroup + ) + poolSize := deleteKeysConcurrencyLimit + if c.concurrency.WasSet { + poolSize = c.concurrency.Value + } + semaphore := make(chan struct{}, poolSize) + + total := len(items) + failedKeys := []string{} + + batchSize := batchLimit + if c.batchSize.WasSet { + batchSize = c.batchSize.Value + } + + // With KV Store we have pagination support and so that natively provides us a + // predefined 'batch' size. Because we don't have pagination with the Config + // Store it means we'll define our own batch size which the user can override. + for i := 0; i < total; i += batchSize { + end := i + batchSize + if end > total { + end = total + } + seg := items[i:end] + + wg.Add(1) + go func(items []*fastly.ConfigStoreItem) { + semaphore <- struct{}{} + defer func() { <-semaphore }() + defer wg.Done() + + for _, item := range items { + text.Output(out, "Deleting key: %s", item.Key) + err := c.Globals.APIClient.DeleteConfigStoreItem(&fastly.DeleteConfigStoreItemInput{StoreID: c.input.StoreID, Key: item.Key}) + if err != nil { + c.Globals.ErrLog.Add(fmt.Errorf("failed to delete key '%s': %s", item.Key, err)) + mu.Lock() + failedKeys = append(failedKeys, item.Key) + mu.Unlock() + } + } + }(seg) + } + + wg.Wait() + close(semaphore) + + if len(failedKeys) > 0 { + return fmt.Errorf("failed to delete keys: %s", strings.Join(failedKeys, ", ")) + } + + text.Success(out, "Deleted all keys from Config Store '%s'", c.input.StoreID) + return nil +} From f54d4ad7c9121a2a7df272a4eedb702662f75ff5 Mon Sep 17 00:00:00 2001 From: Integralist Date: Fri, 28 Jul 2023 12:53:55 +0100 Subject: [PATCH 3/3] feat(metadata): document config-store-entry bulk delete example --- pkg/app/metadata.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/app/metadata.json b/pkg/app/metadata.json index 770991a49..37bb0064f 100644 --- a/pkg/app/metadata.json +++ b/pkg/app/metadata.json @@ -315,6 +315,21 @@ ] } }, + "config-store-entry": { + "delete": { + "examples": [ + { + "cmd": "fastly config-store-entry delete --all --store-id --batch-size 30 --auto-yes", + "description": "Delete all entries from the Config Store in batches of 30 and ignoring the warning prompt.", + "title": "Delete all entries from the Config Store" + } + ], + "apis": [ + "https://developer.fastly.com/reference/api/services/resources/config-store-item/#list-config-store-items", + "https://developer.fastly.com/reference/api/services/resources/config-store-item/#delete-config-store-item" + ] + } + }, "dictionary": { "create": { "apis": [