From 7890e9451da421c0dc1958e530f48c1206c4a2bf Mon Sep 17 00:00:00 2001 From: lew Date: Wed, 17 Dec 2025 22:18:15 +0000 Subject: [PATCH] feat(globs): glob support for dump/restore, extracts some shared logic --- README.md | 2 +- cmd/dump.go | 40 +++++++++++++++++---- cmd/glob.go | 26 ++++++++++++++ cmd/list.go | 26 +++----------- cmd/restore.go | 65 ++++++++++++++++++++++------------- testdata/dump__glob__ok.ct | 8 +++++ testdata/help__dump__ok.ct | 4 +++ testdata/help__restore__ok.ct | 12 ++++--- testdata/restore__glob__ok.ct | 15 ++++++++ 9 files changed, 141 insertions(+), 57 deletions(-) create mode 100644 testdata/dump__glob__ok.ct create mode 100644 testdata/restore__glob__ok.ct diff --git a/README.md b/README.md index 667ce3b..3d44e0d 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ pda set kitty "cat" pda del kitty --glob ?og # remove "kitty", "cog", "dog": are you sure? [y/n] # y -# Default glob separators: "/-_.@:". Override with --glob-sep. +# Default glob separators: "/-_.@: " (space included). Override with --glob-sep. ```

diff --git a/cmd/dump.go b/cmd/dump.go index 2661028..2750225 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -31,6 +31,7 @@ var dumpCmd = &cobra.Command{ func dump(cmd *cobra.Command, args []string) error { store := &Store{} targetDB := "@default" + displayTarget := targetDB if len(args) == 1 { rawArg := args[0] dbName, err := store.parseDB(rawArg, false) @@ -45,23 +46,37 @@ func dump(cmd *cobra.Command, args []string) error { return err } targetDB = "@" + dbName + displayTarget = targetDB } mode, err := cmd.Flags().GetString("encoding") if err != nil { - return fmt.Errorf("cannot dump '%s': %v", args[0], err) + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) } switch mode { case "auto", "base64", "text": default: - return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", args[0], mode) + return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", displayTarget, mode) } includeSecret, err := cmd.Flags().GetBool("secret") if err != nil { return err } + globPatterns, err := cmd.Flags().GetStringSlice("glob") + if err != nil { + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) + } + separators, err := parseGlobSeparators(cmd) + if err != nil { + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) + } + matchers, err := compileGlobMatchers(globPatterns, separators) + if err != nil { + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) + } + var matched bool trans := TransactionArgs{ key: targetDB, readonly: true, @@ -74,6 +89,9 @@ func dump(cmd *cobra.Command, args []string) error { for it.Rewind(); it.Valid(); it.Next() { item := it.Item() key := item.KeyCopy(nil) + if !globMatch(matchers, string(key)) { + continue + } meta := item.UserMeta() isSecret := meta&metaSecret != 0 if isSecret && !includeSecret { @@ -94,7 +112,7 @@ func dump(cmd *cobra.Command, args []string) error { encodeBase64(&entry, v) case "text": if err := encodeText(&entry, key, v); err != nil { - return fmt.Errorf("cannot dump '%s': %v", args[0], err) + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) } case "auto": if utf8.Valid(v) { @@ -106,24 +124,34 @@ func dump(cmd *cobra.Command, args []string) error { } payload, err := json.Marshal(entry) if err != nil { - return fmt.Errorf("cannot dump '%s': %v", args[0], err) + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) } fmt.Fprintln(cmd.OutOrStdout(), string(payload)) + matched = true return nil }); err != nil { - return fmt.Errorf("cannot dump '%s': %v", args[0], err) + return fmt.Errorf("cannot dump '%s': %v", displayTarget, err) } } return nil }, } - return store.Transaction(trans) + if err := store.Transaction(trans); err != nil { + return err + } + + if len(matchers) > 0 && !matched { + return fmt.Errorf("cannot dump '%s': No matches for pattern", displayTarget) + } + return nil } func init() { dumpCmd.Flags().StringP("encoding", "e", "auto", "value encoding: auto, base64, or text") dumpCmd.Flags().Bool("secret", false, "Include entries marked as secret") + dumpCmd.Flags().StringSliceP("glob", "g", nil, "Filter keys with glob pattern (repeatable)") + dumpCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay())) rootCmd.AddCommand(dumpCmd) } diff --git a/cmd/glob.go b/cmd/glob.go index 73da54a..5a7bb7e 100644 --- a/cmd/glob.go +++ b/cmd/glob.go @@ -3,6 +3,7 @@ package cmd import ( "strings" + "github.com/gobwas/glob" "github.com/spf13/cobra" ) @@ -26,3 +27,28 @@ func parseGlobSeparators(cmd *cobra.Command) ([]rune, error) { } return []rune(sepStr), nil } + +func compileGlobMatchers(patterns []string, separators []rune) ([]glob.Glob, error) { + var matchers []glob.Glob + for _, pattern := range patterns { + m, err := glob.Compile(strings.ToLower(pattern), separators...) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + } + return matchers, nil +} + +func globMatch(matchers []glob.Glob, key string) bool { + if len(matchers) == 0 { + return true + } + lowered := strings.ToLower(key) + for _, m := range matchers { + if m.Match(lowered) { + return true + } + } + return false +} diff --git a/cmd/list.go b/cmd/list.go index 63e42c7..10f2db4 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -24,10 +24,8 @@ package cmd import ( "errors" "fmt" - "strings" "github.com/dgraph-io/badger/v4" - "github.com/gobwas/glob" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" ) @@ -73,25 +71,9 @@ func list(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("cannot ls '%s': %v", targetDB, err) } - var matchers []glob.Glob - for _, pattern := range globPatterns { - m, err := glob.Compile(strings.ToLower(pattern), separators...) - if err != nil { - return fmt.Errorf("cannot ls '%s': %v", targetDB, err) - } - matchers = append(matchers, m) - } - - matchesKey := func(k string) bool { - if len(matchers) == 0 { - return true - } - for _, m := range matchers { - if m.Match(k) { - return true - } - } - return false + matchers, err := compileGlobMatchers(globPatterns, separators) + if err != nil { + return fmt.Errorf("cannot ls '%s': %v", targetDB, err) } columnKinds, err := requireColumns(flags) @@ -135,7 +117,7 @@ func list(cmd *cobra.Command, args []string) error { for it.Rewind(); it.Valid(); it.Next() { item := it.Item() key := string(item.KeyCopy(nil)) - if !matchesKey(key) { + if !globMatch(matchers, key) { continue } matchedCount++ diff --git a/cmd/restore.go b/cmd/restore.go index 0caa804..8926416 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -32,10 +32,24 @@ func restore(cmd *cobra.Command, args []string) error { } dbName = parsed } + displayTarget := "@" + dbName + + globPatterns, err := cmd.Flags().GetStringSlice("glob") + if err != nil { + return fmt.Errorf("cannot restore '%s': %v", displayTarget, err) + } + separators, err := parseGlobSeparators(cmd) + if err != nil { + return fmt.Errorf("cannot restore '%s': %v", displayTarget, err) + } + matchers, err := compileGlobMatchers(globPatterns, separators) + if err != nil { + return fmt.Errorf("cannot restore '%s': %v", displayTarget, err) + } reader, closer, err := restoreInput(cmd) if err != nil { - return fmt.Errorf("cannot restore '%s': %v", dbName, err) + return fmt.Errorf("cannot restore '%s': %v", displayTarget, err) } if closer != nil { defer closer.Close() @@ -43,38 +57,38 @@ func restore(cmd *cobra.Command, args []string) error { db, err := store.open(dbName) if err != nil { - return fmt.Errorf("cannot restore '%s': %v", dbName, err) + return fmt.Errorf("cannot restore '%s': %v", displayTarget, err) } defer db.Close() - scanner := bufio.NewScanner(reader) - buf := make([]byte, 1024*1024) - scanner.Buffer(buf, 8*1024*1024) + decoder := json.NewDecoder(bufio.NewReaderSize(reader, 8*1024*1024)) wb := db.NewWriteBatch() defer wb.Cancel() - lineNo := 0 + entryNo := 0 var restored int + var matched bool - for scanner.Scan() { - lineNo++ - line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue - } - + for { var entry dumpEntry - if err := json.Unmarshal([]byte(line), &entry); err != nil { - return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err) + if err := decoder.Decode(&entry); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo+1, err) } + entryNo++ if entry.Key == "" { - return fmt.Errorf("cannot restore '%s': line %d: missing key", dbName, lineNo) + return fmt.Errorf("cannot restore '%s': entry %d: missing key", displayTarget, entryNo) + } + if !globMatch(matchers, entry.Key) { + continue } value, err := decodeEntryValue(entry) if err != nil { - return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err) + return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err) } entryMeta := byte(0x0) @@ -85,23 +99,24 @@ func restore(cmd *cobra.Command, args []string) error { writeEntry := badger.NewEntry([]byte(entry.Key), value).WithMeta(entryMeta) if entry.ExpiresAt != nil { if *entry.ExpiresAt < 0 { - return fmt.Errorf("cannot restore '%s': line %d: expires_at must be >= 0", dbName, lineNo) + return fmt.Errorf("cannot restore '%s': entry %d: expires_at must be >= 0", displayTarget, entryNo) } writeEntry.ExpiresAt = uint64(*entry.ExpiresAt) } if err := wb.SetEntry(writeEntry); err != nil { - return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err) + return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err) } restored++ - } - - if err := scanner.Err(); err != nil { - return err + matched = true } if err := wb.Flush(); err != nil { - return fmt.Errorf("cannot restore '%s': %v", dbName, err) + return fmt.Errorf("cannot restore '%s': %v", displayTarget, err) + } + + if len(matchers) > 0 && !matched { + return fmt.Errorf("cannot restore '%s': No matches for pattern", displayTarget) } fmt.Fprintf(cmd.ErrOrStderr(), "Restored %d entries into @%s\n", restored, dbName) @@ -140,5 +155,7 @@ func decodeEntryValue(entry dumpEntry) ([]byte, error) { func init() { restoreCmd.Flags().StringP("file", "f", "", "Path to an NDJSON dump (defaults to stdin)") + restoreCmd.Flags().StringSliceP("glob", "g", nil, "Restore keys matching glob pattern (repeatable)") + restoreCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay())) rootCmd.AddCommand(restoreCmd) } diff --git a/testdata/dump__glob__ok.ct b/testdata/dump__glob__ok.ct new file mode 100644 index 0000000..7d1ce79 --- /dev/null +++ b/testdata/dump__glob__ok.ct @@ -0,0 +1,8 @@ +$ pda set a1 1 +$ pda set a2 2 +$ pda set b1 3 +$ pda dump --glob a* +{"key":"a1","value":"1","encoding":"text"} +{"key":"a2","value":"2","encoding":"text"} +$ pda dump --glob c* --> FAIL +Error: cannot dump '@default': No matches for pattern diff --git a/testdata/help__dump__ok.ct b/testdata/help__dump__ok.ct index ae6042c..c471a6d 100644 --- a/testdata/help__dump__ok.ct +++ b/testdata/help__dump__ok.ct @@ -10,6 +10,8 @@ Aliases: Flags: -e, --encoding string value encoding: auto, base64, or text (default "auto") + -g, --glob strings Filter keys with glob pattern (repeatable) + --glob-sep string Characters treated as separators for globbing (default "/-_.@: ") -h, --help help for dump --secret Include entries marked as secret Dump all key/value pairs as NDJSON @@ -22,5 +24,7 @@ Aliases: Flags: -e, --encoding string value encoding: auto, base64, or text (default "auto") + -g, --glob strings Filter keys with glob pattern (repeatable) + --glob-sep string Characters treated as separators for globbing (default "/-_.@: ") -h, --help help for dump --secret Include entries marked as secret diff --git a/testdata/help__restore__ok.ct b/testdata/help__restore__ok.ct index 765ac68..876fc96 100644 --- a/testdata/help__restore__ok.ct +++ b/testdata/help__restore__ok.ct @@ -9,8 +9,10 @@ Aliases: restore, import Flags: - -f, --file string Path to an NDJSON dump (defaults to stdin) - -h, --help help for restore + -f, --file string Path to an NDJSON dump (defaults to stdin) + -g, --glob strings Restore keys matching glob pattern (repeatable) + --glob-sep string Characters treated as separators for globbing (default "/-_.@: ") + -h, --help help for restore Restore key/value pairs from an NDJSON dump Usage: @@ -20,5 +22,7 @@ Aliases: restore, import Flags: - -f, --file string Path to an NDJSON dump (defaults to stdin) - -h, --help help for restore + -f, --file string Path to an NDJSON dump (defaults to stdin) + -g, --glob strings Restore keys matching glob pattern (repeatable) + --glob-sep string Characters treated as separators for globbing (default "/-_.@: ") + -h, --help help for restore diff --git a/testdata/restore__glob__ok.ct b/testdata/restore__glob__ok.ct new file mode 100644 index 0000000..3394d26 --- /dev/null +++ b/testdata/restore__glob__ok.ct @@ -0,0 +1,15 @@ +$ pda set a1 1 +$ pda set a2 2 +$ pda set b1 3 +$ fecho dumpfile {"key":"a1","value":"1","encoding":"text"} {"key":"a2","value":"2","encoding":"text"} {"key":"b1","value":"3","encoding":"text"} +$ pda del a1 a2 b1 --force +$ pda restore --glob a* --file dumpfile +Restored 2 entries into @default +$ pda get a1 +1 +$ pda get a2 +2 +$ pda get b1 --> FAIL +Error: cannot get 'b1': Key not found +$ pda restore --glob c* --file dumpfile --> FAIL +Error: cannot restore '@default': No matches for pattern