lew/pda
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(globs): glob support for dump/restore, extracts some shared logic

Browse source
This commit is contained in:
Lewis Wynne 2025-12-17 22:18:15 +00:00
parent 9869b663e2
commit 7890e9451d
9 changed files with 141 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -145,7 +145,7 @@ pda set kitty "cat"
pda del kitty --glob ?og
# remove "kitty", "cog", "dog": are you sure? [y/n]
# y
# Default glob separators: "/-_.@:". Override with --glob-sep.
# Default glob separators: "/-_.@: " (space included). Override with --glob-sep.
```
<p align="center"></p><!-- spacer -->

40
cmd/dump.go
View file

@ -31,6 +31,7 @@ var dumpCmd = &cobra.Command{
func dump(cmd *cobra.Command, args []string) error {
store := &Store{}
targetDB := "@default"
displayTarget := targetDB
if len(args) == 1 {
rawArg := args[0]
dbName, err := store.parseDB(rawArg, false)
@ -45,23 +46,37 @@ func dump(cmd *cobra.Command, args []string) error {
return err
}
targetDB = "@" + dbName
displayTarget = targetDB
}
mode, err := cmd.Flags().GetString("encoding")
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
switch mode {
case "auto", "base64", "text":
default:
return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", args[0], mode)
return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", displayTarget, mode)
}
includeSecret, err := cmd.Flags().GetBool("secret")
if err != nil {
return err
}
globPatterns, err := cmd.Flags().GetStringSlice("glob")
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
separators, err := parseGlobSeparators(cmd)
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
matchers, err := compileGlobMatchers(globPatterns, separators)
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
var matched bool
trans := TransactionArgs{
key: targetDB,
readonly: true,
@ -74,6 +89,9 @@ func dump(cmd *cobra.Command, args []string) error {
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
key := item.KeyCopy(nil)
if !globMatch(matchers, string(key)) {
continue
}
meta := item.UserMeta()
isSecret := meta&metaSecret != 0
if isSecret && !includeSecret {
@ -94,7 +112,7 @@ func dump(cmd *cobra.Command, args []string) error {
encodeBase64(&entry, v)
case "text":
if err := encodeText(&entry, key, v); err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
case "auto":
if utf8.Valid(v) {
@ -106,24 +124,34 @@ func dump(cmd *cobra.Command, args []string) error {
}
payload, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
fmt.Fprintln(cmd.OutOrStdout(), string(payload))
matched = true
return nil
}); err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
}
return nil
},
}
return store.Transaction(trans)
if err := store.Transaction(trans); err != nil {
return err
}
if len(matchers) > 0 && !matched {
return fmt.Errorf("cannot dump '%s': No matches for pattern", displayTarget)
}
return nil
}
func init() {
dumpCmd.Flags().StringP("encoding", "e", "auto", "value encoding: auto, base64, or text")
dumpCmd.Flags().Bool("secret", false, "Include entries marked as secret")
dumpCmd.Flags().StringSliceP("glob", "g", nil, "Filter keys with glob pattern (repeatable)")
dumpCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay()))
rootCmd.AddCommand(dumpCmd)
}

26
cmd/glob.go
View file

@ -3,6 +3,7 @@ package cmd
import (
"strings"
"github.com/gobwas/glob"
"github.com/spf13/cobra"
)
@ -26,3 +27,28 @@ func parseGlobSeparators(cmd *cobra.Command) ([]rune, error) {
}
return []rune(sepStr), nil
}
func compileGlobMatchers(patterns []string, separators []rune) ([]glob.Glob, error) {
var matchers []glob.Glob
for _, pattern := range patterns {
m, err := glob.Compile(strings.ToLower(pattern), separators...)
if err != nil {
return nil, err
}
matchers = append(matchers, m)
}
return matchers, nil
}
func globMatch(matchers []glob.Glob, key string) bool {
if len(matchers) == 0 {
return true
}
lowered := strings.ToLower(key)
for _, m := range matchers {
if m.Match(lowered) {
return true
}
}
return false
}

26
cmd/list.go
View file

@ -24,10 +24,8 @@ package cmd
import (
"errors"
"fmt"
"strings"
"github.com/dgraph-io/badger/v4"
"github.com/gobwas/glob"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
)
@ -73,25 +71,9 @@ func list(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
}
var matchers []glob.Glob
for _, pattern := range globPatterns {
m, err := glob.Compile(strings.ToLower(pattern), separators...)
if err != nil {
return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
}
matchers = append(matchers, m)
}
matchesKey := func(k string) bool {
if len(matchers) == 0 {
return true
}
for _, m := range matchers {
if m.Match(k) {
return true
}
}
return false
matchers, err := compileGlobMatchers(globPatterns, separators)
if err != nil {
return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
}
columnKinds, err := requireColumns(flags)
@ -135,7 +117,7 @@ func list(cmd *cobra.Command, args []string) error {
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
key := string(item.KeyCopy(nil))
if !matchesKey(key) {
if !globMatch(matchers, key) {
continue
}
matchedCount++

65
cmd/restore.go
View file

@ -32,10 +32,24 @@ func restore(cmd *cobra.Command, args []string) error {
}
dbName = parsed
}
displayTarget := "@" + dbName
globPatterns, err := cmd.Flags().GetStringSlice("glob")
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
separators, err := parseGlobSeparators(cmd)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
matchers, err := compileGlobMatchers(globPatterns, separators)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
reader, closer, err := restoreInput(cmd)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", dbName, err)
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
if closer != nil {
defer closer.Close()
@ -43,38 +57,38 @@ func restore(cmd *cobra.Command, args []string) error {
db, err := store.open(dbName)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", dbName, err)
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
defer db.Close()
scanner := bufio.NewScanner(reader)
buf := make([]byte, 1024*1024)
scanner.Buffer(buf, 8*1024*1024)
decoder := json.NewDecoder(bufio.NewReaderSize(reader, 8*1024*1024))
wb := db.NewWriteBatch()
defer wb.Cancel()
lineNo := 0
entryNo := 0
var restored int
var matched bool
for scanner.Scan() {
lineNo++
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
for {
var entry dumpEntry
if err := json.Unmarshal([]byte(line), &entry); err != nil {
return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
if err := decoder.Decode(&entry); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo+1, err)
}
entryNo++
if entry.Key == "" {
return fmt.Errorf("cannot restore '%s': line %d: missing key", dbName, lineNo)
return fmt.Errorf("cannot restore '%s': entry %d: missing key", displayTarget, entryNo)
}
if !globMatch(matchers, entry.Key) {
continue
}
value, err := decodeEntryValue(entry)
if err != nil {
return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err)
}
entryMeta := byte(0x0)
@ -85,23 +99,24 @@ func restore(cmd *cobra.Command, args []string) error {
writeEntry := badger.NewEntry([]byte(entry.Key), value).WithMeta(entryMeta)
if entry.ExpiresAt != nil {
if *entry.ExpiresAt < 0 {
return fmt.Errorf("cannot restore '%s': line %d: expires_at must be >= 0", dbName, lineNo)
return fmt.Errorf("cannot restore '%s': entry %d: expires_at must be >= 0", displayTarget, entryNo)
}
writeEntry.ExpiresAt = uint64(*entry.ExpiresAt)
}
if err := wb.SetEntry(writeEntry); err != nil {
return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err)
}
restored++
}
if err := scanner.Err(); err != nil {
return err
matched = true
}
if err := wb.Flush(); err != nil {
return fmt.Errorf("cannot restore '%s': %v", dbName, err)
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
if len(matchers) > 0 && !matched {
return fmt.Errorf("cannot restore '%s': No matches for pattern", displayTarget)
}
fmt.Fprintf(cmd.ErrOrStderr(), "Restored %d entries into @%s\n", restored, dbName)
@ -140,5 +155,7 @@ func decodeEntryValue(entry dumpEntry) ([]byte, error) {
func init() {
restoreCmd.Flags().StringP("file", "f", "", "Path to an NDJSON dump (defaults to stdin)")
restoreCmd.Flags().StringSliceP("glob", "g", nil, "Restore keys matching glob pattern (repeatable)")
restoreCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay()))
rootCmd.AddCommand(restoreCmd)
}

8
testdata/dump__glob__ok.ct vendored Normal file
View file

@ -0,0 +1,8 @@
$ pda set a1 1
$ pda set a2 2
$ pda set b1 3
$ pda dump --glob a*
{"key":"a1","value":"1","encoding":"text"}
{"key":"a2","value":"2","encoding":"text"}
$ pda dump --glob c* --> FAIL
Error: cannot dump '@default': No matches for pattern

4
testdata/help__dump__ok.ct vendored
View file

@ -10,6 +10,8 @@ Aliases:
Flags:
-e, --encoding string value encoding: auto, base64, or text (default "auto")
-g, --glob strings Filter keys with glob pattern (repeatable)
--glob-sep string Characters treated as separators for globbing (default "/-_.@: ")
-h, --help help for dump
--secret Include entries marked as secret
Dump all key/value pairs as NDJSON
@ -22,5 +24,7 @@ Aliases:
Flags:
-e, --encoding string value encoding: auto, base64, or text (default "auto")
-g, --glob strings Filter keys with glob pattern (repeatable)
--glob-sep string Characters treated as separators for globbing (default "/-_.@: ")
-h, --help help for dump
--secret Include entries marked as secret

12
testdata/help__restore__ok.ct vendored
View file

@ -9,8 +9,10 @@ Aliases:
restore, import
Flags:
-f, --file string Path to an NDJSON dump (defaults to stdin)
-h, --help help for restore
-f, --file string Path to an NDJSON dump (defaults to stdin)
-g, --glob strings Restore keys matching glob pattern (repeatable)
--glob-sep string Characters treated as separators for globbing (default "/-_.@: ")
-h, --help help for restore
Restore key/value pairs from an NDJSON dump
Usage:
@ -20,5 +22,7 @@ Aliases:
restore, import
Flags:
-f, --file string Path to an NDJSON dump (defaults to stdin)
-h, --help help for restore
-f, --file string Path to an NDJSON dump (defaults to stdin)
-g, --glob strings Restore keys matching glob pattern (repeatable)
--glob-sep string Characters treated as separators for globbing (default "/-_.@: ")
-h, --help help for restore

15
testdata/restore__glob__ok.ct vendored Normal file
View file

@ -0,0 +1,15 @@
$ pda set a1 1
$ pda set a2 2
$ pda set b1 3
$ fecho dumpfile {"key":"a1","value":"1","encoding":"text"} {"key":"a2","value":"2","encoding":"text"} {"key":"b1","value":"3","encoding":"text"}
$ pda del a1 a2 b1 --force
$ pda restore --glob a* --file dumpfile
Restored 2 entries into @default
$ pda get a1
1
$ pda get a2
2
$ pda get b1 --> FAIL
Error: cannot get 'b1': Key not found
$ pda restore --glob c* --file dumpfile --> FAIL
Error: cannot restore '@default': No matches for pattern