feat(globs): glob support for dump/restore, extracts some shared logic

README.md

@@ -145,7 +145,7 @@ pda set kitty "cat"
 pda del kitty --glob ?og
 # remove "kitty", "cog", "dog": are you sure? [y/n]
 # y
-# Default glob separators: "/-_.@:". Override with --glob-sep.
+# Default glob separators: "/-_.@: " (space included). Override with --glob-sep.
 ```
 
 <p align="center"></p><!-- spacer -->

cmd/dump.go

@@ -31,6 +31,7 @@ var dumpCmd = &cobra.Command{
 func dump(cmd *cobra.Command, args []string) error {
 	store := &Store{}
 	targetDB := "@default"
+	displayTarget := targetDB
 	if len(args) == 1 {
 		rawArg := args[0]
 		dbName, err := store.parseDB(rawArg, false)
@@ -45,23 +46,37 @@ func dump(cmd *cobra.Command, args []string) error {
 			return err
 		}
 		targetDB = "@" + dbName
+		displayTarget = targetDB
 	}
 
 	mode, err := cmd.Flags().GetString("encoding")
 	if err != nil {
-		return fmt.Errorf("cannot dump '%s': %v", args[0], err)
+		return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
 	}
 	switch mode {
 	case "auto", "base64", "text":
 	default:
-		return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", args[0], mode)
+		return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", displayTarget, mode)
 	}
 
 	includeSecret, err := cmd.Flags().GetBool("secret")
 	if err != nil {
 		return err
 	}
+	globPatterns, err := cmd.Flags().GetStringSlice("glob")
+	if err != nil {
+		return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
+	}
+	separators, err := parseGlobSeparators(cmd)
+	if err != nil {
+		return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
+	}
+	matchers, err := compileGlobMatchers(globPatterns, separators)
+	if err != nil {
+		return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
+	}
 
+	var matched bool
 	trans := TransactionArgs{
 		key:      targetDB,
 		readonly: true,
@@ -74,6 +89,9 @@ func dump(cmd *cobra.Command, args []string) error {
 			for it.Rewind(); it.Valid(); it.Next() {
 				item := it.Item()
 				key := item.KeyCopy(nil)
+				if !globMatch(matchers, string(key)) {
+					continue
+				}
 				meta := item.UserMeta()
 				isSecret := meta&metaSecret != 0
 				if isSecret && !includeSecret {
@@ -94,7 +112,7 @@ func dump(cmd *cobra.Command, args []string) error {
 						encodeBase64(&entry, v)
 					case "text":
 						if err := encodeText(&entry, key, v); err != nil {
-							return fmt.Errorf("cannot dump '%s': %v", args[0], err)
+							return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
 						}
 					case "auto":
 						if utf8.Valid(v) {
@@ -106,24 +124,34 @@ func dump(cmd *cobra.Command, args []string) error {
 					}
 					payload, err := json.Marshal(entry)
 					if err != nil {
-						return fmt.Errorf("cannot dump '%s': %v", args[0], err)
+						return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
 					}
 					fmt.Fprintln(cmd.OutOrStdout(), string(payload))
+					matched = true
 					return nil
 				}); err != nil {
-					return fmt.Errorf("cannot dump '%s': %v", args[0], err)
+					return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
 				}
 			}
 			return nil
 		},
 	}
 
-	return store.Transaction(trans)
+	if err := store.Transaction(trans); err != nil {
+		return err
+	}
+
+	if len(matchers) > 0 && !matched {
+		return fmt.Errorf("cannot dump '%s': No matches for pattern", displayTarget)
+	}
+	return nil
 }
 
 func init() {
 	dumpCmd.Flags().StringP("encoding", "e", "auto", "value encoding: auto, base64, or text")
 	dumpCmd.Flags().Bool("secret", false, "Include entries marked as secret")
+	dumpCmd.Flags().StringSliceP("glob", "g", nil, "Filter keys with glob pattern (repeatable)")
+	dumpCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay()))
 	rootCmd.AddCommand(dumpCmd)
 }
 

cmd/glob.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"strings"
 
+	"github.com/gobwas/glob"
 	"github.com/spf13/cobra"
 )
 
@@ -26,3 +27,28 @@ func parseGlobSeparators(cmd *cobra.Command) ([]rune, error) {
 	}
 	return []rune(sepStr), nil
 }
+
+func compileGlobMatchers(patterns []string, separators []rune) ([]glob.Glob, error) {
+	var matchers []glob.Glob
+	for _, pattern := range patterns {
+		m, err := glob.Compile(strings.ToLower(pattern), separators...)
+		if err != nil {
+			return nil, err
+		}
+		matchers = append(matchers, m)
+	}
+	return matchers, nil
+}
+
+func globMatch(matchers []glob.Glob, key string) bool {
+	if len(matchers) == 0 {
+		return true
+	}
+	lowered := strings.ToLower(key)
+	for _, m := range matchers {
+		if m.Match(lowered) {
+			return true
+		}
+	}
+	return false
+}

cmd/list.go

@@ -24,10 +24,8 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"strings"
 
 	"github.com/dgraph-io/badger/v4"
-	"github.com/gobwas/glob"
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/spf13/cobra"
 )
@@ -73,25 +71,9 @@ func list(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
 	}
-	var matchers []glob.Glob
+	matchers, err := compileGlobMatchers(globPatterns, separators)
-	for _, pattern := range globPatterns {
+	if err != nil {
-		m, err := glob.Compile(strings.ToLower(pattern), separators...)
+		return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
-		if err != nil {
-			return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
-		}
-		matchers = append(matchers, m)
-	}
-
-	matchesKey := func(k string) bool {
-		if len(matchers) == 0 {
-			return true
-		}
-		for _, m := range matchers {
-			if m.Match(k) {
-				return true
-			}
-		}
-		return false
 	}
 
 	columnKinds, err := requireColumns(flags)
@@ -135,7 +117,7 @@ func list(cmd *cobra.Command, args []string) error {
 			for it.Rewind(); it.Valid(); it.Next() {
 				item := it.Item()
 				key := string(item.KeyCopy(nil))
-				if !matchesKey(key) {
+				if !globMatch(matchers, key) {
 					continue
 				}
 				matchedCount++

cmd/restore.go

@@ -32,10 +32,24 @@ func restore(cmd *cobra.Command, args []string) error {
 		}
 		dbName = parsed
 	}
+	displayTarget := "@" + dbName
+
+	globPatterns, err := cmd.Flags().GetStringSlice("glob")
+	if err != nil {
+		return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
+	}
+	separators, err := parseGlobSeparators(cmd)
+	if err != nil {
+		return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
+	}
+	matchers, err := compileGlobMatchers(globPatterns, separators)
+	if err != nil {
+		return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
+	}
 
 	reader, closer, err := restoreInput(cmd)
 	if err != nil {
-		return fmt.Errorf("cannot restore '%s': %v", dbName, err)
+		return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
 	}
 	if closer != nil {
 		defer closer.Close()
@@ -43,38 +57,38 @@ func restore(cmd *cobra.Command, args []string) error {
 
 	db, err := store.open(dbName)
 	if err != nil {
-		return fmt.Errorf("cannot restore '%s': %v", dbName, err)
+		return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
 	}
 	defer db.Close()
 
-	scanner := bufio.NewScanner(reader)
+	decoder := json.NewDecoder(bufio.NewReaderSize(reader, 8*1024*1024))
-	buf := make([]byte, 1024*1024)
-	scanner.Buffer(buf, 8*1024*1024)
 
 	wb := db.NewWriteBatch()
 	defer wb.Cancel()
 
-	lineNo := 0
+	entryNo := 0
 	var restored int
+	var matched bool
 
-	for scanner.Scan() {
+	for {
-		lineNo++
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" {
-			continue
-		}
-
 		var entry dumpEntry
-		if err := json.Unmarshal([]byte(line), &entry); err != nil {
+		if err := decoder.Decode(&entry); err != nil {
-			return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo+1, err)
 		}
+		entryNo++
 		if entry.Key == "" {
-			return fmt.Errorf("cannot restore '%s': line %d: missing key", dbName, lineNo)
+			return fmt.Errorf("cannot restore '%s': entry %d: missing key", displayTarget, entryNo)
+		}
+		if !globMatch(matchers, entry.Key) {
+			continue
 		}
 
 		value, err := decodeEntryValue(entry)
 		if err != nil {
-			return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
+			return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err)
 		}
 
 		entryMeta := byte(0x0)
@@ -85,23 +99,24 @@ func restore(cmd *cobra.Command, args []string) error {
 		writeEntry := badger.NewEntry([]byte(entry.Key), value).WithMeta(entryMeta)
 		if entry.ExpiresAt != nil {
 			if *entry.ExpiresAt < 0 {
-				return fmt.Errorf("cannot restore '%s': line %d: expires_at must be >= 0", dbName, lineNo)
+				return fmt.Errorf("cannot restore '%s': entry %d: expires_at must be >= 0", displayTarget, entryNo)
 			}
 			writeEntry.ExpiresAt = uint64(*entry.ExpiresAt)
 		}
 
 		if err := wb.SetEntry(writeEntry); err != nil {
-			return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
+			return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err)
 		}
 		restored++
-	}
+		matched = true
-
-	if err := scanner.Err(); err != nil {
-		return err
 	}
 
 	if err := wb.Flush(); err != nil {
-		return fmt.Errorf("cannot restore '%s': %v", dbName, err)
+		return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
+	}
+
+	if len(matchers) > 0 && !matched {
+		return fmt.Errorf("cannot restore '%s': No matches for pattern", displayTarget)
 	}
 
 	fmt.Fprintf(cmd.ErrOrStderr(), "Restored %d entries into @%s\n", restored, dbName)
@@ -140,5 +155,7 @@ func decodeEntryValue(entry dumpEntry) ([]byte, error) {
 
 func init() {
 	restoreCmd.Flags().StringP("file", "f", "", "Path to an NDJSON dump (defaults to stdin)")
+	restoreCmd.Flags().StringSliceP("glob", "g", nil, "Restore keys matching glob pattern (repeatable)")
+	restoreCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay()))
 	rootCmd.AddCommand(restoreCmd)
 }

testdata/dump__glob__ok.ct

@@ -0,0 +1,8 @@
+$ pda set a1 1
+$ pda set a2 2
+$ pda set b1 3
+$ pda dump --glob a*
+{"key":"a1","value":"1","encoding":"text"}
+{"key":"a2","value":"2","encoding":"text"}
+$ pda dump --glob c* --> FAIL
+Error: cannot dump '@default': No matches for pattern

testdata/help__dump__ok.ct

@@ -10,6 +10,8 @@ Aliases:
 
 Flags:
   -e, --encoding string   value encoding: auto, base64, or text (default "auto")
+  -g, --glob strings      Filter keys with glob pattern (repeatable)
+      --glob-sep string   Characters treated as separators for globbing (default "/-_.@: ")
   -h, --help              help for dump
       --secret            Include entries marked as secret
 Dump all key/value pairs as NDJSON
@@ -22,5 +24,7 @@ Aliases:
 
 Flags:
   -e, --encoding string   value encoding: auto, base64, or text (default "auto")
+  -g, --glob strings      Filter keys with glob pattern (repeatable)
+      --glob-sep string   Characters treated as separators for globbing (default "/-_.@: ")
   -h, --help              help for dump
       --secret            Include entries marked as secret

testdata/help__restore__ok.ct

@@ -9,8 +9,10 @@ Aliases:
   restore, import
 
 Flags:
-  -f, --file string   Path to an NDJSON dump (defaults to stdin)
+  -f, --file string       Path to an NDJSON dump (defaults to stdin)
-  -h, --help          help for restore
+  -g, --glob strings      Restore keys matching glob pattern (repeatable)
+      --glob-sep string   Characters treated as separators for globbing (default "/-_.@: ")
+  -h, --help              help for restore
 Restore key/value pairs from an NDJSON dump
 
 Usage:
@@ -20,5 +22,7 @@ Aliases:
   restore, import
 
 Flags:
-  -f, --file string   Path to an NDJSON dump (defaults to stdin)
+  -f, --file string       Path to an NDJSON dump (defaults to stdin)
-  -h, --help          help for restore
+  -g, --glob strings      Restore keys matching glob pattern (repeatable)
+      --glob-sep string   Characters treated as separators for globbing (default "/-_.@: ")
+  -h, --help              help for restore

testdata/restore__glob__ok.ct

@@ -0,0 +1,15 @@
+$ pda set a1 1
+$ pda set a2 2
+$ pda set b1 3
+$ fecho dumpfile {"key":"a1","value":"1","encoding":"text"} {"key":"a2","value":"2","encoding":"text"} {"key":"b1","value":"3","encoding":"text"}
+$ pda del a1 a2 b1 --force
+$ pda restore --glob a* --file dumpfile
+Restored 2 entries into @default
+$ pda get a1
+1
+$ pda get a2
+2
+$ pda get b1 --> FAIL
+Error: cannot get 'b1': Key not found
+$ pda restore --glob c* --file dumpfile --> FAIL
+Error: cannot restore '@default': No matches for pattern