lew/pda
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(globs): glob support for dump/restore, extracts some shared logic

Browse source
This commit is contained in:
Lewis Wynne 2025-12-17 22:18:15 +00:00
parent 9869b663e2
commit 7890e9451d
9 changed files with 141 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

40
cmd/dump.go
View file

@ -31,6 +31,7 @@ var dumpCmd = &cobra.Command{
func dump(cmd *cobra.Command, args []string) error {
store := &Store{}
targetDB := "@default"
displayTarget := targetDB
if len(args) == 1 {
rawArg := args[0]
dbName, err := store.parseDB(rawArg, false)
@ -45,23 +46,37 @@ func dump(cmd *cobra.Command, args []string) error {
return err
}
targetDB = "@" + dbName
displayTarget = targetDB
}
mode, err := cmd.Flags().GetString("encoding")
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
switch mode {
case "auto", "base64", "text":
default:
return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", args[0], mode)
return fmt.Errorf("cannot dump '%s': unsupported encoding '%s'", displayTarget, mode)
}
includeSecret, err := cmd.Flags().GetBool("secret")
if err != nil {
return err
}
globPatterns, err := cmd.Flags().GetStringSlice("glob")
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
separators, err := parseGlobSeparators(cmd)
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
matchers, err := compileGlobMatchers(globPatterns, separators)
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
var matched bool
trans := TransactionArgs{
key: targetDB,
readonly: true,
@ -74,6 +89,9 @@ func dump(cmd *cobra.Command, args []string) error {
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
key := item.KeyCopy(nil)
if !globMatch(matchers, string(key)) {
continue
}
meta := item.UserMeta()
isSecret := meta&metaSecret != 0
if isSecret && !includeSecret {
@ -94,7 +112,7 @@ func dump(cmd *cobra.Command, args []string) error {
encodeBase64(&entry, v)
case "text":
if err := encodeText(&entry, key, v); err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
case "auto":
if utf8.Valid(v) {
@ -106,24 +124,34 @@ func dump(cmd *cobra.Command, args []string) error {
}
payload, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
fmt.Fprintln(cmd.OutOrStdout(), string(payload))
matched = true
return nil
}); err != nil {
return fmt.Errorf("cannot dump '%s': %v", args[0], err)
return fmt.Errorf("cannot dump '%s': %v", displayTarget, err)
}
}
return nil
},
}
return store.Transaction(trans)
if err := store.Transaction(trans); err != nil {
return err
}
if len(matchers) > 0 && !matched {
return fmt.Errorf("cannot dump '%s': No matches for pattern", displayTarget)
}
return nil
}
func init() {
dumpCmd.Flags().StringP("encoding", "e", "auto", "value encoding: auto, base64, or text")
dumpCmd.Flags().Bool("secret", false, "Include entries marked as secret")
dumpCmd.Flags().StringSliceP("glob", "g", nil, "Filter keys with glob pattern (repeatable)")
dumpCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay()))
rootCmd.AddCommand(dumpCmd)
}

26
cmd/glob.go
View file

@ -3,6 +3,7 @@ package cmd
import (
"strings"
"github.com/gobwas/glob"
"github.com/spf13/cobra"
)
@ -26,3 +27,28 @@ func parseGlobSeparators(cmd *cobra.Command) ([]rune, error) {
}
return []rune(sepStr), nil
}
func compileGlobMatchers(patterns []string, separators []rune) ([]glob.Glob, error) {
var matchers []glob.Glob
for _, pattern := range patterns {
m, err := glob.Compile(strings.ToLower(pattern), separators...)
if err != nil {
return nil, err
}
matchers = append(matchers, m)
}
return matchers, nil
}
func globMatch(matchers []glob.Glob, key string) bool {
if len(matchers) == 0 {
return true
}
lowered := strings.ToLower(key)
for _, m := range matchers {
if m.Match(lowered) {
return true
}
}
return false
}

26
cmd/list.go
View file

@ -24,10 +24,8 @@ package cmd
import (
"errors"
"fmt"
"strings"
"github.com/dgraph-io/badger/v4"
"github.com/gobwas/glob"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
)
@ -73,25 +71,9 @@ func list(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
}
var matchers []glob.Glob
for _, pattern := range globPatterns {
m, err := glob.Compile(strings.ToLower(pattern), separators...)
if err != nil {
return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
}
matchers = append(matchers, m)
}
matchesKey := func(k string) bool {
if len(matchers) == 0 {
return true
}
for _, m := range matchers {
if m.Match(k) {
return true
}
}
return false
matchers, err := compileGlobMatchers(globPatterns, separators)
if err != nil {
return fmt.Errorf("cannot ls '%s': %v", targetDB, err)
}
columnKinds, err := requireColumns(flags)
@ -135,7 +117,7 @@ func list(cmd *cobra.Command, args []string) error {
for it.Rewind(); it.Valid(); it.Next() {
item := it.Item()
key := string(item.KeyCopy(nil))
if !matchesKey(key) {
if !globMatch(matchers, key) {
continue
}
matchedCount++

65
cmd/restore.go
View file

@ -32,10 +32,24 @@ func restore(cmd *cobra.Command, args []string) error {
}
dbName = parsed
}
displayTarget := "@" + dbName
globPatterns, err := cmd.Flags().GetStringSlice("glob")
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
separators, err := parseGlobSeparators(cmd)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
matchers, err := compileGlobMatchers(globPatterns, separators)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
reader, closer, err := restoreInput(cmd)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", dbName, err)
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
if closer != nil {
defer closer.Close()
@ -43,38 +57,38 @@ func restore(cmd *cobra.Command, args []string) error {
db, err := store.open(dbName)
if err != nil {
return fmt.Errorf("cannot restore '%s': %v", dbName, err)
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
defer db.Close()
scanner := bufio.NewScanner(reader)
buf := make([]byte, 1024*1024)
scanner.Buffer(buf, 8*1024*1024)
decoder := json.NewDecoder(bufio.NewReaderSize(reader, 8*1024*1024))
wb := db.NewWriteBatch()
defer wb.Cancel()
lineNo := 0
entryNo := 0
var restored int
var matched bool
for scanner.Scan() {
lineNo++
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
for {
var entry dumpEntry
if err := json.Unmarshal([]byte(line), &entry); err != nil {
return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
if err := decoder.Decode(&entry); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo+1, err)
}
entryNo++
if entry.Key == "" {
return fmt.Errorf("cannot restore '%s': line %d: missing key", dbName, lineNo)
return fmt.Errorf("cannot restore '%s': entry %d: missing key", displayTarget, entryNo)
}
if !globMatch(matchers, entry.Key) {
continue
}
value, err := decodeEntryValue(entry)
if err != nil {
return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err)
}
entryMeta := byte(0x0)
@ -85,23 +99,24 @@ func restore(cmd *cobra.Command, args []string) error {
writeEntry := badger.NewEntry([]byte(entry.Key), value).WithMeta(entryMeta)
if entry.ExpiresAt != nil {
if *entry.ExpiresAt < 0 {
return fmt.Errorf("cannot restore '%s': line %d: expires_at must be >= 0", dbName, lineNo)
return fmt.Errorf("cannot restore '%s': entry %d: expires_at must be >= 0", displayTarget, entryNo)
}
writeEntry.ExpiresAt = uint64(*entry.ExpiresAt)
}
if err := wb.SetEntry(writeEntry); err != nil {
return fmt.Errorf("cannot restore '%s': line %d: %w", dbName, lineNo, err)
return fmt.Errorf("cannot restore '%s': entry %d: %w", displayTarget, entryNo, err)
}
restored++
}
if err := scanner.Err(); err != nil {
return err
matched = true
}
if err := wb.Flush(); err != nil {
return fmt.Errorf("cannot restore '%s': %v", dbName, err)
return fmt.Errorf("cannot restore '%s': %v", displayTarget, err)
}
if len(matchers) > 0 && !matched {
return fmt.Errorf("cannot restore '%s': No matches for pattern", displayTarget)
}
fmt.Fprintf(cmd.ErrOrStderr(), "Restored %d entries into @%s\n", restored, dbName)
@ -140,5 +155,7 @@ func decodeEntryValue(entry dumpEntry) ([]byte, error) {
func init() {
restoreCmd.Flags().StringP("file", "f", "", "Path to an NDJSON dump (defaults to stdin)")
restoreCmd.Flags().StringSliceP("glob", "g", nil, "Restore keys matching glob pattern (repeatable)")
restoreCmd.Flags().String("glob-sep", "", fmt.Sprintf("Characters treated as separators for globbing (default %q)", defaultGlobSeparatorsDisplay()))
rootCmd.AddCommand(restoreCmd)
}