zix99 · zix99 · May 31, 2025 · May 27, 2025 · May 27, 2025 · May 27, 2025
diff --git a/acceptance.tests b/acceptance.tests
@@ -291,3 +291,10 @@ NAME rare version
 RUN -v
 # any output is okay
 END
+
+NAME walk
+RUN walk -R --include *.sh docs/
+    docs/install.sh
+STDERR
+    Found 1 path(s), 30 excluded
+END
diff --git a/cmd/commands.go b/cmd/commands.go
@@ -20,6 +20,7 @@ var commands []*cli.Command = []*cli.Command{
 	reduceCommand(),
 	docsCommand(),
 	expressionCommand(),
+	walkCommand(),
 }
 
 func GetSupportedCommands() []*cli.Command {

diff --git a/cmd/helpers/extractorBuilder.go b/cmd/helpers/extractorBuilder.go
@@ -182,43 +182,6 @@ func getExtractorFlags() []cli.Flag {
 	workerCount := runtime.NumCPU()/2 + 1
 
 	return []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:     "include",
-			Category: cliCategoryPath,
-			Usage:    "Glob file patterns to include (eg. *.txt)",
-			EnvVars:  []string{"RARE_INCLUDE"},
-		},
-		&cli.StringSliceFlag{
-			Name:     "exclude",
-			Category: cliCategoryPath,
-			Usage:    "Glob file patterns to exclude (eg. *.txt)",
-			EnvVars:  []string{"RARE_EXCLUDE"},
-		},
-		&cli.StringSliceFlag{
-			Name:     "exclude-dir",
-			Category: cliCategoryPath,
-			Usage:    "Glob file patterns to exclude directories",
-			EnvVars:  []string{"RARE_EXCLUDE_DIR"},
-		},
-		&cli.BoolFlag{
-			Name:     "follow-symlinks",
-			Aliases:  []string{"L"},
-			Category: cliCategoryPath,
-			Usage:    "Follow symbolic directory links",
-			EnvVars:  []string{"RARE_FOLLOW_SYMLINKS"},
-		},
-		&cli.BoolFlag{
-			Name:     "read-symlinks",
-			Category: cliCategoryPath,
-			Usage:    "Read files that are symbolic links",
-			Value:    true,
-		},
-		&cli.BoolFlag{
-			Name:     "mount",
-			Category: cliCategoryPath,
-			Usage:    "Don't descend directories on other filesystems (unix only)",
-			Hidden:   !dirwalk.FeatureMountTraversal,
-		},
 		&cli.BoolFlag{
 			Name:     "follow",
 			Aliases:  []string{"f"},
@@ -248,12 +211,6 @@ func getExtractorFlags() []cli.Flag {
 			Category: cliCategoryRead,
 			Usage:    "Attempt to decompress file when reading",
 		},
-		&cli.BoolFlag{
-			Name:     "recursive",
-			Aliases:  []string{"R"},
-			Category: cliCategoryRead,
-			Usage:    "Recursively walk a non-globbing path and search for plain-files",
-		},
 		&cli.BoolFlag{
 			Name:     "posix",
 			Aliases:  []string{"p"},
@@ -321,8 +278,57 @@ func getExtractorFlags() []cli.Flag {
 	}
 }
 
+func GetWalkerFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.BoolFlag{
+			Name:     "recursive",
+			Aliases:  []string{"R"},
+			Category: cliCategoryPath,
+			Usage:    "Recursively walk a non-globbing path and search for plain-files",
+		},
+		&cli.StringSliceFlag{
+			Name:     "include",
+			Category: cliCategoryPath,
+			Usage:    "Glob file patterns to include (eg. *.txt)",
+			EnvVars:  []string{"RARE_INCLUDE"},
+		},
+		&cli.StringSliceFlag{
+			Name:     "exclude",
+			Category: cliCategoryPath,
+			Usage:    "Glob file patterns to exclude (eg. *.txt)",
+			EnvVars:  []string{"RARE_EXCLUDE"},
+		},
+		&cli.StringSliceFlag{
+			Name:     "exclude-dir",
+			Category: cliCategoryPath,
+			Usage:    "Glob file patterns to exclude directories",
+			EnvVars:  []string{"RARE_EXCLUDE_DIR"},
+		},
+		&cli.BoolFlag{
+			Name:     "follow-symlinks",
+			Aliases:  []string{"L"},
+			Category: cliCategoryPath,
+			Usage:    "Follow symbolic directory links",
+			EnvVars:  []string{"RARE_FOLLOW_SYMLINKS"},
+		},
+		&cli.BoolFlag{
+			Name:     "read-symlinks",
+			Category: cliCategoryPath,
+			Usage:    "Read files that are symbolic links",
+			Value:    true,
+		},
+		&cli.BoolFlag{
+			Name:     "mount",
+			Category: cliCategoryPath,
+			Usage:    "Don't descend directories on other filesystems (unix only)",
+			Hidden:   !dirwalk.FeatureMountTraversal,
+		},
+	}
+}
+
 func AdaptCommandForExtractor(command cli.Command) *cli.Command {
 	command.Flags = append(getExtractorFlags(), command.Flags...)
+	command.Flags = append(GetWalkerFlags(), command.Flags...)
 	if command.ArgsUsage == "" {
 		command.ArgsUsage = DefaultArgumentDescriptor
 	}

diff --git a/cmd/walk.go b/cmd/walk.go
@@ -0,0 +1,52 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"rare/cmd/helpers"
+	"rare/pkg/color"
+	"rare/pkg/humanize"
+
+	"github.com/urfave/cli/v2"
+)
+
+func walkFunction(c *cli.Context) error {
+	walk := helpers.BuildPathWalkerFromArguments(c)
+	paths := c.Args().Slice()
+
+	stdout := bufio.NewWriter(os.Stdout)
+
+	for path := range walk.Walk(paths...) {
+		stdout.WriteString(path)
+		stdout.WriteRune('\n')
+	}
+	stdout.Flush()
+
+	if count := walk.TotalCount(); count > 0 {
+		fmt.Fprintf(os.Stderr, "Found %s path(s)", color.Wrap(color.BrightGreen, humanize.Hui(count)))
+		if excluded := walk.ExcludedCount(); excluded > 0 {
+			fmt.Fprintf(os.Stderr, ", %s excluded", color.Wrap(color.Yellow, humanize.Hui(excluded)))
+		}
+		if errors := walk.ErrorCount(); errors > 0 {
+			fmt.Fprintf(os.Stderr, ", %s error(s)", color.Wrap(color.Red, humanize.Hui(errors)))
+		}
+		fmt.Fprint(os.Stderr, "\n")
+	} else {
+		return cli.Exit("No paths found", helpers.ExitCodeNoData)
+	}
+
+	return nil
+}
+
+func walkCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "walk",
+		Usage:       "Output paths discovered via traverse rules",
+		Description: "Find files using include/exclude and traversal rules by outputting paths visited",
+		ArgsUsage:   "<paths...>",
+		Action:      walkFunction,
+		Category:    cmdCatHelp,
+		Flags:       helpers.GetWalkerFlags(),
+	}
+}
diff --git a/cmd/walk_test.go b/cmd/walk_test.go
@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEmptyWalkCommand(t *testing.T) {
+	o, e, err := testCommandCapture(walkCommand(), "")
+	assert.Error(t, err)
+	assert.Equal(t, "", o)
+	assert.Equal(t, "No paths found", e)
+}
+
+func TestWalkTestDataGlob(t *testing.T) {
+	o, e, err := testCommandCapture(walkCommand(), "testdata/*.gz")
+	assert.NoError(t, err)
+	assert.Equal(t, "testdata/log.txt.gz\n", o)
+	assert.Equal(t, "Found 1 path(s)\n", e)
+}
+
+func TestWalkTestDataRecursive(t *testing.T) {
+	o, e, err := testCommandCapture(walkCommand(), "-R testdata/")
+	assert.NoError(t, err)
+	assert.Contains(t, o, "log.txt")
+	assert.Equal(t, "Found 6 path(s)\n", e)
+}
diff --git a/docs/cli-help.md b/docs/cli-help.md
@@ -676,6 +676,24 @@ Evaluate and benchmark expressions
 
 **--stats, -s**: Display stats about the expression
 
+### walk
+
+Output paths discovered via traverse rules
+
+**--exclude**="": Glob file patterns to exclude (eg. *.txt)
+
+**--exclude-dir**="": Glob file patterns to exclude directories
+
+**--follow-symlinks, -L**: Follow symbolic directory links
+
+**--include**="": Glob file patterns to include (eg. *.txt)
+
+**--mount**: Don't descend directories on other filesystems (unix only)
+
+**--read-symlinks**: Read files that are symbolic links
+
+**--recursive, -R**: Recursively walk a non-globbing path and search for plain-files
+
 ### help, h
 
 Shows a list of commands or help for one command
diff --git a/docs/usage/input.md b/docs/usage/input.md
@@ -78,6 +78,21 @@ These options control how the directory space is traversed:
 - `--read-symlinks` Read files that are symbolic links (default: true)
 - `--mount` Don't descend directories on other filesystems (unix only)
 
+#### Testing
+
+You can test path and traversal filters using the `walk` command.
+
+For example:
+
+```sh
+rare walk -R --include '*txt' ./
+```
+
+Which will output matching paths.
+
+This command will output paths that match, regardless of whether
+they can be read, have proper permissions, or even exist.
+
 ### Following File(s)
 
 Like `tail -f`, following files allows you to watch files actively being written to. This is

diff --git a/pkg/extractor/dirwalk/globExpand.go b/pkg/extractor/dirwalk/globExpand.go
@@ -19,20 +19,32 @@ type Walker struct {
 	Recursive       bool // If asked to walk a path, will recurse
 	NoMountTraverse bool // If true, don't traverse into another mount
 
-	OnTraverseError func(error)
+	OnTraverseError func(error) // Called in separate goroutine
 
+	total    atomic.Uint64
 	excluded atomic.Uint64 // Files excluded due to include/exclude rules (not sym or mount rules)
+	error    atomic.Uint64
 }
 
 type Metrics interface {
+	TotalCount() uint64
 	ExcludedCount() uint64
+	ErrorCount() uint64
 }
 
 // Number of paths skipped because of rules (include, exclude, exludedir; NOT skip sym, mounts, etc)
 func (s *Walker) ExcludedCount() uint64 {
 	return s.excluded.Load()
 }
 
+func (s *Walker) TotalCount() uint64 {
+	return s.total.Load()
+}
+
+func (s *Walker) ErrorCount() uint64 {
+	return s.error.Load()
+}
+
 func (s *Walker) Walk(paths ...string) <-chan string {
 	c := make(chan string, 10)
 
@@ -107,13 +119,15 @@ func (s *Walker) recurseWalk(c chan<- string, p string, visited map[string]strin
 			}
 
 			c <- walkPath
+			s.total.Add(1)
 
 		case info.Type().IsRegular(): // regular file
 			if !s.shouldIncludeFilename(info.Name()) {
 				s.excluded.Add(1)
 				break
 			}
 			c <- walkPath
+			s.total.Add(1)
 		}
 		return nil
 	})
@@ -128,12 +142,14 @@ func (s *Walker) globExpand(c chan<- string, p string) {
 		for _, item := range expanded {
 			if s.shouldIncludeFilename(filepath.Base(item)) && s.shouldIncludeDir(item) {
 				c <- item
+				s.total.Add(1)
 			} else {
 				s.excluded.Add(1)
 			}
 		}
 	} else {
 		c <- p
+		s.total.Add(1)
 	}
 }
 
@@ -171,6 +187,7 @@ func (s *Walker) shouldIncludeDir(fullpath string) bool {
 }
 
 func (s *Walker) onError(err error) {
+	s.error.Add(1)
 	if s.OnTraverseError != nil {
 		s.OnTraverseError(err)
 	}

diff --git a/pkg/extractor/dirwalk/globExpand_test.go b/pkg/extractor/dirwalk/globExpand_test.go
@@ -25,6 +25,9 @@ func TestDefaultOnDir(t *testing.T) {
 
 	// Returns dir even though not file
 	assert.ElementsMatch(t, []string{"./"}, files)
+	assert.Equal(t, uint64(1), walk.TotalCount())
+	assert.Equal(t, uint64(0), walk.ErrorCount())
+	assert.Equal(t, uint64(0), walk.ExcludedCount())
 }
 
 func TestDefaultOnMissing(t *testing.T) {
@@ -34,6 +37,9 @@ func TestDefaultOnMissing(t *testing.T) {
 
 	// Returns dir even though not file
 	assert.ElementsMatch(t, []string{p}, files)
+	assert.Equal(t, uint64(1), walk.TotalCount())
+	assert.Equal(t, uint64(0), walk.ErrorCount())
+	assert.Equal(t, uint64(0), walk.ExcludedCount())
 }
 
 func TestDefaultOnGlob(t *testing.T) {
@@ -42,6 +48,9 @@ func TestDefaultOnGlob(t *testing.T) {
 	files := collectChan(walk.Walk(p))
 
 	assert.ElementsMatch(t, []string{"go.mod", "go.sum"}, files)
+	assert.Equal(t, uint64(2), walk.TotalCount())
+	assert.Equal(t, uint64(0), walk.ErrorCount())
+	assert.Equal(t, uint64(0), walk.ExcludedCount())
 }
 
 func TestGlobNoExist(t *testing.T) {
@@ -59,6 +68,9 @@ func TestGlobInclude(t *testing.T) {
 
 	assert.ElementsMatch(t, []string{"go.mod"}, files)
 	assert.Len(t, files, 1)
+	assert.Equal(t, uint64(1), walk.TotalCount())
+	assert.Equal(t, uint64(0), walk.ErrorCount())
+	assert.Equal(t, uint64(1), walk.ExcludedCount())
 }
 
 func TestGlobExclude(t *testing.T) {
@@ -223,6 +235,7 @@ func TestNoInfiniteRecursion(t *testing.T) {
 	assertNoneContains(t, files, "recursive")
 	assertNoneContains(t, files, "recursive2")
 	assert.Equal(t, uint64(0), walker.ExcludedCount())
+	assert.Equal(t, uint64(2), walker.ErrorCount())
 }
 
 func TestNoMountTraverseWithSymlink(t *testing.T) {