diff --git a/cmd/commands.go b/cmd/commands.go index 43a6edb0..6c43c8e2 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -12,6 +12,7 @@ var commands []*cli.Command = []*cli.Command{ filterCommand(), histogramCommand(), heatmapCommand(), + sparkCommand(), bargraphCommand(), analyzeCommand(), tabulateCommand(), diff --git a/cmd/spark.go b/cmd/spark.go new file mode 100644 index 00000000..6016eb08 --- /dev/null +++ b/cmd/spark.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "fmt" + "rare/cmd/helpers" + "rare/pkg/aggregation" + "rare/pkg/color" + "rare/pkg/csv" + "rare/pkg/expressions" + "rare/pkg/multiterm" + "rare/pkg/multiterm/termrenderers" + + "github.com/urfave/cli/v2" +) + +func sparkFunction(c *cli.Context) error { + var ( + delim = c.String("delim") + numRows = c.Int("num") + numCols = c.Int("cols") + noTruncate = c.Bool("notruncate") + scalerName = c.String(helpers.ScaleFlag.Name) + sortRows = c.String("sort-rows") + sortCols = c.String("sort-cols") + ) + + counter := aggregation.NewTable(delim) + + batcher := helpers.BuildBatcherFromArguments(c) + ext := helpers.BuildExtractorFromArguments(c, batcher) + rowSorter := helpers.BuildSorterOrFail(sortRows) + colSorter := helpers.BuildSorterOrFail(sortCols) + + vt := helpers.BuildVTermFromArguments(c) + writer := termrenderers.NewSpark(vt, numRows, numCols) + writer.Scaler = helpers.BuildScalerOrFail(scalerName) + + helpers.RunAggregationLoop(ext, counter, func() { + + // Trim unused data from the data store (keep memory tidy!) + if !noTruncate { + if keepCols := counter.OrderedColumns(colSorter); len(keepCols) > numCols { + keepCols = keepCols[len(keepCols)-numCols:] + keepLookup := make(map[string]struct{}) + for _, item := range keepCols { + keepLookup[item] = struct{}{} + } + counter.Trim(func(col, row string, val int64) bool { + _, ok := keepLookup[col] + return !ok + }) + } + } + + // Write spark + writer.WriteTable(counter, rowSorter, colSorter) + writer.WriteFooter(0, helpers.FWriteExtractorSummary(ext, counter.ParseErrors(), + fmt.Sprintf("(R: %v; C: %v)", color.Wrapi(color.Yellow, counter.RowCount()), color.Wrapi(color.BrightBlue, counter.ColumnCount())))) + writer.WriteFooter(1, batcher.StatusString()) + }) + + // Not deferred intentionally + writer.Close() + + if err := helpers.TryWriteCSV(c, counter, csv.WriteTable); err != nil { + return err + } + + return helpers.DetermineErrorState(batcher, ext, counter) +} + +func sparkCommand() *cli.Command { + return helpers.AdaptCommandForExtractor(cli.Command{ + Name: "spark", + Aliases: []string{"sparkline", "s"}, + Usage: "Create rows of sparkline graphs", + Description: `Create rows of a sparkkline graph, all scaled equally + based on a table like input`, + Category: cmdCatVisualize, + Action: sparkFunction, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "delim", + Usage: "Character to tabulate on. Use {$} helper by default", + Value: expressions.ArraySeparatorString, + }, + &cli.IntFlag{ + Name: "num", + Aliases: []string{"rows", "n"}, + Usage: "Number of elements (rows) to display", + Value: 20, + }, + &cli.IntFlag{ + Name: "cols", + Usage: "Number of columns to display", + Value: multiterm.TermCols() - 15, + }, + &cli.BoolFlag{ + Name: "notruncate", + Usage: "Disable truncating data that doesnt fit in the sparkline", + Value: false, + }, + &cli.StringFlag{ + Name: "sort-rows", + Usage: helpers.DefaultSortFlag.Usage, + Value: "value", + }, + &cli.StringFlag{ + Name: "sort-cols", + Usage: helpers.DefaultSortFlag.Usage, + Value: "numeric", + }, + helpers.SnapshotFlag, + helpers.NoOutFlag, + helpers.CSVFlag, + helpers.ScaleFlag, + }, + }) +} diff --git a/cmd/spark_test.go b/cmd/spark_test.go new file mode 100644 index 00000000..312ad5a9 --- /dev/null +++ b/cmd/spark_test.go @@ -0,0 +1,22 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSparkline(t *testing.T) { + testCommandSet(t, sparkCommand(), + `-m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`, + `-o - -m "(.+) (\d+)" -e "{$ {1} {2}}" testdata/graph.txt`, + ) +} + +func TestSparklineWithTrim(t *testing.T) { + out, eout, err := testCommandCapture(sparkCommand(), `--snapshot -m "(.+) (.+)" -e {1} -e {2} --cols 2 testdata/heat.txt`) + + assert.NoError(t, err) + assert.Empty(t, eout) + assert.Contains(t, out, " First bc Last \ny 1 _█ 2 \nx 1 __ 1 \nMatched: 10 / 10 (R: 2; C: 2)") +} diff --git a/docs/images/README.md b/docs/images/README.md index c716d337..85ddd1b1 100644 --- a/docs/images/README.md +++ b/docs/images/README.md @@ -6,11 +6,12 @@ nvm use --lts npm install -g terminalizer terminalizer record -k output.yml +# Do any yaml cleanup/delays terminalizer render -o temp.gif output.yml gifsicle -O3 --colors 128 -i temp.gif -o output.gif ``` -Note on environment; Make sure bashrc when terminalizer starts is set by changing `command:` in config yaml +Note on environment; Make sure bashrc when terminalizer starts is set by changing `command: bash --rcfile ~/terminalizer/bashrc` in config yaml ```bash export PS1="$ " export PATH="./:$PATH" @@ -62,6 +63,10 @@ rare table -m '\[(.+?)\].*" (\d+)' -e '{buckettime {1} year}' -e '{2}' access.lo rare heatmap -m '\[(.+?)\].*" (\d+)' -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log +### Sparkline + +rare spark -m '\[(.+?)\].*" (\d+)' -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log + ### Analyze bytes sent, only looking at 200's rare analyze -m '(\d{3}) (\d+)' -e '{2}' -i '{neq {1} 200}' access.log diff --git a/docs/images/rare-spark.gif b/docs/images/rare-spark.gif new file mode 100644 index 00000000..a1e30430 Binary files /dev/null and b/docs/images/rare-spark.gif differ diff --git a/docs/usage/aggregators.md b/docs/usage/aggregators.md index 6c42f62d..b91d0026 100644 --- a/docs/usage/aggregators.md +++ b/docs/usage/aggregators.md @@ -217,6 +217,41 @@ Matched: 1,035,666 / 1,035,666 (R: 8; C: 61) ![Gif of heatmap](../images/heatmap.gif) +## Sparkline + +``` +rare help sparkline +``` + +### Summary + +Creates one or more sparklines based on table-style input. Provide +multiple inputs using `{$ a b}` helper. Unlike other output styles, +columns in the spark graph are right-aligned to always show +the most recent data on the right side. + +Supports [alternative scales](#alternative-scales) + +### Example + +```bash +$ rare spark -m '\[(.+?)\].*" (\d+)' \ + -e "{timeattr {time {1}} yearweek}" -e "{2}" access.log + + First 2019-34................................................2020-9 Last +404 15,396 ..._._-.^_._.._..________.____.__.___.____________.______.___ 5,946 +200 7,146 _____________________________________________________________ 4,938 +400 162 _____________________________________________________________ 522 +405 6 _____________________________________________________________ 6 +408 0 _____________________________________________________________ 6 +304 0 _____________________________________________________________ 0 +301 6 _____________________________________________________________ 0 +206 0 _____________________________________________________________ 0 +Matched: 1,034,166 / 1,034,166 (R: 8; C: 61) +``` + +![Gif of sparkline](../images/rare-spark.gif) + ## Reduce ``` diff --git a/pkg/aggregation/table.go b/pkg/aggregation/table.go index 0906edb4..48553dde 100644 --- a/pkg/aggregation/table.go +++ b/pkg/aggregation/table.go @@ -120,32 +120,26 @@ func (s *TableAggregator) OrderedRows(sorter sorting.NameValueSorter) []*TableRo return rows } -func (s *TableAggregator) ComputeMin() (ret int64) { - ret = math.MaxInt64 +func (s *TableAggregator) ComputeMinMax() (min, max int64) { + min, max = math.MaxInt64, math.MinInt64 + for _, r := range s.rows { for colKey := range s.cols { - if val := r.cols[colKey]; val < ret { - ret = val + val := r.cols[colKey] + if val < min { + min = val + } + if val > max { + max = val } } } - if ret == math.MaxInt64 { - return 0 - } - return -} -func (s *TableAggregator) ComputeMax() (ret int64) { - ret = math.MinInt64 - for _, r := range s.rows { - for colKey := range s.cols { - if val := r.cols[colKey]; val > ret { - ret = val - } - } + if min == math.MaxInt64 { + min = 0 } - if ret == math.MinInt64 { - return 0 + if max == math.MinInt64 { + max = 0 } return } @@ -155,6 +149,7 @@ func (s *TableAggregator) ColTotal(k string) int64 { return s.cols[k] } +// Sum all data func (s *TableAggregator) Sum() (ret int64) { for _, v := range s.cols { ret += v @@ -162,6 +157,34 @@ func (s *TableAggregator) Sum() (ret int64) { return } +// Trim data. Returns number of fields trimmed +func (s *TableAggregator) Trim(predicate func(col, row string, val int64) bool) int { + trimmed := 0 + + for colName := range s.cols { + + removeAllInCol := true + for rowName, row := range s.rows { + if predicate(colName, rowName, row.cols[colName]) { + delete(row.cols, colName) + trimmed++ + } else { + removeAllInCol = false + } + + if len(row.cols) == 0 { + delete(s.rows, rowName) + } + } + + if removeAllInCol { + delete(s.cols, colName) + } + } + + return trimmed +} + func (s *TableRow) Name() string { return s.name } diff --git a/pkg/aggregation/table_test.go b/pkg/aggregation/table_test.go index f8710ea3..3e25ef65 100644 --- a/pkg/aggregation/table_test.go +++ b/pkg/aggregation/table_test.go @@ -1,7 +1,9 @@ package aggregation import ( + "fmt" "rare/pkg/aggregation/sorting" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -44,8 +46,9 @@ func TestSimpleTable(t *testing.T) { assert.Equal(t, int64(5), table.Sum()) // Minmax - assert.Equal(t, int64(0), table.ComputeMin()) - assert.Equal(t, int64(3), table.ComputeMax()) + min, max := table.ComputeMinMax() + assert.Equal(t, int64(0), min) + assert.Equal(t, int64(3), max) } func TestTableMultiIncrement(t *testing.T) { @@ -72,8 +75,16 @@ func TestTableMultiIncrement(t *testing.T) { assert.Equal(t, int64(6), table.Sum()) // Minmax - assert.Equal(t, int64(0), table.ComputeMin()) - assert.Equal(t, int64(5), table.ComputeMax()) + min, max := table.ComputeMinMax() + assert.Equal(t, int64(0), min) + assert.Equal(t, int64(5), max) +} + +func TestEmptyTableMinMax(t *testing.T) { + table := NewTable(" ") + min, max := table.ComputeMinMax() + assert.Equal(t, int64(0), min) + assert.Equal(t, int64(0), max) } func TestSingleRowTable(t *testing.T) { @@ -91,3 +102,39 @@ func TestSingleRowTable(t *testing.T) { assert.Equal(t, int64(2), rows[0].Value("a")) assert.Equal(t, int64(1), rows[0].Value("b")) } + +func TestTrimData(t *testing.T) { + table := NewTable(" ") + for i := 0; i < 10; i++ { + table.Sample(fmt.Sprintf("%d a", i)) + table.Sample(fmt.Sprintf("%d b", i)) + } + + assert.Len(t, table.Columns(), 10) + + trimmed := table.Trim(func(col, row string, val int64) bool { + if row == "b" { + return true + } + cVal, _ := strconv.Atoi(col) + return cVal < 5 + }) + + assert.ElementsMatch(t, []string{"5", "6", "7", "8", "9"}, table.Columns()) + assert.Equal(t, 15, trimmed) + assert.Len(t, table.Rows(), 1) + assert.Len(t, table.Rows()[0].cols, 5) +} + +// BenchmarkMinMax-4 1020728 1234 ns/op 0 B/op 0 allocs/op +func BenchmarkMinMax(b *testing.B) { + table := NewTable(" ") + for i := 0; i < 10; i++ { + table.Sample(fmt.Sprintf("%d a", i)) + table.Sample(fmt.Sprintf("%d b", i)) + } + + for i := 0; i < b.N; i++ { + table.ComputeMinMax() + } +} diff --git a/pkg/multiterm/termrenderers/heatmap.go b/pkg/multiterm/termrenderers/heatmap.go index 92771764..3d766278 100644 --- a/pkg/multiterm/termrenderers/heatmap.go +++ b/pkg/multiterm/termrenderers/heatmap.go @@ -61,12 +61,16 @@ func (s *Heatmap) WriteFooter(idx int, line string) { func (s *Heatmap) UpdateMinMaxFromData(agg *aggregation.TableAggregator) { min := s.minVal - if !s.FixedMin { - min = agg.ComputeMin() - } max := s.maxVal - if !s.FixedMax { - max = agg.ComputeMax() + + if !s.FixedMin || !s.FixedMax { + tableMin, tableMax := agg.ComputeMinMax() + if !s.FixedMin { + min = tableMin + } + if !s.FixedMax { + max = tableMax + } } s.UpdateMinMax(min, max) diff --git a/pkg/multiterm/termrenderers/spark.go b/pkg/multiterm/termrenderers/spark.go new file mode 100644 index 00000000..693ce96c --- /dev/null +++ b/pkg/multiterm/termrenderers/spark.go @@ -0,0 +1,88 @@ +package termrenderers + +import ( + "rare/pkg/aggregation" + "rare/pkg/aggregation/sorting" + "rare/pkg/color" + "rare/pkg/humanize" + "rare/pkg/multiterm" + "rare/pkg/multiterm/termscaler" + "rare/pkg/multiterm/termunicode" + "strings" +) + +type Spark struct { + rowCount, colCount int + footerOffset int + Scaler termscaler.Scaler + table *TableWriter +} + +func NewSpark(term multiterm.MultilineTerm, rows, cols int) *Spark { + return &Spark{ + rowCount: rows, + colCount: cols, + Scaler: termscaler.ScalerLinear, + table: NewTable(term, 4, rows+1), + } +} + +func (s *Spark) WriteTable(agg *aggregation.TableAggregator, rowSorter, colSorter sorting.NameValueSorter) { + minVal, maxVal := agg.ComputeMinMax() + + colNames := agg.OrderedColumns(colSorter) + if len(colNames) > s.colCount { + colNames = colNames[len(colNames)-s.colCount:] + } + + // reused buffer + var sb strings.Builder + sb.Grow(len(colNames)) + + // Write header + { + dots := len(colNames) - len(colNames[0]) - len(colNames[len(colNames)-1]) + if dots < 0 { + dots = 0 + } + sb.WriteString(colNames[0]) + writeRepeat(&sb, '.', dots) + sb.WriteString(colNames[len(colNames)-1]) + + s.table.WriteRow(0, "", color.Wrap(color.Underline, "First"), sb.String(), color.Wrap(color.Underline, "Last")) + sb.Reset() + } + + // Each row... + rows := agg.OrderedRows(rowSorter) + rowCount := mini(len(rows), s.rowCount) + for i := 0; i < rowCount; i++ { + row := rows[i] + + for j := 0; j < len(colNames); j++ { + termunicode.SparkWrite(&sb, s.Scaler.Scale(row.Value(colNames[j]), minVal, maxVal)) + } + + vFirst := humanize.Hi(row.Value(colNames[0])) + vLast := humanize.Hi(row.Value(colNames[len(colNames)-1])) + s.table.WriteRow(i+1, color.Wrap(color.Yellow, row.Name()), color.Wrap(color.BrightBlack, vFirst), sb.String(), color.Wrap(color.BrightBlack, vLast)) + + sb.Reset() + } + + // If more rows than can display, write how many were missed + if len(rows) > rowCount { + s.table.WriteFooter(0, color.Wrapf(color.BrightBlack, "(%d more)", len(rows)-rowCount)) + s.footerOffset = 1 + } else { + s.footerOffset = 0 + } +} + +func (s *Spark) Close() { + s.table.Close() +} + +func (s *Spark) WriteFooter(idx int, line string) { + s.table.WriteFooter(s.footerOffset+idx, line) +} diff --git a/pkg/multiterm/termrenderers/spark_test.go b/pkg/multiterm/termrenderers/spark_test.go new file mode 100644 index 00000000..038ac363 --- /dev/null +++ b/pkg/multiterm/termrenderers/spark_test.go @@ -0,0 +1,56 @@ +package termrenderers + +import ( + "rare/pkg/aggregation" + "rare/pkg/aggregation/sorting" + "rare/pkg/multiterm" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSimpleSpark(t *testing.T) { + vt := multiterm.NewVirtualTerm() + s := NewSpark(vt, 2, 2) + + agg := aggregation.NewTable(" ") + agg.Sample("a 1") + agg.Sample("a 2") + + s.WriteTable(agg, sorting.NVNameSorter, sorting.NVNameSorter) + s.WriteFooter(0, "hello") + + assert.Equal(t, " First aa Last ", vt.Get(0)) + assert.Equal(t, "1 1 _ 1 ", vt.Get(1)) + assert.Equal(t, "2 1 _ 1 ", vt.Get(2)) + assert.Equal(t, "hello", vt.Get(3)) + assert.Equal(t, "", vt.Get(4)) + + s.Close() + assert.True(t, vt.IsClosed()) +} + +func TestOverflowSpark(t *testing.T) { + vt := multiterm.NewVirtualTerm() + s := NewSpark(vt, 2, 2) + + agg := aggregation.NewTable(" ") + agg.Sample("1 a") + agg.Sample("2 a") + agg.Sample("2 b") + agg.Sample("2 b") + agg.Sample("1 c") + + s.WriteTable(agg, sorting.NVNameSorter, sorting.NVNameSorter) + s.WriteFooter(0, "hello") + + assert.Equal(t, " First 12 Last ", vt.Get(0)) + assert.Equal(t, "a 1 ▄▄ 1 ", vt.Get(1)) + assert.Equal(t, "b 0 _█ 2 ", vt.Get(2)) + assert.Equal(t, "(1 more)", vt.Get(3)) + assert.Equal(t, "hello", vt.Get(4)) + assert.Equal(t, "", vt.Get(5)) + + s.Close() + assert.True(t, vt.IsClosed()) +} diff --git a/pkg/multiterm/termunicode/spark.go b/pkg/multiterm/termunicode/spark.go new file mode 100644 index 00000000..95d7d9d2 --- /dev/null +++ b/pkg/multiterm/termunicode/spark.go @@ -0,0 +1,32 @@ +package termunicode + +import ( + "io" + "rare/pkg/multiterm/termscaler" +) + +var sparkBlocks = [...]rune{ + '_', + '\u2581', + '\u2582', + '\u2583', + '\u2584', + '\u2585', + '\u2586', + '\u2587', + '\u2588', +} + +var sparkAscii = [...]rune{ + '_', '.', '-', '^', +} + +func SparkWrite(w io.StringWriter, scaled float64) { + if !UnicodeEnabled { + var blockChar = termscaler.Bucket(len(sparkAscii), scaled) + w.WriteString(string(sparkAscii[blockChar])) + } else { + var blockChar = termscaler.Bucket(len(sparkBlocks), scaled) + w.WriteString(string(sparkBlocks[blockChar])) + } +} diff --git a/pkg/multiterm/termunicode/spark_test.go b/pkg/multiterm/termunicode/spark_test.go new file mode 100644 index 00000000..ee244617 --- /dev/null +++ b/pkg/multiterm/termunicode/spark_test.go @@ -0,0 +1,46 @@ +package termunicode + +import ( + "rare/pkg/testutil" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSparkUnicode(t *testing.T) { + var sb strings.Builder + SparkWrite(&sb, 0.0) + SparkWrite(&sb, 0.1) + SparkWrite(&sb, 0.2) + SparkWrite(&sb, 0.3) + SparkWrite(&sb, 0.4) + SparkWrite(&sb, 0.5) + SparkWrite(&sb, 0.6) + SparkWrite(&sb, 0.7) + SparkWrite(&sb, 0.8) + SparkWrite(&sb, 0.9) + SparkWrite(&sb, 1.0) + + assert.Equal(t, "__▁▂▃▄▄▅▆▇█", sb.String()) +} + +func TestSparkAscii(t *testing.T) { + defer testutil.RestoreGlobals() + testutil.SwitchGlobal(&UnicodeEnabled, false) + + var sb strings.Builder + SparkWrite(&sb, 0.0) + SparkWrite(&sb, 0.1) + SparkWrite(&sb, 0.2) + SparkWrite(&sb, 0.3) + SparkWrite(&sb, 0.4) + SparkWrite(&sb, 0.5) + SparkWrite(&sb, 0.6) + SparkWrite(&sb, 0.7) + SparkWrite(&sb, 0.8) + SparkWrite(&sb, 0.9) + SparkWrite(&sb, 1.0) + + assert.Equal(t, "____...---^", sb.String()) +}