zix99 · zix99 · Dec 1, 2019 · Dec 1, 2019 · Dec 1, 2019 · Dec 1, 2019
diff --git a/docs/expressions.md b/docs/expressions.md
@@ -15,10 +15,18 @@ The basic syntax structure is as follows:
  * Anything not surrounded by `{}` is a literal
  * Expressions are surrounded by `{}`. The entire match will always be `{0}`
  * An integer in an expression denotes a matched value from the regex (or other input) eg. `{2}`
+ * A string in an expression is a special key eg. `{src}`
  * When an expression has space(s), the first literal will be the name of a helper function.
    From there, the logic is nested. eg `{coalesce {4} {3} notfound}`
  * Truthiness is the presence of a value.  False is an empty value (or only whitespace)
 
+## Special Keys
+
+The following are special Keys:
+
+ * `{src}`  The source name (eg filename). `stdin` when read from stdin
+ * `{line}` The line numbers of the current match
+
 ## Examples
 
 ### Parsing an nginx access.log file
@@ -105,6 +113,20 @@ Syntax: `{format "%s" ...}`
 
 Formats a string based on `fmt.Sprintf`
 
+## Substring
+
+Syntax: `{substr {0} pos length}`
+
+Takes the substring of the first argument starting at `pos` for `length`
+
+## Select Field
+
+Syntax: `{select {0} 1}`
+
+Assuming that `{0}` is a whitespace-separated value, split the values and select the item at index `1`
+
+Eg. `{select "ab cd ef" 1}` will result in `cd`
+
 ## Humanize Number (Add Commas)
 
 Syntax: `{hf val}`, `{hi val}`
@@ -122,6 +144,16 @@ Concatenates the values of the arguments separated by a table character.
 
 Good for tabulate output separation.
 
+## Paths
+
+Syntax: `{basename a/b/c}`, `{dirname a/b/c}`, `{extname a/b/c.jpg}`
+
+Selects the base, directory, or extension of a path.
+
+`basename a/b/c` = c
+`dirname  a/b/c` = a/b
+`extname a/b/c.jpg` = .jpg 
+
 ## Json
 
 Syntax: `{json field expression}` or `{json expression}`

diff --git a/pkg/expressions/context.go b/pkg/expressions/context.go
@@ -3,4 +3,5 @@ package expressions
 // KeyBuilderContext defines how to get information during run-time
 type KeyBuilderContext interface {
 	GetMatch(idx int) string
+	GetKey(key string) string
 }
diff --git a/pkg/expressions/contextArray.go b/pkg/expressions/contextArray.go
@@ -12,3 +12,9 @@ func (s *KeyBuilderContextArray) GetMatch(idx int) string {
 	}
 	return ""
 }
+
+// GetKey gets a key for a given element
+func (s *KeyBuilderContextArray) GetKey(key string) string {
+	// Unimplemented
+	return ""
+}
diff --git a/pkg/expressions/errors.go b/pkg/expressions/errors.go
@@ -6,4 +6,5 @@ const (
 	ErrorType       = "<BAD-TYPE>"
 	ErrorParsing    = "<PARSE-ERROR>"
 	ErrorArgCount   = "<ARGN>"
+	ErrorArgName    = "<NAME>"
 )
diff --git a/pkg/expressions/funcs.go b/pkg/expressions/funcs.go
@@ -47,6 +47,13 @@ var defaultFunctions = map[string]KeyBuilderFunction{
 	"suffix": KeyBuilderFunction(kfSuffix),
 	"format": KeyBuilderFunction(kfFormat),
 	"tab":    KeyBuilderFunction(kfTab),
+	"substr": KeyBuilderFunction(kfSubstr),
+	"select": KeyBuilderFunction(kfSelect),
+
+	// Pathing
+	"basename": kfPathBase,
+	"dirname":  kfPathDir,
+	"extname":  kfPathExt,
 
 	// Formatting
 	"hi": KeyBuilderFunction(kfHumanizeInt),

diff --git a/pkg/expressions/funcsPath.go b/pkg/expressions/funcsPath.go
@@ -0,0 +1,20 @@
+package expressions
+
+import "path/filepath"
+
+func kfPathManip(manipulator func(string) string) func([]KeyBuilderStage) KeyBuilderStage {
+	return func(args []KeyBuilderStage) KeyBuilderStage {
+		if len(args) != 1 {
+			return stageLiteral(ErrorArgCount)
+		}
+		return KeyBuilderStage(func(context KeyBuilderContext) string {
+			return manipulator(args[0](context))
+		})
+	}
+}
+
+var (
+	kfPathBase = kfPathManip(filepath.Base)
+	kfPathDir  = kfPathManip(filepath.Dir)
+	kfPathExt  = kfPathManip(filepath.Ext)
+)
diff --git a/pkg/expressions/funcsPath_test.go b/pkg/expressions/funcsPath_test.go
@@ -0,0 +1,15 @@
+package expressions
+
+import "testing"
+
+func TestBaseName(t *testing.T) {
+	testExpression(t, mockContext("ab/c/d"), "{basename {0}} {basename a b}", "d <ARGN>")
+}
+
+func TestDirName(t *testing.T) {
+	testExpression(t, mockContext("ab/c/d"), "{dirname {0}}", "ab/c")
+}
+
+func TestExtName(t *testing.T) {
+	testExpression(t, mockContext(), "{extname a/b/c} {extname a/b/c.jpg}", " .jpg")
+}
diff --git a/pkg/expressions/funcsStrings.go b/pkg/expressions/funcsStrings.go
@@ -39,6 +39,52 @@ func kfSuffix(args []KeyBuilderStage) KeyBuilderStage {
 	})
 }
 
+// {substr {0} }
+func kfSubstr(args []KeyBuilderStage) KeyBuilderStage {
+	if len(args) != 3 {
+		return stageLiteral(ErrorArgCount)
+	}
+
+	return KeyBuilderStage(func(context KeyBuilderContext) string {
+		s := args[0](context)
+		left, err1 := strconv.Atoi(args[1](context))
+		length, err2 := strconv.Atoi(args[2](context))
+		if err1 != nil || err2 != nil {
+			return ErrorParsing
+		}
+		right := left + length
+
+		if left < 0 {
+			left = 0
+		}
+		if right > len(s) {
+			right = len(s)
+		}
+		return s[left:right]
+	})
+}
+
+// {select {0} 1}
+func kfSelect(args []KeyBuilderStage) KeyBuilderStage {
+	if len(args) != 2 {
+		return stageLiteral(ErrorArgCount)
+	}
+
+	return KeyBuilderStage(func(context KeyBuilderContext) string {
+		s := args[0](context)
+		idx, err := strconv.Atoi(args[1](context))
+		if err != nil {
+			return ErrorParsing
+		}
+
+		fields := strings.Fields(s)
+		if idx >= 0 && idx < len(fields) {
+			return fields[idx]
+		}
+		return ""
+	})
+}
+
 // {format str args...}
 // just like fmt.Sprintf
 func kfFormat(args []KeyBuilderStage) KeyBuilderStage {

diff --git a/pkg/expressions/keyBuilder_test.go b/pkg/expressions/keyBuilder_test.go
@@ -9,13 +9,20 @@ import (
 )
 
 var testData = []string{"ab", "cd", "123"}
+var testKeyData = map[string]string{
+	"test": "testval",
+}
 
 type TestContext struct{}
 
 func (s *TestContext) GetMatch(idx int) string {
 	return testData[idx]
 }
 
+func (s *TestContext) GetKey(key string) string {
+	return testKeyData[key]
+}
+
 var testContext = TestContext{}
 
 func TestSimpleKey(t *testing.T) {
@@ -58,6 +65,12 @@ func TestDeepKeys(t *testing.T) {
 	assert.Equal(t, "<Err:{1}> is bucketed", key)
 }
 
+func TestStringKey(t *testing.T) {
+	kb, _ := NewKeyBuilder().Compile("{test} {some} key")
+	key := kb.BuildKey(&testContext)
+	assert.Equal(t, "testval  key", key)
+}
+
 func BenchmarkSimpleReplacement(b *testing.B) {
 	kb, _ := NewKeyBuilder().Compile("{0} is awesome")
 	for n := 0; n < b.N; n++ {

diff --git a/pkg/expressions/stage.go b/pkg/expressions/stage.go
@@ -17,7 +17,9 @@ func stageLiteral(s string) KeyBuilderStage {
 func stageSimpleVariable(s string) KeyBuilderStage {
 	index, err := strconv.Atoi(s)
 	if err != nil {
-		return stageLiteral(ErrorType)
+		return KeyBuilderStage(func(context KeyBuilderContext) string {
+			return context.GetKey(s)
+		})
 	}
 	return KeyBuilderStage(func(context KeyBuilderContext) string {
 		return context.GetMatch(index)

diff --git a/pkg/extractor/extractor.go b/pkg/extractor/extractor.go
@@ -88,6 +88,8 @@ func (s *Extractor) processLineSync(source string, lineNum uint64, line BString)
 		expContext := SliceSpaceExpressionContext{
 			linePtr: lineStringPtr,
 			indices: matches,
+			source:  source,
+			lineNum: lineNum,
 		}
 		if s.ignore == nil || !s.ignore.IgnoreMatch(&expContext) {
 			extractedKey := s.keyBuilder.BuildKey(&expContext)

diff --git a/pkg/extractor/extractor_test.go b/pkg/extractor/extractor_test.go
@@ -37,6 +37,23 @@ func TestBasicExtractor(t *testing.T) {
 	assert.Equal(t, uint64(4), ex.ReadLines())
 }
 
+func TestSourceAndLine(t *testing.T) {
+	input := ConvertReaderToStringChan("test", ioutil.NopCloser(strings.NewReader(testData)), 1)
+	ex, err := New(input, &Config{
+		Regex:   `(\d+)`,
+		Extract: "{src} {line} val:{1} {bad}",
+		Workers: 1,
+	})
+	assert.NoError(t, err)
+
+	vals := unbatchMatches(ex.ReadChan())
+	assert.Equal(t, "test 1 val:123 <NAME>", vals[0].Extracted)
+	assert.Equal(t, uint64(1), vals[0].LineNumber)
+
+	assert.Equal(t, "test 2 val:245 <NAME>", vals[1].Extracted)
+	assert.Equal(t, "test 3 val:123 <NAME>", vals[2].Extracted)
+}
+
 func TestGH10SliceBoundsPanic(t *testing.T) {
 	input := ConvertReaderToStringChan("", ioutil.NopCloser(strings.NewReader("this is an [ERROR] message")), 1)
 	ex, err := New(input, &Config{

diff --git a/pkg/extractor/sliceSpaceExpressionContext.go b/pkg/extractor/sliceSpaceExpressionContext.go
@@ -1,8 +1,15 @@
 package extractor
 
+import (
+	"rare/pkg/expressions"
+	"strconv"
+)
+
 type SliceSpaceExpressionContext struct {
 	linePtr string
 	indices []int
+	source  string
+	lineNum uint64
 }
 
 func (s *SliceSpaceExpressionContext) GetMatch(idx int) string {
@@ -17,3 +24,13 @@ func (s *SliceSpaceExpressionContext) GetMatch(idx int) string {
 	}
 	return s.linePtr[start:end]
 }
+
+func (s *SliceSpaceExpressionContext) GetKey(key string) string {
+	switch key {
+	case "src":
+		return s.source
+	case "line":
+		return strconv.FormatUint(s.lineNum, 10)
+	}
+	return expressions.ErrorArgName
+}