Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions docs/usage/expressions.md
Original file line number Diff line number Diff line change
Expand Up @@ -550,15 +550,16 @@ Syntax: `{time str "[format:cache]" "[tz:utc]"}`

Parse a given time-string into a unix second time (default format: `cache`)

##### Format Auto-Detection
##### Format

If the format argument is omitted or set to "auto", it will attempt to resolve the format of the time.
There are several ways to define time formats, with varying ability. Usually `cache` is good enough, and is the default.

If the format is unable to be resolved, it must be specified manually with a format below, or a custom format.

If omitted or "cache": The first seen date will determine the format for all dates going forward (faster)

If "auto": The date format will be auto-detected with each parse. This can be used if the date could be in different formats (slower)
| Format | Description |
|-------------------|--------------------------------------------------------------------------------------------------------|
| `cache` (default) | Same as `auto`, but caches the first detected format (faster). If format changes, will generate errors |
| `auto` | For each evaluation, parse the date. Checks for unix time, and all known timestamp formats |
| `epoch`, `unix` | Parses unix timestamp (seconds since epoch) |
| All else | Use this as the go-style date format, described below |

##### Timezones

Expand All @@ -580,9 +581,9 @@ These are special values to output:

#### Time Format

Syntax: `{timeformat unixtime "[format:RFC3339]" "[tz:utc]"}`
Syntax: `{timeformat time "[format:RFC3339]" "[tz:utc]"}`

Takes a unix time, and formats it (default: `RFC3339`)
Takes a time, and formats it (default: `RFC3339`)

To reformat a time, you need to parse it first, eg: `{timeformat {time {0}} RFC3339}`

Expand All @@ -593,17 +594,16 @@ ANSIC, UNIX, RUBY, RFC822, RFC822Z, RFC1123, RFC1123Z, RFC3339, RFC3339, RFC3339
MONTH, MONTHNAME, MNTH, DAY, WEEKDAY, WDAY, YEAR, HOUR, MINUTE, SECOND, TIMEZONE, NTIMEZONE

**Custom formats:**
You can provide a custom format using go's well-known date. Here's an exercept from the docs:

To define your own format, write down what the reference time would look like formatted your way; see the values of constants
like ANSIC, StampMicro or Kitchen for examples. The model is to demonstrate what the reference time looks like so that the Format
and Parse methods can apply the same transformation to a general time value.
You can provide a custom format using go's well-known date. Here's an exercept from the [docs](https://pkg.go.dev/time#Layout):

The reference time used in the layouts is the specific time: `Mon Jan 2 15:04:05 MST 2006`
> To define your own format, write down what the reference time would look like formatted your way; see the values of constants
like ANSIC, StampMicro or Kitchen for examples. The model is to demonstrate what the reference time looks like so that the Format
and Parse methods can apply the same transformation to a general time value.
The reference time used in the layouts is the specific time: `Mon Jan 2 15:04:05 MST 2006`

#### Time Attribute

Syntax: `{timeattr unixtime attr [tz:utc]"}`
Syntax: `{timeattr time attr [tz:utc]"}`

Extracts an attribute about a given datetime

Expand All @@ -625,7 +625,7 @@ Formats a duration (in seconds) to a human-readable time, (eg. 4h0m0s)

#### Time Bucket

Syntax: `{buckettime str bucket "[format]" "[tz:utc]"}`
Syntax: `{buckettime time bucket "[format:RFC3339]" "[tz:utc]"}`

Truncate the time to a given bucket (*n*ano, *s*econd, *m*inute, *h*our, *d*ay, *mo*nth, *y*ear)

Expand Down
62 changes: 37 additions & 25 deletions pkg/expressions/stdlib/funcsTime.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,31 @@ func smartDateParseWrapper(format string, tz *time.Location, dateStage KeyBuilde
case "auto": // Auto will attempt to parse every time
return KeyBuilderStage(func(context KeyBuilderContext) string {
strTime := dateStage(context)

if v, ok := simpleParseNumeric(strTime); ok {
return f(time.Unix(v, 0).In(tz))
}

val, err := dateparse.ParseIn(strTime, tz)
if err != nil {
return ErrorParsing
}
return f(val)
}), nil

case "epoch", "unix": // Epoch/Unix time
return KeyBuilderStage(func(context KeyBuilderContext) string {
strTime := dateStage(context)
if unixSecs, ok := simpleParseNumeric(strTime); ok {
return f(time.Unix(unixSecs, 0).In(tz))
}
return ErrorParsing
}), nil

case "", "cache": // Empty format will auto-detect on first successful entry
var atomicFormat atomic.Value
atomicFormat.Store("")
const FORMAT_UNIX = "UNIX"

return KeyBuilderStage(func(context KeyBuilderContext) string {
strTime := dateStage(context)
Expand All @@ -79,13 +94,25 @@ func smartDateParseWrapper(format string, tz *time.Location, dateStage KeyBuilde
// This may end up run by a few different threads, but it comes at the benefit
// of not needing a mutex
var err error
liveFormat, err = dateparse.ParseFormat(strTime)
if err != nil {

// check if unix time
if _, ok := simpleParseNumeric(strTime); ok {
liveFormat = FORMAT_UNIX
} else if liveFormat, err = dateparse.ParseFormat(strTime); err != nil {
return ErrorParsing
}

atomicFormat.Store(liveFormat)
}

if liveFormat == FORMAT_UNIX {
if unixSecs, ok := simpleParseNumeric(strTime); ok {
return f(time.Unix(unixSecs, 0).In(tz))
} else {
return ErrorParsing
}
}

val, err := time.ParseInLocation(liveFormat, strTime, tz)
if err != nil {
return ErrorParsing
Expand Down Expand Up @@ -160,17 +187,9 @@ func kfTimeFormat(args []KeyBuilderStage) (KeyBuilderStage, error) {
return stageArgError(ErrParsing, 2)
}

return KeyBuilderStage(func(context KeyBuilderContext) string {
strUnixTime := args[0](context)
unixTime, err := strconv.ParseInt(strUnixTime, 10, 64)
if err != nil {
return ErrorNum
}

t := time.Unix(unixTime, 0).In(tz)

return smartDateParseWrapper("", tz, args[0], func(t time.Time) string {
return t.Format(format)
}), nil
})
}

// {func <duration_string>}
Expand Down Expand Up @@ -272,17 +291,17 @@ var attrType = map[string](func(t time.Time) string){
},
}

// {func <time> <attr> [tz:utc]}
// {timeattr <time> <attr> [format:auto] [tz:utc]}
func kfTimeAttr(args []KeyBuilderStage) (KeyBuilderStage, error) {
if len(args) < 2 || len(args) > 3 {
return stageErrArgRange(args, "2-3")
if !isArgCountBetween(args, 2, 4) {
return stageErrArgRange(args, "2-4")
}

attrName, hasAttrName := EvalStaticStage(args[1])
if !hasAttrName {
return stageArgError(ErrConst, 1)
}
tz, tzOk := parseTimezoneLocation(EvalStageIndexOrDefault(args, 2, ""))
tz, tzOk := parseTimezoneLocation(EvalStageIndexOrDefault(args, 3, ""))
if !tzOk {
return stageArgError(ErrParsing, 2)
}
Expand All @@ -292,16 +311,9 @@ func kfTimeAttr(args []KeyBuilderStage) (KeyBuilderStage, error) {
return stageArgError(ErrEnum, 1)
}

return KeyBuilderStage(func(context KeyBuilderContext) string {
unixTime, err := strconv.ParseInt(args[0](context), 10, 64)
if err != nil {
return ErrorNum
}

t := time.Unix(unixTime, 0).In(tz)
timeFormat := EvalStageIndexOrDefault(args, 2, "")

return attrFunc(t)
}), nil
return smartDateParseWrapper(timeFormat, tz, args[0], attrFunc)
}

// Pass in "", "local", "utc" or a valid unix timezone
Expand Down
95 changes: 77 additions & 18 deletions pkg/expressions/stdlib/funcsTime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
"testing"
"time"

"github.com/zix99/rare/pkg/expressions"

"github.com/araddon/dateparse"
"github.com/stretchr/testify/assert"
)
Expand All @@ -18,16 +16,63 @@ func TestTimeExpressionErr(t *testing.T) {
mockContext("14/Apr/2016:19:12:25 +0200"),
"{time {0} NGINX}",
"1460653945")
testExpression(t,
mockContext("14/Apr/2016:19:12:25 +0200"),
"{time {0} auto}",
"1460653945")
testExpression(t,
mockContext("14/Apr/2016:19:12:25 +0200"),
"{time {0} cache}",
"1460653945")
testExpression(t,
mockContext("14/Apr/2016:19:12:25 +0200"),
"{time {0}}",
"1460653945")

// epoch/unix
testExpression(t, mockContext(), "{time 1460653945}", "1460653945")
testExpression(t, mockContext(), "{time 1460653945 auto}", "1460653945")
testExpression(t, mockContext(), "{time 1460653945 unix}", "1460653945")
testExpression(t, mockContext(), "{time 1460653945 cache}", "1460653945")

// Error states
testExpression(t, mockContext(""), "{time a}", "<PARSE-ERROR>")
testExpression(t, mockContext(""), "{time a auto}", "<PARSE-ERROR>")
testExpression(t, mockContext(""), "{time a cache}", "<PARSE-ERROR>")
testExpression(t, mockContext(""), "{time a epoch}", "<PARSE-ERROR>")
testExpressionErr(t, mockContext(""), "{time a b c d e}", "<ARGN>", ErrArgCount)
}

func TestCachedParsingTimestamp(t *testing.T) {
kb, err := NewStdKeyBuilder().Compile("{time {0} cache}")
assert.Nil(t, err)

assert.Equal(t, "1460653945", kb.BuildKey(mockContext("14/Apr/2016:19:12:25 +0200")))
assert.Equal(t, "1460653945", kb.BuildKey(mockContext("14/Apr/2016:19:12:25 +0200")))
assert.Equal(t, "<PARSE-ERROR>", kb.BuildKey(mockContext("1460653945"))) // Can't parse epoch, locked into real
assert.Equal(t, "<PARSE-ERROR>", kb.BuildKey(mockContext("not a time")))
}

func TestCachedParsingEpoch(t *testing.T) {
kb, err := NewStdKeyBuilder().Compile("{time {0} cache}")
assert.Nil(t, err)

assert.Equal(t, "1460653945", kb.BuildKey(mockContext("1460653945")))
assert.Equal(t, "<PARSE-ERROR>", kb.BuildKey(mockContext("14/Apr/2016:19:12:25 +0200"))) // Can't parse real time, locked into epoch
assert.Equal(t, "<PARSE-ERROR>", kb.BuildKey(mockContext("not a time")))
}

func TestFormatExpression(t *testing.T) {
// Defined type
testExpression(t,
mockContext("14/Apr/2016:19:12:25 +0200"),
"{timeformat {time {0} NGINX} RFC3339 utc}",
"2016-04-14T17:12:25Z")
// Implicit parse
testExpression(t,
mockContext("14/Apr/2016:19:12:25"),
"{timeformat {0} RFC3339}",
"2016-04-14T19:12:25Z")
// Explicit
testExpression(t,
mockContext("14/Apr/2016:19:12:25 +0200"),
Expand Down Expand Up @@ -186,6 +231,32 @@ func TestTimeAttr(t *testing.T) {
"{timeattr {time {0}} quarter}",
"2")

// Test with implicit time parsing
testExpression(t,
mockContext("14/Apr/2016 01:00:00"),
"{timeattr {0} weekday}",
"4")
testExpression(t,
mockContext("14/Apr/2016 01:00:00"),
"{timeattr {0} week}",
"15")
testExpression(t,
mockContext("14/Apr/2016 01:00:00"),
"{timeattr {0} Yearweek}",
"2016-15")
testExpression(t,
mockContext("14/Apr/2016 01:00:00"),
"{timeattr {0} quarter}",
"2")
testExpression(t,
mockContext("1460595600"),
"{timeattr {0} weekday}",
"4")
testExpression(t,
mockContext("1460595600"),
"{timeattr {0} weekday epoch utc}",
"4")

testExpressionErr(t, mockContext("a"), "{timeattr {time now} {0}}", "<CONST>")
testExpressionErr(t, mockContext("a"), "{timeattr {time now} bad-value}", "<ENUM>")
}
Expand All @@ -200,7 +271,7 @@ func TestTimeAttrToLocal(t *testing.T) {
func TestTimeAttrToBadTZ(t *testing.T) {
testExpressionErr(t,
mockContext("14/Apr/2016 01:00:00"),
"{timeattr {time {0}} weekday asdf}",
"{timeattr {time {0}} weekday auto asdf}",
"<PARSE-ERROR>", ErrParsing)
}

Expand All @@ -214,7 +285,7 @@ func TestTimeAttrArgError(t *testing.T) {
func TestTimeAttrArgErrorExtra(t *testing.T) {
testExpressionErr(t,
mockContext("14/Apr/2016 01:00:00"),
"{timeattr {time {0}} a b c}",
"{timeattr {time {0}} a b c e}",
"<ARGN>", ErrArgCount)
}

Expand All @@ -237,21 +308,9 @@ func TestLoadingTimezone(t *testing.T) {
assert.False(t, ok)
}

// BenchmarkTimeParseExpression-4 537970 2133 ns/op 536 B/op 9 allocs/op
// BenchmarkTimeParseExpression/{time_"14/Apr/2016:19:12:25_+0200"_auto}-4 761017 1645 ns/op 336 B/op 4 allocs/op
func BenchmarkTimeParseExpression(b *testing.B) {
stage, _ := kfTimeParse([]expressions.KeyBuilderStage{
func(kbc expressions.KeyBuilderContext) string {
return kbc.GetMatch(0)
},
literal("auto"),
})
for i := 0; i < b.N; i++ {
stage(&expressions.KeyBuilderContextArray{
Elements: []string{
"14/Apr/2016:19:12:25 +0200",
},
})
}
benchmarkExpression(b, mockContext(), `{time "14/Apr/2016:19:12:25 +0200" auto}`, "1460653945")
}

// BenchmarkTimeParse-4 1686390 654.7 ns/op 120 B/op 4 allocs/op
Expand Down
19 changes: 19 additions & 0 deletions pkg/expressions/stdlib/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,22 @@ func isPartialString(s, word string) bool {
func isArgCountBetween(args []expressions.KeyBuilderStage, min, max int) bool {
return len(args) >= min && len(args) <= max
}

// Check if string is positive numeric quickly for unix time
// this only works for positive, decimal, non-fractional numbers (eg. just 0-9)
// strconv.ParseInt makes 3 allocs and is significantly slower in the non-numeric case
func simpleParseNumeric(s string) (int64, bool) {
if len(s) == 0 {
return 0, false
}

var n int64
for i := 0; i < len(s); i++ {
c := s[i]
if c < '0' || c > '9' {
return 0, false
}
n = n*10 + int64(c-'0')
}
return n, true
}
Loading
Loading