feat: add manual scanner benchmarks

PedroRosalba · guiferpa · commit 29286c75fbde · 2026-02-03T13:17:25.000-03:00
diff --git a/lexer/bench_manual_scanner.md b/lexer/bench_manual_scanner.md
@@ -0,0 +1,22 @@
+goos: darwin
+goarch: arm64
+pkg: github.com/guiferpa/aurora/lexer
+cpu: Apple M3
+BenchmarkScanToken_Keyword-8         	78417979	        15.25 ns/op	       0 B/op	       0 allocs/op
+BenchmarkScanToken_Identifier-8      	73019718	        17.32 ns/op	       0 B/op	       0 allocs/op
+BenchmarkScanToken_Number-8          	131816068	         9.118 ns/op	       0 B/op	       0 allocs/op
+BenchmarkScanToken_HexNumber-8       	100000000	        10.64 ns/op	       0 B/op	       0 allocs/op
+BenchmarkScanToken_String-8          	151201761	         7.927 ns/op	       0 B/op	       0 allocs/op
+BenchmarkScanToken_SingleChar-8      	192291577	         7.268 ns/op	       0 B/op	       0 allocs/op
+BenchmarkScanToken_Comment-8         	167061290	         7.180 ns/op	       0 B/op	       0 allocs/op
+BenchmarkGetTokens_Simple-8          	 3057578	       444.7 ns/op	    1360 B/op	      14 allocs/op
+BenchmarkGetTokens_Medium-8          	 1000000	      1201 ns/op	    3984 B/op	      37 allocs/op
+BenchmarkGetTokens_Complex-8         	  349312	      4002 ns/op	   13488 B/op	     102 allocs/op
+BenchmarkGetTokens_Hex-8             	  981447	      1231 ns/op	    3984 B/op	      37 allocs/op
+BenchmarkGetTokens_Strings-8         	 1000000	      1116 ns/op	    3696 B/op	      34 allocs/op
+BenchmarkGetTokens_ManyTokens-8      	  346030	      3432 ns/op	   13104 B/op	      98 allocs/op
+BenchmarkGetFilledTokens_Complex-8   	  223297	      5263 ns/op	   15648 B/op	     109 allocs/op
+BenchmarkIsIdentChar-8               	145176751	         8.246 ns/op	       0 B/op	       0 allocs/op
+BenchmarkKeywordsLookup-8            	27211629	        50.22 ns/op	       0 B/op	       0 allocs/op
+PASS
+ok  	github.com/guiferpa/aurora/lexer	25.273s
diff --git a/lexer/scanner_bench_test.go b/lexer/scanner_bench_test.go
@@ -0,0 +1,140 @@
+package lexer
+
+import (
+	"testing"
+)
+
+var (
+	// Simple expression
+	simpleInput = []byte(`ident foo = 123;`)
+
+	// Medium complexity - if statement
+	mediumInput = []byte(`ident result = if 10 bigger 5 { 100; } else { 200; };`)
+
+	// Complex - multiple statements with function calls
+	complexInput = []byte(`
+ident fib = {
+  ident n = arguments 0;
+  if n smaller 1 or n equals 1 { n; } else { fib(n - 1) + fib(n - 2); };
+};
+ident result = fib(10);
+print result;
+`)
+
+	// Hexadecimal numbers
+	hexInput = []byte(`ident hex_ff = 0xFF; ident hex_10 = 0x10; ident sum = hex_ff + hex_10;`)
+
+	// String heavy
+	stringInput = []byte(`ident greeting = "hello"; ident name = "world"; print greeting; print name;`)
+
+	// Many tokens
+	manyTokensInput = []byte(`ident a = 1; ident b = 2; ident c = 3; ident d = 4; ident e = 5; ident f = 6; ident g = 7; ident h = 8; ident i = 9; ident j = 10;`)
+)
+
+func BenchmarkScanToken_Keyword(b *testing.B) {
+	input := []byte(`ident`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkScanToken_Identifier(b *testing.B) {
+	input := []byte(`myVariable123`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkScanToken_Number(b *testing.B) {
+	input := []byte(`123456`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkScanToken_HexNumber(b *testing.B) {
+	input := []byte(`0xABCDEF`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkScanToken_String(b *testing.B) {
+	input := []byte(`"hello world"`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkScanToken_SingleChar(b *testing.B) {
+	input := []byte(`(`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkScanToken_Comment(b *testing.B) {
+	input := []byte(`#- this is a comment`)
+	for i := 0; i < b.N; i++ {
+		ScanToken(input)
+	}
+}
+
+func BenchmarkGetTokens_Simple(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetTokens(simpleInput)
+	}
+}
+
+func BenchmarkGetTokens_Medium(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetTokens(mediumInput)
+	}
+}
+
+func BenchmarkGetTokens_Complex(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetTokens(complexInput)
+	}
+}
+
+func BenchmarkGetTokens_Hex(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetTokens(hexInput)
+	}
+}
+
+func BenchmarkGetTokens_Strings(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetTokens(stringInput)
+	}
+}
+
+func BenchmarkGetTokens_ManyTokens(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetTokens(manyTokensInput)
+	}
+}
+
+func BenchmarkGetFilledTokens_Complex(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		GetFilledTokens(complexInput)
+	}
+}
+func BenchmarkIsIdentChar(b *testing.B) {
+	chars := []byte("abcABC123_")
+	for i := 0; i < b.N; i++ {
+		for _, c := range chars {
+			isIdentChar(c)
+		}
+	}
+}
+
+func BenchmarkKeywordsLookup(b *testing.B) {
+	words := []string{"if", "else", "ident", "print", "foo", "bar", "myVar"}
+	for i := 0; i < b.N; i++ {
+		for _, w := range words {
+			_ = Keywords[w]
+		}
+	}
+}
