Add EOLComment as a token.

Thirumalai-Shaktivel · Thirumalai-Shaktivel · commit fa68083d08c2 · 2022-04-01T15:03:35.000+05:30
Add and update tests.
diff --git a/src/lpython/parser/tokenizer.re b/src/lpython/parser/tokenizer.re
@@ -808,6 +808,7 @@ std::string token2text(const int token)
 
         T(TK_STRING, "string")
         T(TK_COMMENT, "comment")
+        T(TK_EOLCOMMENT, "eolcomment")
         T(TK_LABEL, "label")
 
         T(TK_DBL_DOT, "..")
diff --git a/tests/reference/cpp-comment1-2f8ab90.json b/tests/reference/cpp-comment1-2f8ab90.json
@@ -0,0 +1,13 @@
+{
+    "basename": "cpp-comment1-2f8ab90",
+    "cmd": "lpython --no-color --show-tokens {infile} -o {outfile}",
+    "infile": "tests/tokens/comment1.py",
+    "infile_hash": "27258459bf39e8c5bcaa41e7fef8a593775a915273787edc46c73b4d",
+    "outfile": null,
+    "outfile_hash": null,
+    "stdout": "cpp-comment1-2f8ab90.stdout",
+    "stdout_hash": "da6cd47ba8ae6f208980f8e2b43c6b8eea0ccacf90b54fde79d3e55e",
+    "stderr": null,
+    "stderr_hash": null,
+    "returncode": 0
+}
diff --git a/tests/reference/cpp-comment1-2f8ab90.stdout b/tests/reference/cpp-comment1-2f8ab90.stdout
@@ -0,0 +1,23 @@
+(TOKEN "eolcomment") 0:10
+(TOKEN "identifier" def) 11:13
+(TOKEN "identifier" test) 15:18
+(TOKEN "(") 19:19
+(TOKEN ")") 20:20
+(TOKEN ":") 21:21
+(TOKEN "eolcomment") 23:34
+(TOKEN "comment") 39:50
+(KEYWORD "print") 55:59
+(TOKEN "(") 60:60
+(TOKEN ")") 61:61
+(NEWLINE) 62:62
+(TOKEN "comment") 63:74
+(TOKEN "identifier" test) 75:78
+(TOKEN "(") 79:79
+(TOKEN ")") 80:80
+(TOKEN "eolcomment") 82:93
+(TOKEN "comment") 94:105
+(NEWLINE) 106:106
+(TOKEN "comment") 107:110
+(TOKEN "comment") 111:123
+(TOKEN "comment") 124:134
+(EOF) 135:135
diff --git a/tests/tests.toml b/tests/tests.toml
@@ -217,3 +217,7 @@ asr = true
 [[test]]
 filename = "tokens/docstring1.py"
 tokens = true
+
+[[test]]
+filename = "tokens/comment1.py"
+tokens = true
diff --git a/tests/tokens/comment1.py b/tests/tokens/comment1.py
@@ -0,0 +1,11 @@
+#Comment 1
+def test(): # Comment 2
+    # Comment 3
+    print()
+# Comment 4
+test() # Comment 5
+# Comment 5
+
+# A
+# Multi-line
+# Comments
