Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 689e4fc

Browse files
jwdeveloperjwdeveloper
committed
:)
1 parent 6386db5 commit 689e4fc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1812
-297
lines changed

‎SL.Core/Api/Exceptions/EndOfParsingException.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
namespace SL.Core.Api.Exceptions;
2+
3+
public class EndOfParsingException : Exception
4+
{
5+
public EndOfParsingException(string message) : base(message)
6+
{
7+
8+
}
9+
}

‎SL.Core/Api/Exceptions/SyntaxException.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
namespace SL.Core.Api.Exceptions;
2+
3+
public class SyntaxException : Exception
4+
{
5+
public SyntaxException(string message, IPosition position) : base(ParseMessage(message,position))
6+
{
7+
8+
}
9+
10+
private static string ParseMessage(string message, IPosition position)
11+
{
12+
return $"{message} at position {position}";
13+
}
14+
}

‎SL.Core/Api/Exceptions/TokenSyntaxException.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
using SL.Core.Common;
2+
3+
namespace SL.Core.Api.Exceptions;
4+
5+
public class BadTokenException : Exception
6+
{
7+
8+
public BadTokenException(string excepted, Token current) : base(ParseMessage(excepted,current))
9+
{
10+
11+
}
12+
13+
14+
private static string ParseMessage(string excepted, Token current)
15+
{
16+
return $"Excepted token: {excepted} instead of {current.Value} at position {current.Position}";
17+
}
18+
}

‎SL.Core/Api/ITokenIterator.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using SL.Core.Common;
2+
3+
namespace SL.Core.Api;
4+
5+
public interface ITokenIterator
6+
{
7+
public Token NextToken();
8+
9+
public Token NextToken(string value);
10+
11+
public Token NextToken(params TokenType[] type);
12+
13+
public Token CurrentToken();
14+
15+
public Token LookUp();
16+
17+
public Token LookUp(string value);
18+
19+
public Token LookUp(TokenType value);
20+
21+
public bool IsValid();
22+
}

‎SL.Core/Common/Token.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,24 @@ public enum TokenType
2525

2626
OPEN_ARGUMETNS,
2727
CLOSE_ARGUMENTS,
28+
COMMA,
2829

2930

3031
//Variable
3132
STRING,
3233
NUMBER,
3334
BOOL,
34-
LITERRAL,
35+
IDENTIFIER,
3536

3637

37-
BINARY_OPERATION,
38-
EQUAL,
38+
ASSIGMENT,
39+
COMPLEX_ASSIGMENT,
40+
BINARY_ADDATIVE_OPERATOR,
41+
BINARY_MULTIPLICATION_OPERATOR,
42+
EQUALITY_OPREATOR,
43+
LOGICAL_OPERATOR,
3944

4045
KEYWORLD,
46+
OBJECT_TYPE,
4147
DOT
4248
}

‎SL.Core/Lexing/CharIterator.cs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,13 @@ public char Peek(int offset)
5959
return _target[index];
6060
}
6161

62-
63-
62+
public bool IsEnded()
63+
{
64+
return _current == _endValue;
65+
}
6466

67+
public IPosition Position()
68+
{
69+
return _position;
70+
}
6571
}

‎SL.Core/Lexing/BasicTokenHandler.cs‎ renamed to ‎SL.Core/Lexing/Handlers/BasicTokenHandler.cs‎

File renamed without changes.

‎SL.Core/Lexing/Handlers/StringHandler.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public async Task<Token> Handle(string symbol, IIterator<char> iterator, Cancell
3434

3535
}
3636

37+
3738
return Token.BadToken($"String is not closed: {sb}", iterator.Position());
3839
}
3940
}

‎SL.Core/Lexing/Lexer.cs‎

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,17 @@
1-
2-
3-
4-
5-
6-
7-
8-
91
using SL.Core.Api;
102
using SL.Core.Common;
113

124
namespace SL.Core.Lexing;
135

146
public class Lexer
157
{
16-
private CharIterator _iterator;
17-
private Dictionary<string, ITokenHandler> _handlers;
18-
private HashSet<string> _ignores;
8+
private readonly CharIterator _iterator;
9+
private readonly Dictionary<string, ITokenHandler> _handlers;
10+
private readonly HashSet<string> _ignores;
1911

2012
public Lexer(CharIterator iterator,
21-
Dictionary<string, ITokenHandler> handlers,
22-
HashSet<string> ignores)
13+
Dictionary<string, ITokenHandler> handlers,
14+
HashSet<string> ignores)
2315
{
2416
_iterator = iterator;
2517
_handlers = handlers;
@@ -57,7 +49,7 @@ public async Task<Token> Lex(CancellationToken ctx)
5749
var nextSymbol = _iterator.Peek(1).ToString();
5850
if (_handlers.ContainsKey(nextSymbol) || _ignores.Contains(nextSymbol))
5951
{
60-
return new Token(TokenType.LITERRAL, symbol, _iterator.Position());
52+
return new Token(TokenType.IDENTIFIER, symbol, _iterator.Position());
6153
}
6254

6355
_iterator.Advance();
@@ -78,10 +70,42 @@ public async Task<Token> Lex(CancellationToken ctx)
7870
do
7971
{
8072
token = await Lex(ctx);
73+
if (token.Type == TokenType.IGNORED)
74+
{
75+
continue;
76+
}
77+
8178
tokens.Add(token);
82-
}
83-
while (token.Type != TokenType.END_OF_FILE && token.Type != TokenType.BAD_TOKEN);
79+
} while (!ctx.IsCancellationRequested &&
80+
(token.Type != TokenType.END_OF_FILE && token.Type != TokenType.BAD_TOKEN));
81+
82+
var cleared = new List<Token>();
83+
for (var i = 1; i < tokens.Count(); i++)
84+
{
85+
var one = tokens[i - 1];
86+
var two = tokens[i];
8487

85-
return tokens;
88+
var value = one.Value + two.Value;
89+
if (_handlers.ContainsKey(value))
90+
{
91+
var handler = _handlers[value];
92+
var tokenaa = await handler.Handle(value, _iterator, ctx);
93+
cleared.Add(tokenaa);
94+
i += 1;
95+
continue;
96+
}
97+
else
98+
{
99+
cleared.Add(one);
100+
}
101+
}
102+
103+
104+
return cleared;
105+
}
106+
107+
public async Task<TokenIterator> LexAllToInterator(CancellationToken ctx = new CancellationToken())
108+
{
109+
return new TokenIterator(await LexAll(ctx), Token.EndOfFile(null), ctx);
86110
}
87111
}

‎SL.Core/Lexing/TokenIterator.cs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
using SL.Core.Api;
2+
using SL.Core.Api.Exceptions;
3+
using SL.Core.Common;
4+
5+
namespace SL.Core.Lexing;
6+
7+
public class TokenIterator : AbstractIterator<Token>, ITokenIterator
8+
{
9+
10+
private readonly CancellationToken _ctx;
11+
12+
13+
public TokenIterator(List<Token> target, Token defaultValue, CancellationToken cancellationToken) : base(target, defaultValue)
14+
{
15+
_ctx = cancellationToken;
16+
}
17+
18+
public Token NextToken()
19+
{
20+
return ValiateToken(Advance());
21+
}
22+
23+
public Token NextToken(string value)
24+
{
25+
var token = Advance();
26+
if (token.Value != value)
27+
{
28+
throw new BadTokenException(value, token);
29+
}
30+
return ValiateToken(token);
31+
}
32+
33+
34+
public Token NextToken(params TokenType[] types)
35+
{
36+
var token = Advance();
37+
38+
var hasType = types.Any(e => e == token.Type);
39+
if (!hasType)
40+
{
41+
throw new BadTokenException(types[0].ToString(), token);
42+
}
43+
return ValiateToken(token);
44+
}
45+
46+
public Token CurrentToken()
47+
{
48+
return ValiateToken(Current());
49+
}
50+
51+
public Token LookUp()
52+
{
53+
return Peek(1);
54+
}
55+
56+
public Token LookUp(string value)
57+
{
58+
var token = Peek(1);
59+
if (token.Value != value)
60+
{
61+
throw new BadTokenException(value, token);
62+
}
63+
64+
return token;
65+
}
66+
67+
public Token LookUp(TokenType value)
68+
{
69+
var token = Peek(1);
70+
if (token.Type != value)
71+
{
72+
throw new BadTokenException(value.ToString(), token);
73+
}
74+
75+
return token;
76+
}
77+
78+
public new bool IsValid()
79+
{
80+
if (_ctx.IsCancellationRequested)
81+
{
82+
return false;
83+
}
84+
85+
return true;
86+
}
87+
88+
private Token ValiateToken(Token token)
89+
{
90+
if (_ctx.IsCancellationRequested)
91+
throw new EndOfParsingException("Cancellation request");
92+
93+
if (token.Type == TokenType.END_OF_FILE)
94+
throw new EndOfParsingException("End of file");
95+
96+
if (token.Type == TokenType.BAD_TOKEN)
97+
throw new EndOfParsingException("Bad token");
98+
99+
return token;
100+
}
101+
}

0 commit comments

Comments
 (0)