Skip to content

Commit 8d69a02

Browse files
Mr-RmMr-Rm
committed
разделение литералов/слов на типы и рефакторинг лексера
1 parent 3b043e4 commit 8d69a02

File tree

6 files changed

+246
-228
lines changed

6 files changed

+246
-228
lines changed

‎src/OneScript.Language/IdentifiersTrie.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,22 @@ public bool Remove(string key)
9595
throw new System.NotImplementedException();
9696
}
9797

98+
public bool Contains(string str)
99+
{
100+
var node = _root.sibl;
101+
foreach (char ch in str)
102+
{
103+
var key = node.Find(ch);
104+
if (key == null)
105+
return false;
106+
107+
node = key.next;
108+
}
109+
110+
return node.HasValue;
111+
}
112+
113+
98114
public T Get(string str)
99115
{
100116
var node = _root;

‎src/OneScript.Language/LanguageDef.cs‎

Lines changed: 41 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@ This Source Code Form is subject to the terms of the
55
at http://mozilla.org/MPL/2.0/.
66
----------------------------------------------------------*/
77

8+
using OneScript.Language.LexicalAnalysis;
89
using System;
910
using System.Collections.Generic;
1011
using System.Runtime.CompilerServices;
11-
using OneScript.Language.LexicalAnalysis;
12+
using static OneScript.Language.LanguageDef;
1213

1314
namespace OneScript.Language
1415
{
@@ -21,14 +22,20 @@ public static class LanguageDef
2122

2223
private static readonly IdentifiersTrie<Token> _stringToToken = new IdentifiersTrie<Token>();
2324

24-
private static readonly IdentifiersTrie<bool> _undefined = new IdentifiersTrie<bool>();
25-
private static readonly IdentifiersTrie<bool> _booleans = new IdentifiersTrie<bool>();
26-
private static readonly IdentifiersTrie<bool> _logicalOp = new IdentifiersTrie<bool>();
27-
28-
private static readonly IdentifiersTrie<bool> _preprocImport = new IdentifiersTrie<bool>();
29-
3025
const int BUILTINS_INDEX = (int)Token.ByValParam;
3126

27+
public enum WordType
28+
{
29+
Undefined,
30+
Boolean,
31+
Logical,
32+
Null,
33+
Preproc,
34+
None
35+
};
36+
37+
private static readonly IdentifiersTrie<WordType> _specwords = new IdentifiersTrie<WordType>();
38+
3239
static LanguageDef()
3340
{
3441
_priority.Add(Token.Plus, 5);
@@ -52,21 +59,26 @@ static LanguageDef()
5259

5360
#region constants
5461

55-
_undefined.Add("Undefined", true);
56-
_undefined.Add("Неопределено", true);
62+
_specwords.Add("Undefined", WordType.Undefined);
63+
_specwords.Add("Неопределено", WordType.Undefined);
5764

58-
_booleans.Add("True", true);
59-
_booleans.Add("False", true);
60-
_booleans.Add("Истина", true);
61-
_booleans.Add("Ложь", true);
65+
_specwords.Add("True", WordType.Boolean);
66+
_specwords.Add("False", WordType.Boolean);
67+
_specwords.Add("Истина", WordType.Boolean);
68+
_specwords.Add("Ложь", WordType.Boolean);
6269

63-
_logicalOp.Add("And", true);
64-
_logicalOp.Add("Or", true);
65-
_logicalOp.Add("Not", true);
70+
_specwords.Add("And", WordType.Logical);
71+
_specwords.Add("Or", WordType.Logical);
72+
_specwords.Add("Not", WordType.Logical);
6673

67-
_logicalOp.Add("И", true);
68-
_logicalOp.Add("ИЛИ", true);
69-
_logicalOp.Add("НЕ", true);
74+
_specwords.Add("И", WordType.Logical);
75+
_specwords.Add("ИЛИ", WordType.Logical);
76+
_specwords.Add("НЕ", WordType.Logical);
77+
78+
_specwords.Add("NULL", WordType.Null);
79+
80+
_specwords.Add("Использовать", WordType.Preproc);
81+
_specwords.Add("Use", WordType.Preproc);
7082

7183
#endregion
7284

@@ -216,8 +228,6 @@ static LanguageDef()
216228

217229
#endregion
218230

219-
_preprocImport.Add("Использовать", true);
220-
_preprocImport.Add("Use", true);
221231
}
222232

223233
private static void AddToken(Token token, string name)
@@ -433,18 +443,24 @@ public static bool IsEndOfBlockToken(Token token)
433443
default:
434444
return false;
435445
}
446+
}
447+
448+
449+
public static WordType GetWordType(string value)
450+
{
451+
return _specwords.TryGetValue(value, out var wordType)? wordType : WordType.None;
436452
}
437453

438454
[MethodImpl(MethodImplOptions.AggressiveInlining)]
439455
public static bool IsBooleanLiteralString(string value)
440456
{
441-
return _booleans.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled;
457+
return _specwords.TryGetValue(value, out var wordType) && wordType == WordType.Boolean;
442458
}
443459

444460
[MethodImpl(MethodImplOptions.AggressiveInlining)]
445461
public static bool IsUndefinedString(string value)
446462
{
447-
return _undefined.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled;
463+
return _specwords.TryGetValue(value, out var wordType) && wordType == WordType.Undefined;
448464
}
449465

450466
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -456,13 +472,13 @@ public static bool IsNullString(string value)
456472
[MethodImpl(MethodImplOptions.AggressiveInlining)]
457473
public static bool IsLogicalOperatorString(string content)
458474
{
459-
return _logicalOp.TryGetValue(content, out var nodeIsFilled) && nodeIsFilled;
475+
return _specwords.TryGetValue(content, out var wordType) && wordType == WordType.Logical;
460476
}
461477

462478
[MethodImpl(MethodImplOptions.AggressiveInlining)]
463479
public static bool IsImportDirective(string value)
464480
{
465-
return _preprocImport.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled;
481+
return _specwords.TryGetValue(value, out var wordType) && wordType == WordType.Preproc;
466482
}
467483
}
468484
}

‎src/OneScript.Language/LexicalAnalysis/StringLexerState.cs‎

Lines changed: 97 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,97 +1,97 @@
1-
/*----------------------------------------------------------
2-
This Source Code Form is subject to the terms of the
3-
Mozilla Public License, v.2.0. If a copy of the MPL
4-
was not distributed with this file, You can obtain one
5-
at http://mozilla.org/MPL/2.0/.
6-
----------------------------------------------------------*/
7-
8-
using System.Text;
9-
10-
namespace OneScript.Language.LexicalAnalysis
11-
{
12-
public class StringLexerState : LexerState
13-
{
14-
private void SkipSpacesAndComments(SourceCodeIterator iterator)
15-
{
16-
while (true)
17-
{ /* Пропускаем все пробелы и комментарии */
18-
iterator.SkipSpaces();
19-
20-
if (iterator.CurrentSymbol == '/')
21-
{
22-
if (!iterator.MoveNext())
23-
throw CreateExceptionOnCurrentLine("Некорректный символ", iterator);
24-
25-
if (iterator.CurrentSymbol != '/')
26-
throw CreateExceptionOnCurrentLine("Некорректный символ", iterator);
27-
28-
do
29-
{
30-
if (!iterator.MoveNext())
31-
break;
32-
33-
} while (iterator.CurrentSymbol != '\n');
34-
35-
}
36-
else
37-
break;
38-
}
39-
}
40-
41-
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
42-
{
43-
StringBuilder contentBuilder = new StringBuilder();
44-
45-
while (iterator.MoveNext())
46-
{
47-
var cs = iterator.CurrentSymbol;
48-
49-
if (cs == SpecialChars.StringQuote)
50-
{
51-
if (iterator.MoveNext())
52-
{
53-
if (iterator.CurrentSymbol == SpecialChars.StringQuote)
54-
{
55-
/* Двойная кавычка */
56-
contentBuilder.Append("\"");
57-
continue;
58-
}
59-
60-
/* Завершение строки */
61-
SkipSpacesAndComments(iterator);
62-
63-
if (iterator.CurrentSymbol == SpecialChars.StringQuote)
64-
{
65-
/* Сразу же началась новая строка */
66-
contentBuilder.Append('\n');
67-
continue;
68-
}
69-
}
70-
71-
var lex = new Lexem
72-
{
73-
Type = LexemType.StringLiteral,
74-
Content = contentBuilder.ToString()
75-
};
76-
return lex;
77-
}
78-
79-
if (cs == '\n')
80-
{
81-
iterator.MoveNext();
82-
SkipSpacesAndComments(iterator);
83-
84-
if (iterator.CurrentSymbol != '|')
85-
throw CreateExceptionOnCurrentLine("Некорректный строковый литерал!", iterator);
86-
87-
contentBuilder.Append('\n');
88-
}
89-
else if(cs != '\r')
90-
contentBuilder.Append(cs);
91-
92-
}
93-
94-
throw CreateExceptionOnCurrentLine("Незавершённый строковой интервал!", iterator);
95-
}
96-
}
97-
}
1+
/*----------------------------------------------------------
2+
This Source Code Form is subject to the terms of the
3+
Mozilla Public License, v.2.0. If a copy of the MPL
4+
was not distributed with this file, You can obtain one
5+
at http://mozilla.org/MPL/2.0/.
6+
----------------------------------------------------------*/
7+
8+
using System.Text;
9+
10+
namespace OneScript.Language.LexicalAnalysis
11+
{
12+
public class StringLexerState : LexerState
13+
{
14+
private void SkipSpacesAndComments(SourceCodeIterator iterator)
15+
{
16+
while (true)
17+
{ /* Пропускаем все пробелы и комментарии */
18+
iterator.SkipSpaces();
19+
20+
if (iterator.CurrentSymbol == '/')
21+
{
22+
if (!iterator.MoveNext())
23+
throw CreateExceptionOnCurrentLine("Некорректный символ", iterator);
24+
25+
if (iterator.CurrentSymbol != '/')
26+
throw CreateExceptionOnCurrentLine("Некорректный символ", iterator);
27+
28+
do
29+
{
30+
if (!iterator.MoveNext())
31+
break;
32+
33+
} while (iterator.CurrentSymbol != '\n');
34+
35+
}
36+
else
37+
break;
38+
}
39+
}
40+
41+
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
42+
{
43+
StringBuilder contentBuilder = new StringBuilder();
44+
45+
while (iterator.MoveNext())
46+
{
47+
var cs = iterator.CurrentSymbol;
48+
49+
if (cs == SpecialChars.StringQuote)
50+
{
51+
if (iterator.MoveNext())
52+
{
53+
if (iterator.CurrentSymbol == SpecialChars.StringQuote)
54+
{
55+
/* Двойная кавычка */
56+
contentBuilder.Append("\"");
57+
continue;
58+
}
59+
60+
/* Завершение строки */
61+
SkipSpacesAndComments(iterator);
62+
63+
if (iterator.CurrentSymbol == SpecialChars.StringQuote)
64+
{
65+
/* Сразу же началась новая строка */
66+
contentBuilder.Append('\n');
67+
continue;
68+
}
69+
}
70+
71+
var lex = new Lexem
72+
{
73+
Type = LexemType.StringLiteral,
74+
Content = contentBuilder.ToString()
75+
};
76+
return lex;
77+
}
78+
79+
if (cs == '\n')
80+
{
81+
iterator.MoveNext();
82+
SkipSpacesAndComments(iterator);
83+
84+
if (iterator.CurrentSymbol != '|')
85+
throw CreateExceptionOnCurrentLine("Некорректный строковый литерал!", iterator);
86+
87+
contentBuilder.Append('\n');
88+
}
89+
else if(cs != '\r')
90+
contentBuilder.Append(cs);
91+
92+
}
93+
94+
throw CreateExceptionOnCurrentLine("Незавершённый строковый литерал!", iterator);
95+
}
96+
}
97+
}

0 commit comments

Comments
 (0)