Skip to content

Commit f028554

Browse files
jakubmisekjakubmisek
committed
bump parser
- PHP 8.0-8.5 syntax - correct namespace name parsing according to selected language level - correct keywords parsing according to selected language level - optimizations & cleanups
1 parent 608bf30 commit f028554

9 files changed

Lines changed: 90 additions & 103 deletions

File tree

‎Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242

4343
<PeachpieLibraryRegularExpressionsVersion>1.7.0</PeachpieLibraryRegularExpressionsVersion>
4444
<MySqlConnectorVersion>2.0.0</MySqlConnectorVersion>
45-
<ParserVersion>8.5.17973</ParserVersion>
45+
<ParserVersion>8.5.18350</ParserVersion>
4646
<PeachpieMicrosoftCodeAnalysisVersion>3.7.4</PeachpieMicrosoftCodeAnalysisVersion>
4747

4848
</PropertyGroup>

‎src/Peachpie.CodeAnalysis/Semantics/SemanticsBinder.cs‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ protected ImmutableArray<BoundExpression> BindExpressions(IEnumerable<AST.Expres
201201

202202
protected BoundExpression BindExpression(AST.Expression expr) => BindExpression(expr, BoundAccess.Read);
203203

204-
protected BoundArgument BindArgument(AST.Expression expr, bool isByRef = false, bool isUnpack = false, string name = null)
204+
protected BoundArgument BindArgument(AST.IExpression expr, bool isByRef = false, bool isUnpack = false, string name = null)
205205
{
206206
//if (isUnpack)
207207
//{
@@ -217,6 +217,21 @@ protected BoundArgument BindArgument(AST.Expression expr, bool isByRef = false,
217217
: BoundArgument.Create(bound, name);
218218
}
219219

220+
protected ImmutableArray<BoundArgument> BindArguments(AST.EchoStmt.ParametersEnumerable expressions)
221+
{
222+
var arguments = new BoundArgument[expressions.Count/*mostly 1*/];
223+
var i = 0;
224+
foreach (var expr in expressions)
225+
{
226+
arguments[i++] = BindArgument(expr);
227+
}
228+
229+
Debug.Assert(i == arguments.Length);
230+
231+
//
232+
return ImmutableArray.Create(arguments);
233+
}
234+
220235
protected ImmutableArray<BoundArgument> BindArguments(params AST.Expression[] expressions)
221236
{
222237
Debug.Assert(expressions != null);
@@ -968,7 +983,7 @@ AST.Expression BindArm(AST.IExpression value, AST.MatchArm arm, AST.IExpression
968983
// value === condition[i]
969984
var cond = AST.BinaryEx.Create(
970985
arm.ConditionList[i].Span,
971-
AST.Operations.Identical,
986+
Tokens.T_IS_IDENTICAL, // AST.Operations.Identical,
972987
(AST.Expression)value,
973988
(AST.Expression)arm.ConditionList[i])
974989
;
@@ -980,7 +995,7 @@ AST.Expression BindArm(AST.IExpression value, AST.MatchArm arm, AST.IExpression
980995
}
981996
else
982997
{
983-
condition = AST.BinaryEx.Create(Span.Invalid, AST.Operations.Or, condition, cond);
998+
condition = AST.BinaryEx.Create(Span.Invalid, Tokens.T_BOOLEAN_OR /*AST.Operations.Or*/, condition, cond);
984999
condition.ContainingElement = arm;
9851000
}
9861001
}

‎src/Peachpie.CodeAnalysis/Syntax/AdditionalSyntaxProvider.cs‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
using Pchp.CodeAnalysis;
1111
using Pchp.CodeAnalysis.Symbols;
1212
using Peachpie.CodeAnalysis.Utilities;
13+
using static Peachpie.CodeAnalysis.Syntax.PhpTokenProvider;
1314
using TSpan = Devsense.PHP.Text.Span;
1415
using TValue = Devsense.PHP.Syntax.SemanticValueType;
1516

@@ -40,18 +41,16 @@ public AdditionalSyntaxProvider(PhpTokenProvider provider, NodesFactory nodes, F
4041

4142
#region ITokenProvider
4243

43-
public TValue TokenValue
44-
{
45-
get => _provider.TokenValue;
46-
set => _provider.TokenValue = value;
47-
}
44+
public TValue TokenValue => _provider.TokenValue;
4845

4946
public TSpan TokenPosition => _provider.TokenPosition;
5047

5148
public string TokenText => _provider.TokenText;
5249

5350
public ReadOnlySpan<char> TokenTextSpan => _provider.TokenTextSpan;
5451

52+
public ReadOnlyMemory<char> TokenSource => _provider.TokenSource;
53+
5554
public IDocBlock DocComment { get => _provider.DocComment; set => _provider.DocComment = value; }
5655

5756
public int GetNextToken()
@@ -148,9 +147,9 @@ bool TryMatchSyntaxExtension(Tokens t)
148147
if (t == Tokens.T_FN && (_lastToken == Tokens.T_NS_SEPARATOR || _lastToken == Tokens.T_CLASS))
149148
{
150149
// TODO: warning, use of reserved identifier "fn"
151-
var text = _provider.TokenText;
150+
var source = _provider.TokenSource;
152151

153-
_provider.UpdateToken(new CompleteToken(Tokens.T_STRING, new TValue { String = text }, _provider.TokenPosition, text));
152+
_provider.UpdateToken(new TokenSnapshot(Tokens.T_STRING, _provider.TokenPosition, source, new TValue { String = source.ToString() }));
154153
return true;
155154
}
156155

@@ -430,25 +429,20 @@ bool TryMatchSyntaxExtension(Tokens t)
430429
/// Matches token at position and advances the position to next token.
431430
/// If token does not match, function return <c>false</c> and position is not advanced.
432431
/// </summary>
433-
bool MatchToken(ref int idx, Tokens t) => MatchToken(ref idx, t, out var _, false);
432+
bool MatchToken(ref int idx, Tokens t) => MatchToken(ref idx, t, out var _);
434433

435434
/// <summary>
436435
/// Matches token at position and advances the position to next token.
437436
/// If token does not match, function return <c>false</c> and position is not advanced.
438437
/// </summary>
439-
bool MatchToken(ref int idx, Tokens t, out CompleteToken token, bool ensureTokenText = false)
438+
bool MatchToken(ref int idx, Tokens t, out TokenSnapshot token)
440439
{
441440
var oldidx = idx;
442441

443442
token = NextToken(ref idx);
444443

445444
if (token.Token == t)
446445
{
447-
if (ensureTokenText)
448-
{
449-
token = _provider.WithTokenText(token);
450-
}
451-
452446
return true;
453447
}
454448

@@ -457,9 +451,9 @@ bool MatchToken(ref int idx, Tokens t, out CompleteToken token, bool ensureToken
457451
return false;
458452
}
459453

460-
CompleteToken NextToken(ref int idx)
454+
TokenSnapshot NextToken(ref int idx)
461455
{
462-
CompleteToken token;
456+
TokenSnapshot token;
463457

464458
do
465459
{

‎src/Peachpie.CodeAnalysis/Syntax/PhpSourceUnit.cs‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,29 +38,27 @@ public ReadOnlySpan<char> GetSourceCode(Span span)
3838
}
3939

4040
public void Parse(NodesFactory factory, IErrorSink<Span> errors,
41-
IErrorRecovery recovery = null,
4241
LanguageFeatures features = LanguageFeatures.Basic,
4342
Lexer.LexicalStates state = Lexer.LexicalStates.INITIAL)
4443
{
4544
var parser = new Parser();
4645

47-
using (var source = new StringReader(SourceText.ToString()))
48-
{
49-
using (var provider = new AdditionalSyntaxProvider(
46+
using (
47+
var provider = new AdditionalSyntaxProvider(
5048
new PhpTokenProvider(
51-
new Lexer(source, Encoding.UTF8, errors, features, initialState: state),
52-
this),
49+
new Lexer(SourceText.ToString().AsMemory(), Encoding.UTF8, errors, features, initialState: state),
50+
this
51+
),
5352
factory,
5453
parser.CreateTypeRef))
55-
{
56-
this.Ast = (GlobalCode)parser.Parse(provider, factory, features, errors, recovery);
57-
}
54+
{
55+
this.Ast = (GlobalCode)parser.Parse(provider, factory, features, errors);
5856
}
5957
}
6058

61-
public override void Parse(INodesFactory<LangElement, Span> factory, IErrorSink<Span> errors, IErrorRecovery recovery = null)
59+
public override void Parse(INodesFactory<LangElement, Span> factory, IErrorSink<Span> errors)
6260
{
63-
Parse((NodesFactory)factory, errors, recovery, LanguageFeatures.Basic, Lexer.LexicalStates.INITIAL);
61+
Parse((NodesFactory)factory, errors, LanguageFeatures.Basic, Lexer.LexicalStates.INITIAL);
6462
}
6563

6664
static ILineBreaks CreateLineBreaks(SourceText source) => new SourceLineBreaks(source);

‎src/Peachpie.CodeAnalysis/Syntax/PhpTokenProvider.cs‎

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,49 @@ namespace Peachpie.CodeAnalysis.Syntax
1818
/// </summary>
1919
sealed class PhpTokenProvider : ITokenProvider<TValue, TSpan>, IDisposable
2020
{
21+
#region TokenSnapshot
22+
23+
public readonly struct TokenSnapshot // <ValueType, PositionType>
24+
{
25+
public readonly Tokens Token;
26+
27+
public readonly TSpan TokenPosition;
28+
29+
public readonly ReadOnlyMemory<char> TokenSource;
30+
31+
public readonly TValue TokenValue;
32+
33+
public TokenSnapshot(Tokens token, ITokenProvider<TValue, TSpan> lexer)
34+
: this(token, lexer.TokenPosition, lexer.TokenSource, lexer.TokenValue)
35+
{
36+
}
37+
38+
public TokenSnapshot(Tokens token, TSpan position, ReadOnlyMemory<char> source, TValue value)
39+
{
40+
Token = token;
41+
TokenPosition = position;
42+
TokenSource = source;
43+
TokenValue = value;
44+
}
45+
46+
public TokenSnapshot WithToken(Tokens token) => new TokenSnapshot(token, TokenPosition, TokenSource, TokenValue);
47+
}
48+
49+
#endregion
50+
2151
readonly ITokenProvider<TValue, TSpan> _provider;
52+
2253
readonly PhpSourceUnit _sourceunit;
2354

2455
StringTable _strings;
56+
2557
IDocBlock _docblock;
2658

2759
/// <summary>
2860
/// Buffered tokens.
2961
/// </summary>
30-
readonly List<CompleteToken> _buffer = new List<CompleteToken>();
62+
readonly List<TokenSnapshot> _buffer = new List<TokenSnapshot>();
63+
3164
int _bufferidx = 0;
3265

3366
public PhpTokenProvider(ITokenProvider<TValue, TSpan> provider, PhpSourceUnit sourceunit)
@@ -41,22 +74,9 @@ public PhpTokenProvider(ITokenProvider<TValue, TSpan> provider, PhpSourceUnit so
4174

4275
public Tokens Token => (_bufferidx < _buffer.Count) ? _buffer[_bufferidx].Token : default;
4376

44-
public TValue TokenValue
45-
{
46-
get => (_bufferidx < _buffer.Count) ? _buffer[_bufferidx].TokenValue : _provider.TokenValue;
47-
set
48-
{
49-
if (_bufferidx < _buffer.Count)
50-
{
51-
var previous = _buffer[_bufferidx];
52-
_buffer[_bufferidx] = new CompleteToken(previous.Token, value, previous.TokenPosition, previous.TokenText);
53-
}
54-
else
55-
{
56-
_provider.TokenValue = value;
57-
}
58-
}
59-
}
77+
public TValue TokenValue => (_bufferidx < _buffer.Count) ? _buffer[_bufferidx].TokenValue : _provider.TokenValue;
78+
79+
public ReadOnlyMemory<char> TokenSource => (_bufferidx < _buffer.Count) ? _buffer[_bufferidx].TokenSource : _provider.TokenSource;
6080

6181
public TSpan TokenPosition => (_bufferidx < _buffer.Count) ? _buffer[_bufferidx].TokenPosition : _provider.TokenPosition;
6282

@@ -66,13 +86,7 @@ public string TokenText
6686
{
6787
if (_bufferidx < _buffer.Count)
6888
{
69-
var tinfo = _buffer[_bufferidx];
70-
if (tinfo.TokenText == null)
71-
{
72-
_buffer[_bufferidx] = tinfo = tinfo.WithTokenText(_strings.Add(_sourceunit.GetSourceCode(TokenPosition)));
73-
}
74-
75-
return tinfo.TokenText;
89+
return _buffer[_bufferidx].TokenSource.ToString();
7690
}
7791
else
7892
{
@@ -81,7 +95,7 @@ public string TokenText
8195
}
8296
}
8397

84-
public ReadOnlySpan<char> TokenTextSpan => TokenText.AsSpan();
98+
public ReadOnlySpan<char> TokenTextSpan => TokenSource.Span;
8599

86100
public IDocBlock DocComment
87101
{
@@ -123,7 +137,7 @@ void BufferTokens(int count = 1)
123137
t = (Tokens)_provider.GetNextToken();
124138

125139
// add to buffer
126-
_buffer.Add(new CompleteToken(t, _provider.TokenValue, _provider.TokenPosition, null));
140+
_buffer.Add(new TokenSnapshot(t, _provider));
127141

128142
} while (t != Tokens.END && _buffer.Count < count);
129143
}
@@ -143,15 +157,15 @@ void IDisposable.Dispose()
143157
(_provider as IDisposable)?.Dispose();
144158
}
145159

146-
public void UpdateToken(CompleteToken t)
160+
public void UpdateToken(TokenSnapshot t)
147161
{
148162
_buffer[_bufferidx] = t;
149163
}
150164

151165
/// <summary>
152166
/// Gets token information.
153167
/// </summary>
154-
public CompleteToken Lookup(int index)
168+
public TokenSnapshot Lookup(int index)
155169
{
156170
if (index < 0) throw new ArgumentOutOfRangeException();
157171

@@ -184,15 +198,5 @@ public void Remove(int start, int count)
184198
_buffer[idx] = _buffer[idx].WithToken(Tokens.T_WHITESPACE);
185199
}
186200
}
187-
188-
public CompleteToken WithTokenText(CompleteToken token)
189-
{
190-
if (token.TokenText == null)
191-
{
192-
token = token.WithTokenText(_strings.Add(_sourceunit.GetSourceCode(token.TokenPosition)));
193-
}
194-
195-
return token;
196-
}
197201
}
198202
}

‎src/Peachpie.CodeAnalysis/Utilities/AstUtils.cs‎

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
using Pchp.CodeAnalysis.FlowAnalysis;
1818
using Pchp.CodeAnalysis.Semantics.Graph;
1919
using Peachpie.CodeAnalysis.Syntax;
20+
using static Peachpie.CodeAnalysis.Syntax.PhpTokenProvider;
2021

2122
namespace Pchp.CodeAnalysis
2223
{
@@ -67,34 +68,15 @@ internal static Operations BinaryToCompoundOp(Operations op)
6768
}
6869
}
6970

70-
/// <summary>
71-
/// Creates new struct with updated <see cref="CompleteToken.TokenText"/>.
72-
/// </summary>
73-
public static CompleteToken WithTokenText(this CompleteToken t, string text)
74-
{
75-
return new CompleteToken(t.Token, t.TokenValue, t.TokenPosition, text);
76-
}
77-
78-
/// <summary>
79-
/// Creates new struct with updated <see cref="CompleteToken.TokenText"/>.
80-
/// </summary>
81-
public static CompleteToken WithToken(this CompleteToken t, Tokens token)
82-
{
83-
return new CompleteToken(token, t.TokenValue, t.TokenPosition, t.TokenText);
84-
}
85-
8671
/// <summary>
8772
/// Gets value indicating the token is an ignored token - whitespace or comment.
8873
/// </summary>
89-
public static bool IsWhitespace(this CompleteToken t) => t.Token == Tokens.T_WHITESPACE || t.Token == Tokens.T_COMMENT; // not T_DOC_COMMENT
74+
internal static bool IsWhitespace(this TokenSnapshot t) => t.Token == Tokens.T_WHITESPACE || t.Token == Tokens.T_COMMENT; // not T_DOC_COMMENT
9075

9176
/// <summary>
9277
/// Determines whether method has <c>$this</c> variable.
9378
/// </summary>
94-
public static bool HasThisVariable(MethodDecl method)
95-
{
96-
return method != null && (method.Modifiers & PhpMemberAttributes.Static) == 0;
97-
}
79+
public static bool HasThisVariable(MethodDecl method) => method != null && (method.Modifiers & PhpMemberAttributes.Static) == 0;
9880

9981
public static Span BodySpanOrInvalid(this AstNode routine)
10082
{

‎src/Peachpie.Library.Scripting/Highlight.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public static PhpValue highlight_string(Context ctx, string source, bool @return
9696
xmlwriter.WriteStartElement("span");
9797
xmlwriter.WriteAttributeString("style", $"color: {config.html}");
9898

99-
using (var tokenizer = new Lexer(new StringReader(source ?? string.Empty), Encoding.UTF8))
99+
using (var tokenizer = new Lexer(source.AsMemory(), Encoding.UTF8))
100100
{
101101
Tokens t;
102102
while ((t = tokenizer.GetNextToken()) != Tokens.EOF)

0 commit comments

Comments
 (0)