Namespaces

Classes

Interfaces

Exceptions

1: <?php declare(strict_types=1);
2:
3: namespace PhpParser;
4:
5: use PhpParser\Parser\Tokens;
6:
7: class Lexer
8: {
9: protected $code;
10: protected $tokens;
11: protected $pos;
12: protected $line;
13: protected $filePos;
14: protected $prevCloseTagHasNewline;
15:
16: protected $tokenMap;
17: protected $dropTokens;
18: protected $identifierTokens;
19:
20: private $attributeStartLineUsed;
21: private $attributeEndLineUsed;
22: private $attributeStartTokenPosUsed;
23: private $attributeEndTokenPosUsed;
24: private $attributeStartFilePosUsed;
25: private $attributeEndFilePosUsed;
26: private $attributeCommentsUsed;
27:
28: /**
29: * Creates a Lexer.
30: *
31: * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
32: * which is an array of attributes to add to the AST nodes. Possible
33: * attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
34: * 'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
35: * first three. For more info see getNextToken() docs.
36: */
37: public function __construct(array $options = []) {
38: // Create Map from internal tokens to PhpParser tokens.
39: $this->defineCompatibilityTokens();
40: $this->tokenMap = $this->createTokenMap();
41: $this->identifierTokens = $this->createIdentifierTokenMap();
42:
43: // map of tokens to drop while lexing (the map is only used for isset lookup,
44: // that's why the value is simply set to 1; the value is never actually used.)
45: $this->dropTokens = array_fill_keys(
46: [\T_WHITESPACE, \T_OPEN_TAG, \T_COMMENT, \T_DOC_COMMENT, \T_BAD_CHARACTER], 1
47: );
48:
49: $defaultAttributes = ['comments', 'startLine', 'endLine'];
50: $usedAttributes = array_fill_keys($options['usedAttributes'] ?? $defaultAttributes, true);
51:
52: // Create individual boolean properties to make these checks faster.
53: $this->attributeStartLineUsed = isset($usedAttributes['startLine']);
54: $this->attributeEndLineUsed = isset($usedAttributes['endLine']);
55: $this->attributeStartTokenPosUsed = isset($usedAttributes['startTokenPos']);
56: $this->attributeEndTokenPosUsed = isset($usedAttributes['endTokenPos']);
57: $this->attributeStartFilePosUsed = isset($usedAttributes['startFilePos']);
58: $this->attributeEndFilePosUsed = isset($usedAttributes['endFilePos']);
59: $this->attributeCommentsUsed = isset($usedAttributes['comments']);
60: }
61:
62: /**
63: * Initializes the lexer for lexing the provided source code.
64: *
65: * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
66: * the getErrors() method.
67: *
68: * @param string $code The source code to lex
69: * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
70: * ErrorHandler\Throwing
71: */
72: public function startLexing(string $code, ?ErrorHandler $errorHandler = null) {
73: if (null === $errorHandler) {
74: $errorHandler = new ErrorHandler\Throwing();
75: }
76:
77: $this->code = $code; // keep the code around for __halt_compiler() handling
78: $this->pos = -1;
79: $this->line = 1;
80: $this->filePos = 0;
81:
82: // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
83: // This ensures proper composability, because having a newline is the "safe" assumption.
84: $this->prevCloseTagHasNewline = true;
85:
86: $scream = ini_set('xdebug.scream', '0');
87:
88: $this->tokens = @token_get_all($code);
89: $this->postprocessTokens($errorHandler);
90:
91: if (false !== $scream) {
92: ini_set('xdebug.scream', $scream);
93: }
94: }
95:
96: private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
97: $tokens = [];
98: for ($i = $start; $i < $end; $i++) {
99: $chr = $this->code[$i];
100: if ($chr === "\0") {
101: // PHP cuts error message after null byte, so need special case
102: $errorMsg = 'Unexpected null byte';
103: } else {
104: $errorMsg = sprintf(
105: 'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
106: );
107: }
108:
109: $tokens[] = [\T_BAD_CHARACTER, $chr, $line];
110: $errorHandler->handleError(new Error($errorMsg, [
111: 'startLine' => $line,
112: 'endLine' => $line,
113: 'startFilePos' => $i,
114: 'endFilePos' => $i,
115: ]));
116: }
117: return $tokens;
118: }
119:
120: /**
121: * Check whether comment token is unterminated.
122: *
123: * @return bool
124: */
125: private function isUnterminatedComment($token) : bool {
126: return ($token[0] === \T_COMMENT || $token[0] === \T_DOC_COMMENT)
127: && substr($token[1], 0, 2) === '/*'
128: && substr($token[1], -2) !== '*/';
129: }
130:
131: protected function postprocessTokens(ErrorHandler $errorHandler) {
132: // PHP's error handling for token_get_all() is rather bad, so if we want detailed
133: // error information we need to compute it ourselves. Invalid character errors are
134: // detected by finding "gaps" in the token array. Unterminated comments are detected
135: // by checking if a trailing comment has a "*/" at the end.
136: //
137: // Additionally, we perform a number of canonicalizations here:
138: // * Use the PHP 8.0 comment format, which does not include trailing whitespace anymore.
139: // * Use PHP 8.0 T_NAME_* tokens.
140: // * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
141: // T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
142:
143: $filePos = 0;
144: $line = 1;
145: $numTokens = \count($this->tokens);
146: for ($i = 0; $i < $numTokens; $i++) {
147: $token = $this->tokens[$i];
148:
149: // Since PHP 7.4 invalid characters are represented by a T_BAD_CHARACTER token.
150: // In this case we only need to emit an error.
151: if ($token[0] === \T_BAD_CHARACTER) {
152: $this->handleInvalidCharacterRange($filePos, $filePos + 1, $line, $errorHandler);
153: }
154:
155: if ($token[0] === \T_COMMENT && substr($token[1], 0, 2) !== '/*'
156: && preg_match('/(\r\n|\n|\r)$/D', $token[1], $matches)) {
157: $trailingNewline = $matches[0];
158: $token[1] = substr($token[1], 0, -strlen($trailingNewline));
159: $this->tokens[$i] = $token;
160: if (isset($this->tokens[$i + 1]) && $this->tokens[$i + 1][0] === \T_WHITESPACE) {
161: // Move trailing newline into following T_WHITESPACE token, if it already exists.
162: $this->tokens[$i + 1][1] = $trailingNewline . $this->tokens[$i + 1][1];
163: $this->tokens[$i + 1][2]--;
164: } else {
165: // Otherwise, we need to create a new T_WHITESPACE token.
166: array_splice($this->tokens, $i + 1, 0, [
167: [\T_WHITESPACE, $trailingNewline, $line],
168: ]);
169: $numTokens++;
170: }
171: }
172:
173: // Emulate PHP 8 T_NAME_* tokens, by combining sequences of T_NS_SEPARATOR and T_STRING
174: // into a single token.
175: if (\is_array($token)
176: && ($token[0] === \T_NS_SEPARATOR || isset($this->identifierTokens[$token[0]]))) {
177: $lastWasSeparator = $token[0] === \T_NS_SEPARATOR;
178: $text = $token[1];
179: for ($j = $i + 1; isset($this->tokens[$j]); $j++) {
180: if ($lastWasSeparator) {
181: if (!isset($this->identifierTokens[$this->tokens[$j][0]])) {
182: break;
183: }
184: $lastWasSeparator = false;
185: } else {
186: if ($this->tokens[$j][0] !== \T_NS_SEPARATOR) {
187: break;
188: }
189: $lastWasSeparator = true;
190: }
191: $text .= $this->tokens[$j][1];
192: }
193: if ($lastWasSeparator) {
194: // Trailing separator is not part of the name.
195: $j--;
196: $text = substr($text, 0, -1);
197: }
198: if ($j > $i + 1) {
199: if ($token[0] === \T_NS_SEPARATOR) {
200: $type = \T_NAME_FULLY_QUALIFIED;
201: } else if ($token[0] === \T_NAMESPACE) {
202: $type = \T_NAME_RELATIVE;
203: } else {
204: $type = \T_NAME_QUALIFIED;
205: }
206: $token = [$type, $text, $line];
207: array_splice($this->tokens, $i, $j - $i, [$token]);
208: $numTokens -= $j - $i - 1;
209: }
210: }
211:
212: if ($token === '&') {
213: $next = $i + 1;
214: while (isset($this->tokens[$next]) && $this->tokens[$next][0] === \T_WHITESPACE) {
215: $next++;
216: }
217: $followedByVarOrVarArg = isset($this->tokens[$next]) &&
218: ($this->tokens[$next][0] === \T_VARIABLE || $this->tokens[$next][0] === \T_ELLIPSIS);
219: $this->tokens[$i] = $token = [
220: $followedByVarOrVarArg
221: ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG
222: : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG,
223: '&',
224: $line,
225: ];
226: }
227:
228: $tokenValue = \is_string($token) ? $token : $token[1];
229: $tokenLen = \strlen($tokenValue);
230:
231: if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
232: // Something is missing, must be an invalid character
233: $nextFilePos = strpos($this->code, $tokenValue, $filePos);
234: $badCharTokens = $this->handleInvalidCharacterRange(
235: $filePos, $nextFilePos, $line, $errorHandler);
236: $filePos = (int) $nextFilePos;
237:
238: array_splice($this->tokens, $i, 0, $badCharTokens);
239: $numTokens += \count($badCharTokens);
240: $i += \count($badCharTokens);
241: }
242:
243: $filePos += $tokenLen;
244: $line += substr_count($tokenValue, "\n");
245: }
246:
247: if ($filePos !== \strlen($this->code)) {
248: if (substr($this->code, $filePos, 2) === '/*') {
249: // Unlike PHP, HHVM will drop unterminated comments entirely
250: $comment = substr($this->code, $filePos);
251: $errorHandler->handleError(new Error('Unterminated comment', [
252: 'startLine' => $line,
253: 'endLine' => $line + substr_count($comment, "\n"),
254: 'startFilePos' => $filePos,
255: 'endFilePos' => $filePos + \strlen($comment),
256: ]));
257:
258: // Emulate the PHP behavior
259: $isDocComment = isset($comment[3]) && $comment[3] === '*';
260: $this->tokens[] = [$isDocComment ? \T_DOC_COMMENT : \T_COMMENT, $comment, $line];
261: } else {
262: // Invalid characters at the end of the input
263: $badCharTokens = $this->handleInvalidCharacterRange(
264: $filePos, \strlen($this->code), $line, $errorHandler);
265: $this->tokens = array_merge($this->tokens, $badCharTokens);
266: }
267: return;
268: }
269:
270: if (count($this->tokens) > 0) {
271: // Check for unterminated comment
272: $lastToken = $this->tokens[count($this->tokens) - 1];
273: if ($this->isUnterminatedComment($lastToken)) {
274: $errorHandler->handleError(new Error('Unterminated comment', [
275: 'startLine' => $line - substr_count($lastToken[1], "\n"),
276: 'endLine' => $line,
277: 'startFilePos' => $filePos - \strlen($lastToken[1]),
278: 'endFilePos' => $filePos,
279: ]));
280: }
281: }
282: }
283:
284: /**
285: * Fetches the next token.
286: *
287: * The available attributes are determined by the 'usedAttributes' option, which can
288: * be specified in the constructor. The following attributes are supported:
289: *
290: * * 'comments' => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
291: * representing all comments that occurred between the previous
292: * non-discarded token and the current one.
293: * * 'startLine' => Line in which the node starts.
294: * * 'endLine' => Line in which the node ends.
295: * * 'startTokenPos' => Offset into the token array of the first token in the node.
296: * * 'endTokenPos' => Offset into the token array of the last token in the node.
297: * * 'startFilePos' => Offset into the code string of the first character that is part of the node.
298: * * 'endFilePos' => Offset into the code string of the last character that is part of the node.
299: *
300: * @param mixed $value Variable to store token content in
301: * @param mixed $startAttributes Variable to store start attributes in
302: * @param mixed $endAttributes Variable to store end attributes in
303: *
304: * @return int Token id
305: */
306: public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) : int {
307: $startAttributes = [];
308: $endAttributes = [];
309:
310: while (1) {
311: if (isset($this->tokens[++$this->pos])) {
312: $token = $this->tokens[$this->pos];
313: } else {
314: // EOF token with ID 0
315: $token = "\0";
316: }
317:
318: if ($this->attributeStartLineUsed) {
319: $startAttributes['startLine'] = $this->line;
320: }
321: if ($this->attributeStartTokenPosUsed) {
322: $startAttributes['startTokenPos'] = $this->pos;
323: }
324: if ($this->attributeStartFilePosUsed) {
325: $startAttributes['startFilePos'] = $this->filePos;
326: }
327:
328: if (\is_string($token)) {
329: $value = $token;
330: if (isset($token[1])) {
331: // bug in token_get_all
332: $this->filePos += 2;
333: $id = ord('"');
334: } else {
335: $this->filePos += 1;
336: $id = ord($token);
337: }
338: } elseif (!isset($this->dropTokens[$token[0]])) {
339: $value = $token[1];
340: $id = $this->tokenMap[$token[0]];
341: if (\T_CLOSE_TAG === $token[0]) {
342: $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n")
343: || false !== strpos($token[1], "\r");
344: } elseif (\T_INLINE_HTML === $token[0]) {
345: $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
346: }
347:
348: $this->line += substr_count($value, "\n");
349: $this->filePos += \strlen($value);
350: } else {
351: $origLine = $this->line;
352: $origFilePos = $this->filePos;
353: $this->line += substr_count($token[1], "\n");
354: $this->filePos += \strlen($token[1]);
355:
356: if (\T_COMMENT === $token[0] || \T_DOC_COMMENT === $token[0]) {
357: if ($this->attributeCommentsUsed) {
358: $comment = \T_DOC_COMMENT === $token[0]
359: ? new Comment\Doc($token[1],
360: $origLine, $origFilePos, $this->pos,
361: $this->line, $this->filePos - 1, $this->pos)
362: : new Comment($token[1],
363: $origLine, $origFilePos, $this->pos,
364: $this->line, $this->filePos - 1, $this->pos);
365: $startAttributes['comments'][] = $comment;
366: }
367: }
368: continue;
369: }
370:
371: if ($this->attributeEndLineUsed) {
372: $endAttributes['endLine'] = $this->line;
373: }
374: if ($this->attributeEndTokenPosUsed) {
375: $endAttributes['endTokenPos'] = $this->pos;
376: }
377: if ($this->attributeEndFilePosUsed) {
378: $endAttributes['endFilePos'] = $this->filePos - 1;
379: }
380:
381: return $id;
382: }
383:
384: throw new \RuntimeException('Reached end of lexer loop');
385: }
386:
387: /**
388: * Returns the token array for current code.
389: *
390: * The token array is in the same format as provided by the
391: * token_get_all() function and does not discard tokens (i.e.
392: * whitespace and comments are included). The token position
393: * attributes are against this token array.
394: *
395: * @return array Array of tokens in token_get_all() format
396: */
397: public function getTokens() : array {
398: return $this->tokens;
399: }
400:
401: /**
402: * Handles __halt_compiler() by returning the text after it.
403: *
404: * @return string Remaining text
405: */
406: public function handleHaltCompiler() : string {
407: // text after T_HALT_COMPILER, still including ();
408: $textAfter = substr($this->code, $this->filePos);
409:
410: // ensure that it is followed by ();
411: // this simplifies the situation, by not allowing any comments
412: // in between of the tokens.
413: if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
414: throw new Error('__HALT_COMPILER must be followed by "();"');
415: }
416:
417: // prevent the lexer from returning any further tokens
418: $this->pos = count($this->tokens);
419:
420: // return with (); removed
421: return substr($textAfter, strlen($matches[0]));
422: }
423:
424: private function defineCompatibilityTokens() {
425: static $compatTokensDefined = false;
426: if ($compatTokensDefined) {
427: return;
428: }
429:
430: $compatTokens = [
431: // PHP 7.4
432: 'T_BAD_CHARACTER',
433: 'T_FN',
434: 'T_COALESCE_EQUAL',
435: // PHP 8.0
436: 'T_NAME_QUALIFIED',
437: 'T_NAME_FULLY_QUALIFIED',
438: 'T_NAME_RELATIVE',
439: 'T_MATCH',
440: 'T_NULLSAFE_OBJECT_OPERATOR',
441: 'T_ATTRIBUTE',
442: // PHP 8.1
443: 'T_ENUM',
444: 'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
445: 'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
446: 'T_READONLY',
447: ];
448:
449: // PHP-Parser might be used together with another library that also emulates some or all
450: // of these tokens. Perform a sanity-check that all already defined tokens have been
451: // assigned a unique ID.
452: $usedTokenIds = [];
453: foreach ($compatTokens as $token) {
454: if (\defined($token)) {
455: $tokenId = \constant($token);
456: $clashingToken = $usedTokenIds[$tokenId] ?? null;
457: if ($clashingToken !== null) {
458: throw new \Error(sprintf(
459: 'Token %s has same ID as token %s, ' .
460: 'you may be using a library with broken token emulation',
461: $token, $clashingToken
462: ));
463: }
464: $usedTokenIds[$tokenId] = $token;
465: }
466: }
467:
468: // Now define any tokens that have not yet been emulated. Try to assign IDs from -1
469: // downwards, but skip any IDs that may already be in use.
470: $newTokenId = -1;
471: foreach ($compatTokens as $token) {
472: if (!\defined($token)) {
473: while (isset($usedTokenIds[$newTokenId])) {
474: $newTokenId--;
475: }
476: \define($token, $newTokenId);
477: $newTokenId--;
478: }
479: }
480:
481: $compatTokensDefined = true;
482: }
483:
484: /**
485: * Creates the token map.
486: *
487: * The token map maps the PHP internal token identifiers
488: * to the identifiers used by the Parser. Additionally it
489: * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
490: *
491: * @return array The token map
492: */
493: protected function createTokenMap() : array {
494: $tokenMap = [];
495:
496: // 256 is the minimum possible token number, as everything below
497: // it is an ASCII value
498: for ($i = 256; $i < 1000; ++$i) {
499: if (\T_DOUBLE_COLON === $i) {
500: // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
501: $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
502: } elseif(\T_OPEN_TAG_WITH_ECHO === $i) {
503: // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
504: $tokenMap[$i] = Tokens::T_ECHO;
505: } elseif(\T_CLOSE_TAG === $i) {
506: // T_CLOSE_TAG is equivalent to ';'
507: $tokenMap[$i] = ord(';');
508: } elseif ('UNKNOWN' !== $name = token_name($i)) {
509: if ('T_HASHBANG' === $name) {
510: // HHVM uses a special token for #! hashbang lines
511: $tokenMap[$i] = Tokens::T_INLINE_HTML;
512: } elseif (defined($name = Tokens::class . '::' . $name)) {
513: // Other tokens can be mapped directly
514: $tokenMap[$i] = constant($name);
515: }
516: }
517: }
518:
519: // HHVM uses a special token for numbers that overflow to double
520: if (defined('T_ONUMBER')) {
521: $tokenMap[\T_ONUMBER] = Tokens::T_DNUMBER;
522: }
523: // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
524: if (defined('T_COMPILER_HALT_OFFSET')) {
525: $tokenMap[\T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
526: }
527:
528: // Assign tokens for which we define compatibility constants, as token_name() does not know them.
529: $tokenMap[\T_FN] = Tokens::T_FN;
530: $tokenMap[\T_COALESCE_EQUAL] = Tokens::T_COALESCE_EQUAL;
531: $tokenMap[\T_NAME_QUALIFIED] = Tokens::T_NAME_QUALIFIED;
532: $tokenMap[\T_NAME_FULLY_QUALIFIED] = Tokens::T_NAME_FULLY_QUALIFIED;
533: $tokenMap[\T_NAME_RELATIVE] = Tokens::T_NAME_RELATIVE;
534: $tokenMap[\T_MATCH] = Tokens::T_MATCH;
535: $tokenMap[\T_NULLSAFE_OBJECT_OPERATOR] = Tokens::T_NULLSAFE_OBJECT_OPERATOR;
536: $tokenMap[\T_ATTRIBUTE] = Tokens::T_ATTRIBUTE;
537: $tokenMap[\T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
538: $tokenMap[\T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG] = Tokens::T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG;
539: $tokenMap[\T_ENUM] = Tokens::T_ENUM;
540: $tokenMap[\T_READONLY] = Tokens::T_READONLY;
541:
542: return $tokenMap;
543: }
544:
545: private function createIdentifierTokenMap(): array {
546: // Based on semi_reserved production.
547: return array_fill_keys([
548: \T_STRING,
549: \T_STATIC, \T_ABSTRACT, \T_FINAL, \T_PRIVATE, \T_PROTECTED, \T_PUBLIC, \T_READONLY,
550: \T_INCLUDE, \T_INCLUDE_ONCE, \T_EVAL, \T_REQUIRE, \T_REQUIRE_ONCE, \T_LOGICAL_OR, \T_LOGICAL_XOR, \T_LOGICAL_AND,
551: \T_INSTANCEOF, \T_NEW, \T_CLONE, \T_EXIT, \T_IF, \T_ELSEIF, \T_ELSE, \T_ENDIF, \T_ECHO, \T_DO, \T_WHILE,
552: \T_ENDWHILE, \T_FOR, \T_ENDFOR, \T_FOREACH, \T_ENDFOREACH, \T_DECLARE, \T_ENDDECLARE, \T_AS, \T_TRY, \T_CATCH,
553: \T_FINALLY, \T_THROW, \T_USE, \T_INSTEADOF, \T_GLOBAL, \T_VAR, \T_UNSET, \T_ISSET, \T_EMPTY, \T_CONTINUE, \T_GOTO,
554: \T_FUNCTION, \T_CONST, \T_RETURN, \T_PRINT, \T_YIELD, \T_LIST, \T_SWITCH, \T_ENDSWITCH, \T_CASE, \T_DEFAULT,
555: \T_BREAK, \T_ARRAY, \T_CALLABLE, \T_EXTENDS, \T_IMPLEMENTS, \T_NAMESPACE, \T_TRAIT, \T_INTERFACE, \T_CLASS,
556: \T_CLASS_C, \T_TRAIT_C, \T_FUNC_C, \T_METHOD_C, \T_LINE, \T_FILE, \T_DIR, \T_NS_C, \T_HALT_COMPILER, \T_FN,
557: \T_MATCH,
558: ], true);
559: }
560: }
561:
PHPStan API documentation generated by ApiGen dev-master