File vendor/phpstan/phpdoc-parser/src/Lexer/Lexer.php

1:	<?php declare(strict_types = 1);
2:
3:	namespace PHPStan\PhpDocParser\Lexer;
4:
5:	use PHPStan\PhpDocParser\ParserConfig;
6:	use function implode;
7:	use function preg_match_all;
8:
9:	/**
10:	* Implementation based on Nette Tokenizer (New BSD License; https://github.com/nette/tokenizer)
11:	*/
12:	class Lexer
13:	{
14:
15:	public const TOKEN_REFERENCE = 0;
16:	public const TOKEN_UNION = 1;
17:	public const TOKEN_INTERSECTION = 2;
18:	public const TOKEN_NULLABLE = 3;
19:	public const TOKEN_OPEN_PARENTHESES = 4;
20:	public const TOKEN_CLOSE_PARENTHESES = 5;
21:	public const TOKEN_OPEN_ANGLE_BRACKET = 6;
22:	public const TOKEN_CLOSE_ANGLE_BRACKET = 7;
23:	public const TOKEN_OPEN_SQUARE_BRACKET = 8;
24:	public const TOKEN_CLOSE_SQUARE_BRACKET = 9;
25:	public const TOKEN_COMMA = 10;
26:	public const TOKEN_VARIADIC = 11;
27:	public const TOKEN_DOUBLE_COLON = 12;
28:	public const TOKEN_DOUBLE_ARROW = 13;
29:	public const TOKEN_EQUAL = 14;
30:	public const TOKEN_OPEN_PHPDOC = 15;
31:	public const TOKEN_CLOSE_PHPDOC = 16;
32:	public const TOKEN_PHPDOC_TAG = 17;
33:	public const TOKEN_DOCTRINE_TAG = 18;
34:	public const TOKEN_FLOAT = 19;
35:	public const TOKEN_INTEGER = 20;
36:	public const TOKEN_SINGLE_QUOTED_STRING = 21;
37:	public const TOKEN_DOUBLE_QUOTED_STRING = 22;
38:	public const TOKEN_DOCTRINE_ANNOTATION_STRING = 23;
39:	public const TOKEN_IDENTIFIER = 24;
40:	public const TOKEN_THIS_VARIABLE = 25;
41:	public const TOKEN_VARIABLE = 26;
42:	public const TOKEN_HORIZONTAL_WS = 27;
43:	public const TOKEN_PHPDOC_EOL = 28;
44:	public const TOKEN_OTHER = 29;
45:	public const TOKEN_END = 30;
46:	public const TOKEN_COLON = 31;
47:	public const TOKEN_WILDCARD = 32;
48:	public const TOKEN_OPEN_CURLY_BRACKET = 33;
49:	public const TOKEN_CLOSE_CURLY_BRACKET = 34;
50:	public const TOKEN_NEGATED = 35;
51:	public const TOKEN_ARROW = 36;
52:
53:	public const TOKEN_COMMENT = 37;
54:
55:	public const TOKEN_LABELS = [
56:	self::TOKEN_REFERENCE => '\'&\'',
57:	self::TOKEN_UNION => '\'\|\'',
58:	self::TOKEN_INTERSECTION => '\'&\'',
59:	self::TOKEN_NULLABLE => '\'?\'',
60:	self::TOKEN_NEGATED => '\'!\'',
61:	self::TOKEN_OPEN_PARENTHESES => '\'(\'',
62:	self::TOKEN_CLOSE_PARENTHESES => '\')\'',
63:	self::TOKEN_OPEN_ANGLE_BRACKET => '\'<\'',
64:	self::TOKEN_CLOSE_ANGLE_BRACKET => '\'>\'',
65:	self::TOKEN_OPEN_SQUARE_BRACKET => '\'[\'',
66:	self::TOKEN_CLOSE_SQUARE_BRACKET => '\']\'',
67:	self::TOKEN_OPEN_CURLY_BRACKET => '\'{\'',
68:	self::TOKEN_CLOSE_CURLY_BRACKET => '\'}\'',
69:	self::TOKEN_COMMA => '\',\'',
70:	self::TOKEN_COMMENT => '\'//\'',
71:	self::TOKEN_COLON => '\':\'',
72:	self::TOKEN_VARIADIC => '\'...\'',
73:	self::TOKEN_DOUBLE_COLON => '\'::\'',
74:	self::TOKEN_DOUBLE_ARROW => '\'=>\'',
75:	self::TOKEN_ARROW => '\'->\'',
76:	self::TOKEN_EQUAL => '\'=\'',
77:	self::TOKEN_OPEN_PHPDOC => '\'/**\'',
78:	self::TOKEN_CLOSE_PHPDOC => '\'*/\'',
79:	self::TOKEN_PHPDOC_TAG => 'TOKEN_PHPDOC_TAG',
80:	self::TOKEN_DOCTRINE_TAG => 'TOKEN_DOCTRINE_TAG',
81:	self::TOKEN_PHPDOC_EOL => 'TOKEN_PHPDOC_EOL',
82:	self::TOKEN_FLOAT => 'TOKEN_FLOAT',
83:	self::TOKEN_INTEGER => 'TOKEN_INTEGER',
84:	self::TOKEN_SINGLE_QUOTED_STRING => 'TOKEN_SINGLE_QUOTED_STRING',
85:	self::TOKEN_DOUBLE_QUOTED_STRING => 'TOKEN_DOUBLE_QUOTED_STRING',
86:	self::TOKEN_DOCTRINE_ANNOTATION_STRING => 'TOKEN_DOCTRINE_ANNOTATION_STRING',
87:	self::TOKEN_IDENTIFIER => 'type',
88:	self::TOKEN_THIS_VARIABLE => '\'$this\'',
89:	self::TOKEN_VARIABLE => 'variable',
90:	self::TOKEN_HORIZONTAL_WS => 'TOKEN_HORIZONTAL_WS',
91:	self::TOKEN_OTHER => 'TOKEN_OTHER',
92:	self::TOKEN_END => 'TOKEN_END',
93:	self::TOKEN_WILDCARD => '*',
94:	];
95:
96:	public const VALUE_OFFSET = 0;
97:	public const TYPE_OFFSET = 1;
98:	public const LINE_OFFSET = 2;
99:
100:	private ParserConfig $config; // @phpstan-ignore property.onlyWritten
101:
102:	private ?string $regexp = null;
103:
104:	public function __construct(ParserConfig $config)
105:	{
106:	$this->config = $config;
107:	}
108:
109:	/**
110:	* @return list<array{string, int, int}>
111:	*/
112:	public function tokenize(string $s): array
113:	{
114:	if ($this->regexp === null) {
115:	$this->regexp = $this->generateRegexp();
116:	}
117:
118:	// PREG_PATTERN_ORDER, not PREG_SET_ORDER: it collects the whole PHPDoc
119:	// into two arrays instead of allocating one array per token. This only
120:	// pays off while the token patterns have no capturing groups, because
121:	// every group would get an array of its own, one entry per token.
122:	preg_match_all($this->regexp, $s, $matches);
123:
124:	$values = $matches[0];
125:	if ($values === []) {
126:	return [['', self::TOKEN_END, 1]];
127:	}
128:
129:	$marks = $matches['MARK'];
130:
131:	$tokens = [];
132:	$line = 1;
133:	foreach ($values as $i => $value) {
134:	$type = (int) $marks[$i];
135:	$tokens[] = [$value, $type, $line];
136:	if ($type !== self::TOKEN_PHPDOC_EOL) {
137:	continue;
138:	}
139:
140:	$line++;
141:	}
142:
143:	$tokens[] = ['', self::TOKEN_END, $line];
144:
145:	return $tokens;
146:	}
147:
148:	private function generateRegexp(): string
149:	{
150:	// every group in here must be non-capturing, see tokenize()
151:	$patterns = [
152:	self::TOKEN_HORIZONTAL_WS => '[\\x09\\x20]++',
153:
154:	self::TOKEN_IDENTIFIER => '(?:[\\\\]?+[a-z_\\x80-\\xFF][0-9a-z_\\x80-\\xFF-]*+)++',
155:	self::TOKEN_THIS_VARIABLE => '\\$this(?![0-9a-z_\\x80-\\xFF])',
156:	self::TOKEN_VARIABLE => '\\$[a-z_\\x80-\\xFF][0-9a-z_\\x80-\\xFF]*+',
157:
158:	// '&' followed by TOKEN_VARIADIC, TOKEN_VARIABLE, TOKEN_EQUAL, TOKEN_EQUAL or TOKEN_CLOSE_PARENTHESES
159:	self::TOKEN_REFERENCE => '&(?=\\s*+(?:[,=)]\|\\.\\.\\.\|(?:\\$(?!this(?![0-9a-z_\\x80-\\xFF])))))',
160:	self::TOKEN_UNION => '\\\|',
161:	self::TOKEN_INTERSECTION => '&',
162:	self::TOKEN_NULLABLE => '\\?',
163:	self::TOKEN_NEGATED => '!',
164:
165:	self::TOKEN_OPEN_PARENTHESES => '\\(',
166:	self::TOKEN_CLOSE_PARENTHESES => '\\)',
167:	self::TOKEN_OPEN_ANGLE_BRACKET => '<',
168:	self::TOKEN_CLOSE_ANGLE_BRACKET => '>',
169:	self::TOKEN_OPEN_SQUARE_BRACKET => '\\[',
170:	self::TOKEN_CLOSE_SQUARE_BRACKET => '\\]',
171:	self::TOKEN_OPEN_CURLY_BRACKET => '\\{',
172:	self::TOKEN_CLOSE_CURLY_BRACKET => '\\}',
173:
174:	self::TOKEN_COMMA => ',',
175:	self::TOKEN_COMMENT => '\/\/[^\\r\\n](?=\n\|\r\|\/)',
176:	self::TOKEN_VARIADIC => '\\.\\.\\.',
177:	self::TOKEN_DOUBLE_COLON => '::',
178:	self::TOKEN_DOUBLE_ARROW => '=>',
179:	self::TOKEN_ARROW => '->',
180:	self::TOKEN_EQUAL => '=',
181:	self::TOKEN_COLON => ':',
182:
183:	self::TOKEN_OPEN_PHPDOC => '/\\\\(?=\\s)\\x20?+',
184:	self::TOKEN_CLOSE_PHPDOC => '\\*/',
185:	self::TOKEN_PHPDOC_TAG => '@(?:[a-z][a-z0-9-\\\\]+:)?[a-z][a-z0-9-\\\\]*+',
186:	self::TOKEN_DOCTRINE_TAG => '@[a-z_\\\\][a-z0-9_\:\\\\][a-z_][a-z0-9_]',
187:	self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]+(?:\\(?!/)\\x20?+)?',
188:
189:	self::TOKEN_FLOAT => '[+\-]?(?:(?:[0-9]++(?:_[0-9]++)\\.[0-9]+(?:_[0-9]++)(?:e[+\-]?[0-9]++(?:_[0-9]++))?)\|(?:[0-9]+(?:_[0-9]++)\\.[0-9]++(?:_[0-9]++)(?:e[+\-]?[0-9]++(?:_[0-9]++))?)\|(?:[0-9]++(?:_[0-9]++)e[+\-]?[0-9]++(?:_[0-9]++)))',
190:	self::TOKEN_INTEGER => '[+\-]?(?:(?:0b[0-1]++(?:_[0-1]++))\|(?:0o[0-7]++(?:_[0-7]++))\|(?:0x[0-9a-f]++(?:_[0-9a-f]++))\|(?:[0-9]++(?:_[0-9]++)))',
191:	self::TOKEN_SINGLE_QUOTED_STRING => '\'(?:\\\\[^\\r\\n]\|[^\'\\r\\n\\\\])*+\'',
192:	self::TOKEN_DOUBLE_QUOTED_STRING => '"(?:\\\\[^\\r\\n]\|[^"\\r\\n\\\\])*+"',
193:	self::TOKEN_DOCTRINE_ANNOTATION_STRING => '"(?:""\|[^"])*+"',
194:
195:	self::TOKEN_WILDCARD => '\\*',
196:
197:	// anything but TOKEN_CLOSE_PHPDOC or TOKEN_HORIZONTAL_WS or TOKEN_EOL
198:	self::TOKEN_OTHER => '(?:(?!\\*/)[^\\s])++',
199:	];
200:
201:	foreach ($patterns as $type => &$pattern) {
202:	$pattern = '(?:' . $pattern . ')(*MARK:' . $type . ')';
203:	}
204:
205:	return '~' . implode('\|', $patterns) . '~Asi';
206:	}
207:
208:	}
209:

Namespaces

Classes