Lexer.php 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. <?php
  2. namespace PhpParser;
  3. use PhpParser\Parser\Tokens;
  4. class Lexer
  5. {
  6. protected $code;
  7. protected $tokens;
  8. protected $pos;
  9. protected $line;
  10. protected $filePos;
  11. protected $prevCloseTagHasNewline;
  12. protected $tokenMap;
  13. protected $dropTokens;
  14. protected $usedAttributes;
  15. /**
  16. * Creates a Lexer.
  17. *
  18. * @param array $options Options array. Currently only the 'usedAttributes' option is supported,
  19. * which is an array of attributes to add to the AST nodes. Possible
  20. * attributes are: 'comments', 'startLine', 'endLine', 'startTokenPos',
  21. * 'endTokenPos', 'startFilePos', 'endFilePos'. The option defaults to the
  22. * first three. For more info see getNextToken() docs.
  23. */
  24. public function __construct(array $options = array()) {
  25. // map from internal tokens to PhpParser tokens
  26. $this->tokenMap = $this->createTokenMap();
  27. // map of tokens to drop while lexing (the map is only used for isset lookup,
  28. // that's why the value is simply set to 1; the value is never actually used.)
  29. $this->dropTokens = array_fill_keys(
  30. array(T_WHITESPACE, T_OPEN_TAG, T_COMMENT, T_DOC_COMMENT), 1
  31. );
  32. // the usedAttributes member is a map of the used attribute names to a dummy
  33. // value (here "true")
  34. $options += array(
  35. 'usedAttributes' => array('comments', 'startLine', 'endLine'),
  36. );
  37. $this->usedAttributes = array_fill_keys($options['usedAttributes'], true);
  38. }
  39. /**
  40. * Initializes the lexer for lexing the provided source code.
  41. *
  42. * This function does not throw if lexing errors occur. Instead, errors may be retrieved using
  43. * the getErrors() method.
  44. *
  45. * @param string $code The source code to lex
  46. * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
  47. * ErrorHandler\Throwing
  48. */
  49. public function startLexing($code, ErrorHandler $errorHandler = null) {
  50. if (null === $errorHandler) {
  51. $errorHandler = new ErrorHandler\Throwing();
  52. }
  53. $this->code = $code; // keep the code around for __halt_compiler() handling
  54. $this->pos = -1;
  55. $this->line = 1;
  56. $this->filePos = 0;
  57. // If inline HTML occurs without preceding code, treat it as if it had a leading newline.
  58. // This ensures proper composability, because having a newline is the "safe" assumption.
  59. $this->prevCloseTagHasNewline = true;
  60. $scream = ini_set('xdebug.scream', '0');
  61. $this->resetErrors();
  62. $this->tokens = @token_get_all($code);
  63. $this->handleErrors($errorHandler);
  64. if (false !== $scream) {
  65. ini_set('xdebug.scream', $scream);
  66. }
  67. }
  68. protected function resetErrors() {
  69. if (function_exists('error_clear_last')) {
  70. error_clear_last();
  71. } else {
  72. // set error_get_last() to defined state by forcing an undefined variable error
  73. set_error_handler(function() { return false; }, 0);
  74. @$undefinedVariable;
  75. restore_error_handler();
  76. }
  77. }
  78. private function handleInvalidCharacterRange($start, $end, $line, ErrorHandler $errorHandler) {
  79. for ($i = $start; $i < $end; $i++) {
  80. $chr = $this->code[$i];
  81. if ($chr === 'b' || $chr === 'B') {
  82. // HHVM does not treat b" tokens correctly, so ignore these
  83. continue;
  84. }
  85. if ($chr === "\0") {
  86. // PHP cuts error message after null byte, so need special case
  87. $errorMsg = 'Unexpected null byte';
  88. } else {
  89. $errorMsg = sprintf(
  90. 'Unexpected character "%s" (ASCII %d)', $chr, ord($chr)
  91. );
  92. }
  93. $errorHandler->handleError(new Error($errorMsg, [
  94. 'startLine' => $line,
  95. 'endLine' => $line,
  96. 'startFilePos' => $i,
  97. 'endFilePos' => $i,
  98. ]));
  99. }
  100. }
  101. private function isUnterminatedComment($token) {
  102. return ($token[0] === T_COMMENT || $token[0] === T_DOC_COMMENT)
  103. && substr($token[1], 0, 2) === '/*'
  104. && substr($token[1], -2) !== '*/';
  105. }
  106. private function errorMayHaveOccurred() {
  107. if (defined('HHVM_VERSION')) {
  108. // In HHVM token_get_all() does not throw warnings, so we need to conservatively
  109. // assume that an error occurred
  110. return true;
  111. }
  112. $error = error_get_last();
  113. return null !== $error
  114. && false === strpos($error['message'], 'Undefined variable');
  115. }
  116. protected function handleErrors(ErrorHandler $errorHandler) {
  117. if (!$this->errorMayHaveOccurred()) {
  118. return;
  119. }
  120. // PHP's error handling for token_get_all() is rather bad, so if we want detailed
  121. // error information we need to compute it ourselves. Invalid character errors are
  122. // detected by finding "gaps" in the token array. Unterminated comments are detected
  123. // by checking if a trailing comment has a "*/" at the end.
  124. $filePos = 0;
  125. $line = 1;
  126. foreach ($this->tokens as $i => $token) {
  127. $tokenValue = \is_string($token) ? $token : $token[1];
  128. $tokenLen = \strlen($tokenValue);
  129. if (substr($this->code, $filePos, $tokenLen) !== $tokenValue) {
  130. // Something is missing, must be an invalid character
  131. $nextFilePos = strpos($this->code, $tokenValue, $filePos);
  132. $this->handleInvalidCharacterRange(
  133. $filePos, $nextFilePos, $line, $errorHandler);
  134. $filePos = $nextFilePos;
  135. }
  136. $filePos += $tokenLen;
  137. $line += substr_count($tokenValue, "\n");
  138. }
  139. if ($filePos !== \strlen($this->code)) {
  140. if (substr($this->code, $filePos, 2) === '/*') {
  141. // Unlike PHP, HHVM will drop unterminated comments entirely
  142. $comment = substr($this->code, $filePos);
  143. $errorHandler->handleError(new Error('Unterminated comment', [
  144. 'startLine' => $line,
  145. 'endLine' => $line + substr_count($comment, "\n"),
  146. 'startFilePos' => $filePos,
  147. 'endFilePos' => $filePos + \strlen($comment),
  148. ]));
  149. // Emulate the PHP behavior
  150. $isDocComment = isset($comment[3]) && $comment[3] === '*';
  151. $this->tokens[] = [$isDocComment ? T_DOC_COMMENT : T_COMMENT, $comment, $line];
  152. } else {
  153. // Invalid characters at the end of the input
  154. $this->handleInvalidCharacterRange(
  155. $filePos, \strlen($this->code), $line, $errorHandler);
  156. }
  157. return;
  158. }
  159. // Check for unterminated comment
  160. $lastToken = $this->tokens[count($this->tokens) - 1];
  161. if ($this->isUnterminatedComment($lastToken)) {
  162. $errorHandler->handleError(new Error('Unterminated comment', [
  163. 'startLine' => $line - substr_count($lastToken[1], "\n"),
  164. 'endLine' => $line,
  165. 'startFilePos' => $filePos - \strlen($lastToken[1]),
  166. 'endFilePos' => $filePos,
  167. ]));
  168. }
  169. }
  170. /**
  171. * Fetches the next token.
  172. *
  173. * The available attributes are determined by the 'usedAttributes' option, which can
  174. * be specified in the constructor. The following attributes are supported:
  175. *
  176. * * 'comments' => Array of PhpParser\Comment or PhpParser\Comment\Doc instances,
  177. * representing all comments that occurred between the previous
  178. * non-discarded token and the current one.
  179. * * 'startLine' => Line in which the node starts.
  180. * * 'endLine' => Line in which the node ends.
  181. * * 'startTokenPos' => Offset into the token array of the first token in the node.
  182. * * 'endTokenPos' => Offset into the token array of the last token in the node.
  183. * * 'startFilePos' => Offset into the code string of the first character that is part of the node.
  184. * * 'endFilePos' => Offset into the code string of the last character that is part of the node.
  185. *
  186. * @param mixed $value Variable to store token content in
  187. * @param mixed $startAttributes Variable to store start attributes in
  188. * @param mixed $endAttributes Variable to store end attributes in
  189. *
  190. * @return int Token id
  191. */
  192. public function getNextToken(&$value = null, &$startAttributes = null, &$endAttributes = null) {
  193. $startAttributes = array();
  194. $endAttributes = array();
  195. while (1) {
  196. if (isset($this->tokens[++$this->pos])) {
  197. $token = $this->tokens[$this->pos];
  198. } else {
  199. // EOF token with ID 0
  200. $token = "\0";
  201. }
  202. if (isset($this->usedAttributes['startLine'])) {
  203. $startAttributes['startLine'] = $this->line;
  204. }
  205. if (isset($this->usedAttributes['startTokenPos'])) {
  206. $startAttributes['startTokenPos'] = $this->pos;
  207. }
  208. if (isset($this->usedAttributes['startFilePos'])) {
  209. $startAttributes['startFilePos'] = $this->filePos;
  210. }
  211. if (\is_string($token)) {
  212. $value = $token;
  213. if (isset($token[1])) {
  214. // bug in token_get_all
  215. $this->filePos += 2;
  216. $id = ord('"');
  217. } else {
  218. $this->filePos += 1;
  219. $id = ord($token);
  220. }
  221. } elseif (!isset($this->dropTokens[$token[0]])) {
  222. $value = $token[1];
  223. $id = $this->tokenMap[$token[0]];
  224. if (T_CLOSE_TAG === $token[0]) {
  225. $this->prevCloseTagHasNewline = false !== strpos($token[1], "\n");
  226. } else if (T_INLINE_HTML === $token[0]) {
  227. $startAttributes['hasLeadingNewline'] = $this->prevCloseTagHasNewline;
  228. }
  229. $this->line += substr_count($value, "\n");
  230. $this->filePos += \strlen($value);
  231. } else {
  232. if (T_COMMENT === $token[0] || T_DOC_COMMENT === $token[0]) {
  233. if (isset($this->usedAttributes['comments'])) {
  234. $comment = T_DOC_COMMENT === $token[0]
  235. ? new Comment\Doc($token[1], $this->line, $this->filePos)
  236. : new Comment($token[1], $this->line, $this->filePos);
  237. $startAttributes['comments'][] = $comment;
  238. }
  239. }
  240. $this->line += substr_count($token[1], "\n");
  241. $this->filePos += \strlen($token[1]);
  242. continue;
  243. }
  244. if (isset($this->usedAttributes['endLine'])) {
  245. $endAttributes['endLine'] = $this->line;
  246. }
  247. if (isset($this->usedAttributes['endTokenPos'])) {
  248. $endAttributes['endTokenPos'] = $this->pos;
  249. }
  250. if (isset($this->usedAttributes['endFilePos'])) {
  251. $endAttributes['endFilePos'] = $this->filePos - 1;
  252. }
  253. return $id;
  254. }
  255. throw new \RuntimeException('Reached end of lexer loop');
  256. }
  257. /**
  258. * Returns the token array for current code.
  259. *
  260. * The token array is in the same format as provided by the
  261. * token_get_all() function and does not discard tokens (i.e.
  262. * whitespace and comments are included). The token position
  263. * attributes are against this token array.
  264. *
  265. * @return array Array of tokens in token_get_all() format
  266. */
  267. public function getTokens() {
  268. return $this->tokens;
  269. }
  270. /**
  271. * Handles __halt_compiler() by returning the text after it.
  272. *
  273. * @return string Remaining text
  274. */
  275. public function handleHaltCompiler() {
  276. // text after T_HALT_COMPILER, still including ();
  277. $textAfter = substr($this->code, $this->filePos);
  278. // ensure that it is followed by ();
  279. // this simplifies the situation, by not allowing any comments
  280. // in between of the tokens.
  281. if (!preg_match('~^\s*\(\s*\)\s*(?:;|\?>\r?\n?)~', $textAfter, $matches)) {
  282. throw new Error('__HALT_COMPILER must be followed by "();"');
  283. }
  284. // prevent the lexer from returning any further tokens
  285. $this->pos = count($this->tokens);
  286. // return with (); removed
  287. return (string) substr($textAfter, strlen($matches[0])); // (string) converts false to ''
  288. }
  289. /**
  290. * Creates the token map.
  291. *
  292. * The token map maps the PHP internal token identifiers
  293. * to the identifiers used by the Parser. Additionally it
  294. * maps T_OPEN_TAG_WITH_ECHO to T_ECHO and T_CLOSE_TAG to ';'.
  295. *
  296. * @return array The token map
  297. */
  298. protected function createTokenMap() {
  299. $tokenMap = array();
  300. // 256 is the minimum possible token number, as everything below
  301. // it is an ASCII value
  302. for ($i = 256; $i < 1000; ++$i) {
  303. if (T_DOUBLE_COLON === $i) {
  304. // T_DOUBLE_COLON is equivalent to T_PAAMAYIM_NEKUDOTAYIM
  305. $tokenMap[$i] = Tokens::T_PAAMAYIM_NEKUDOTAYIM;
  306. } elseif(T_OPEN_TAG_WITH_ECHO === $i) {
  307. // T_OPEN_TAG_WITH_ECHO with dropped T_OPEN_TAG results in T_ECHO
  308. $tokenMap[$i] = Tokens::T_ECHO;
  309. } elseif(T_CLOSE_TAG === $i) {
  310. // T_CLOSE_TAG is equivalent to ';'
  311. $tokenMap[$i] = ord(';');
  312. } elseif ('UNKNOWN' !== $name = token_name($i)) {
  313. if ('T_HASHBANG' === $name) {
  314. // HHVM uses a special token for #! hashbang lines
  315. $tokenMap[$i] = Tokens::T_INLINE_HTML;
  316. } else if (defined($name = 'PhpParser\Parser\Tokens::' . $name)) {
  317. // Other tokens can be mapped directly
  318. $tokenMap[$i] = constant($name);
  319. }
  320. }
  321. }
  322. // HHVM uses a special token for numbers that overflow to double
  323. if (defined('T_ONUMBER')) {
  324. $tokenMap[T_ONUMBER] = Tokens::T_DNUMBER;
  325. }
  326. // HHVM also has a separate token for the __COMPILER_HALT_OFFSET__ constant
  327. if (defined('T_COMPILER_HALT_OFFSET')) {
  328. $tokenMap[T_COMPILER_HALT_OFFSET] = Tokens::T_STRING;
  329. }
  330. return $tokenMap;
  331. }
  332. }