LexerTest.php 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. <?php
  2. namespace PhpParser;
  3. use PhpParser\Parser\Tokens;
  4. class LexerTest extends \PHPUnit_Framework_TestCase
  5. {
  6. /* To allow overwriting in parent class */
  7. protected function getLexer(array $options = array()) {
  8. return new Lexer($options);
  9. }
  10. /**
  11. * @dataProvider provideTestError
  12. */
  13. public function testError($code, $messages) {
  14. if (defined('HHVM_VERSION')) {
  15. $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
  16. }
  17. $errorHandler = new ErrorHandler\Collecting();
  18. $lexer = $this->getLexer(['usedAttributes' => [
  19. 'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
  20. ]]);
  21. $lexer->startLexing($code, $errorHandler);
  22. $errors = $errorHandler->getErrors();
  23. $this->assertSame(count($messages), count($errors));
  24. for ($i = 0; $i < count($messages); $i++) {
  25. $this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
  26. }
  27. }
  28. public function provideTestError() {
  29. return array(
  30. array("<?php /*", array("Unterminated comment from 1:7 to 1:9")),
  31. array("<?php \1", array("Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7")),
  32. array("<?php \0", array("Unexpected null byte from 1:7 to 1:7")),
  33. // Error with potentially emulated token
  34. array("<?php ?? \0", array("Unexpected null byte from 1:10 to 1:10")),
  35. array("<?php\n\0\1 foo /* bar", array(
  36. "Unexpected null byte from 2:1 to 2:1",
  37. "Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
  38. "Unterminated comment from 2:8 to 2:14"
  39. )),
  40. );
  41. }
  42. /**
  43. * @dataProvider provideTestLex
  44. */
  45. public function testLex($code, $options, $tokens) {
  46. $lexer = $this->getLexer($options);
  47. $lexer->startLexing($code);
  48. while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
  49. $token = array_shift($tokens);
  50. $this->assertSame($token[0], $id);
  51. $this->assertSame($token[1], $value);
  52. $this->assertEquals($token[2], $startAttributes);
  53. $this->assertEquals($token[3], $endAttributes);
  54. }
  55. }
  56. public function provideTestLex() {
  57. return array(
  58. // tests conversion of closing PHP tag and drop of whitespace and opening tags
  59. array(
  60. '<?php tokens ?>plaintext',
  61. array(),
  62. array(
  63. array(
  64. Tokens::T_STRING, 'tokens',
  65. array('startLine' => 1), array('endLine' => 1)
  66. ),
  67. array(
  68. ord(';'), '?>',
  69. array('startLine' => 1), array('endLine' => 1)
  70. ),
  71. array(
  72. Tokens::T_INLINE_HTML, 'plaintext',
  73. array('startLine' => 1, 'hasLeadingNewline' => false),
  74. array('endLine' => 1)
  75. ),
  76. )
  77. ),
  78. // tests line numbers
  79. array(
  80. '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
  81. array(),
  82. array(
  83. array(
  84. ord('$'), '$',
  85. array('startLine' => 2), array('endLine' => 2)
  86. ),
  87. array(
  88. Tokens::T_STRING, 'token',
  89. array('startLine' => 2), array('endLine' => 2)
  90. ),
  91. array(
  92. ord('$'), '$',
  93. array(
  94. 'startLine' => 3,
  95. 'comments' => array(
  96. new Comment\Doc('/** doc' . "\n" . 'comment */', 2, 14),
  97. )
  98. ),
  99. array('endLine' => 3)
  100. ),
  101. )
  102. ),
  103. // tests comment extraction
  104. array(
  105. '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
  106. array(),
  107. array(
  108. array(
  109. Tokens::T_STRING, 'token',
  110. array(
  111. 'startLine' => 2,
  112. 'comments' => array(
  113. new Comment('/* comment */', 1, 6),
  114. new Comment('// comment' . "\n", 1, 20),
  115. new Comment\Doc('/** docComment 1 */', 2, 31),
  116. new Comment\Doc('/** docComment 2 */', 2, 50),
  117. ),
  118. ),
  119. array('endLine' => 2)
  120. ),
  121. )
  122. ),
  123. // tests differing start and end line
  124. array(
  125. '<?php "foo' . "\n" . 'bar"',
  126. array(),
  127. array(
  128. array(
  129. Tokens::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
  130. array('startLine' => 1), array('endLine' => 2)
  131. ),
  132. )
  133. ),
  134. // tests exact file offsets
  135. array(
  136. '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
  137. array('usedAttributes' => array('startFilePos', 'endFilePos')),
  138. array(
  139. array(
  140. Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
  141. array('startFilePos' => 6), array('endFilePos' => 8)
  142. ),
  143. array(
  144. ord(';'), ';',
  145. array('startFilePos' => 9), array('endFilePos' => 9)
  146. ),
  147. array(
  148. Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
  149. array('startFilePos' => 18), array('endFilePos' => 20)
  150. ),
  151. array(
  152. ord(';'), ';',
  153. array('startFilePos' => 21), array('endFilePos' => 21)
  154. ),
  155. )
  156. ),
  157. // tests token offsets
  158. array(
  159. '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
  160. array('usedAttributes' => array('startTokenPos', 'endTokenPos')),
  161. array(
  162. array(
  163. Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
  164. array('startTokenPos' => 1), array('endTokenPos' => 1)
  165. ),
  166. array(
  167. ord(';'), ';',
  168. array('startTokenPos' => 2), array('endTokenPos' => 2)
  169. ),
  170. array(
  171. Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
  172. array('startTokenPos' => 5), array('endTokenPos' => 5)
  173. ),
  174. array(
  175. ord(';'), ';',
  176. array('startTokenPos' => 6), array('endTokenPos' => 6)
  177. ),
  178. )
  179. ),
  180. // tests all attributes being disabled
  181. array(
  182. '<?php /* foo */ $bar;',
  183. array('usedAttributes' => array()),
  184. array(
  185. array(
  186. Tokens::T_VARIABLE, '$bar',
  187. array(), array()
  188. ),
  189. array(
  190. ord(';'), ';',
  191. array(), array()
  192. )
  193. )
  194. )
  195. );
  196. }
  197. /**
  198. * @dataProvider provideTestHaltCompiler
  199. */
  200. public function testHandleHaltCompiler($code, $remaining) {
  201. $lexer = $this->getLexer();
  202. $lexer->startLexing($code);
  203. while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
  204. $this->assertSame($remaining, $lexer->handleHaltCompiler());
  205. $this->assertSame(0, $lexer->getNextToken());
  206. }
  207. public function provideTestHaltCompiler() {
  208. return array(
  209. array('<?php ... __halt_compiler();Remaining Text', 'Remaining Text'),
  210. array('<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'),
  211. array('<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'),
  212. //array('<?php ... __halt_compiler();' . "\0", "\0"),
  213. //array('<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'),
  214. );
  215. }
  216. /**
  217. * @expectedException \PhpParser\Error
  218. * @expectedExceptionMessage __HALT_COMPILER must be followed by "();"
  219. */
  220. public function testHandleHaltCompilerError() {
  221. $lexer = $this->getLexer();
  222. $lexer->startLexing('<?php ... __halt_compiler invalid ();');
  223. while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
  224. $lexer->handleHaltCompiler();
  225. }
  226. public function testGetTokens() {
  227. $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
  228. $expectedTokens = array(
  229. array(T_OPEN_TAG, '<?php ', 1),
  230. array(T_CONSTANT_ENCAPSED_STRING, '"a"', 1),
  231. ';',
  232. array(T_WHITESPACE, "\n", 1),
  233. array(T_COMMENT, '// foo' . "\n", 2),
  234. array(T_CONSTANT_ENCAPSED_STRING, '"b"', 3),
  235. ';',
  236. );
  237. $lexer = $this->getLexer();
  238. $lexer->startLexing($code);
  239. $this->assertSame($expectedTokens, $lexer->getTokens());
  240. }
  241. }