InputStream.php 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. <?php
  2. /*
  3. Copyright 2009 Geoffrey Sneddon <http://gsnedders.com/>
  4. Permission is hereby granted, free of charge, to any person obtaining a
  5. copy of this software and associated documentation files (the
  6. "Software"), to deal in the Software without restriction, including
  7. without limitation the rights to use, copy, modify, merge, publish,
  8. distribute, sublicense, and/or sell copies of the Software, and to
  9. permit persons to whom the Software is furnished to do so, subject to
  10. the following conditions:
  11. The above copyright notice and this permission notice shall be included
  12. in all copies or substantial portions of the Software.
  13. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  14. OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  15. MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  16. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  17. CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  18. TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  19. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  20. */
  21. // Some conventions:
  22. // /* */ indicates verbatim text from the HTML 5 specification
  23. // // indicates regular comments
  24. class HTML5_InputStream {
  25. /**
  26. * The string data we're parsing.
  27. */
  28. private $data;
  29. /**
  30. * The current integer byte position we are in $data
  31. */
  32. private $char;
  33. /**
  34. * Length of $data; when $char === $data, we are at the end-of-file.
  35. */
  36. private $EOF;
  37. /**
  38. * Parse errors.
  39. */
  40. public $errors = [];
  41. /**
  42. * @param $data | Data to parse
  43. * @throws Exception
  44. */
  45. public function __construct($data) {
  46. /* Given an encoding, the bytes in the input stream must be
  47. converted to Unicode characters for the tokeniser, as
  48. described by the rules for that encoding, except that the
  49. leading U+FEFF BYTE ORDER MARK character, if any, must not
  50. be stripped by the encoding layer (it is stripped by the rule below).
  51. Bytes or sequences of bytes in the original byte stream that
  52. could not be converted to Unicode characters must be converted
  53. to U+FFFD REPLACEMENT CHARACTER code points. */
  54. // XXX currently assuming input data is UTF-8; once we
  55. // build encoding detection this will no longer be the case
  56. //
  57. // We previously had an mbstring implementation here, but that
  58. // implementation is heavily non-conforming, so it's been
  59. // omitted.
  60. if (extension_loaded('iconv')) {
  61. // non-conforming
  62. $data = @iconv('UTF-8', 'UTF-8//IGNORE', $data);
  63. } else {
  64. // we can make a conforming native implementation
  65. throw new Exception('Not implemented, please install iconv');
  66. }
  67. /* One leading U+FEFF BYTE ORDER MARK character must be
  68. ignored if any are present. */
  69. if (substr($data, 0, 3) === "\xEF\xBB\xBF") {
  70. $data = substr($data, 3);
  71. }
  72. /* All U+0000 NULL characters in the input must be replaced
  73. by U+FFFD REPLACEMENT CHARACTERs. Any occurrences of such
  74. characters is a parse error. */
  75. for ($i = 0, $count = substr_count($data, "\0"); $i < $count; $i++) {
  76. $this->errors[] = [
  77. 'type' => HTML5_Tokenizer::PARSEERROR,
  78. 'data' => 'null-character'
  79. ];
  80. }
  81. /* U+000D CARRIAGE RETURN (CR) characters and U+000A LINE FEED
  82. (LF) characters are treated specially. Any CR characters
  83. that are followed by LF characters must be removed, and any
  84. CR characters not followed by LF characters must be converted
  85. to LF characters. Thus, newlines in HTML DOMs are represented
  86. by LF characters, and there are never any CR characters in the
  87. input to the tokenization stage. */
  88. $data = str_replace(
  89. [
  90. "\0",
  91. "\r\n",
  92. "\r"
  93. ],
  94. [
  95. "\xEF\xBF\xBD",
  96. "\n",
  97. "\n"
  98. ],
  99. $data
  100. );
  101. /* Any occurrences of any characters in the ranges U+0001 to
  102. U+0008, U+000B, U+000E to U+001F, U+007F to U+009F,
  103. U+D800 to U+DFFF , U+FDD0 to U+FDEF, and
  104. characters U+FFFE, U+FFFF, U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF,
  105. U+3FFFE, U+3FFFF, U+4FFFE, U+4FFFF, U+5FFFE, U+5FFFF, U+6FFFE,
  106. U+6FFFF, U+7FFFE, U+7FFFF, U+8FFFE, U+8FFFF, U+9FFFE, U+9FFFF,
  107. U+AFFFE, U+AFFFF, U+BFFFE, U+BFFFF, U+CFFFE, U+CFFFF, U+DFFFE,
  108. U+DFFFF, U+EFFFE, U+EFFFF, U+FFFFE, U+FFFFF, U+10FFFE, and
  109. U+10FFFF are parse errors. (These are all control characters
  110. or permanently undefined Unicode characters.) */
  111. // Check PCRE is loaded.
  112. if (extension_loaded('pcre')) {
  113. $count = preg_match_all(
  114. '/(?:
  115. [\x01-\x08\x0B\x0E-\x1F\x7F] # U+0001 to U+0008, U+000B, U+000E to U+001F and U+007F
  116. |
  117. \xC2[\x80-\x9F] # U+0080 to U+009F
  118. |
  119. \xED(?:\xA0[\x80-\xFF]|[\xA1-\xBE][\x00-\xFF]|\xBF[\x00-\xBF]) # U+D800 to U+DFFFF
  120. |
  121. \xEF\xB7[\x90-\xAF] # U+FDD0 to U+FDEF
  122. |
  123. \xEF\xBF[\xBE\xBF] # U+FFFE and U+FFFF
  124. |
  125. [\xF0-\xF4][\x8F-\xBF]\xBF[\xBE\xBF] # U+nFFFE and U+nFFFF (1 <= n <= 10_{16})
  126. )/x',
  127. $data,
  128. $matches
  129. );
  130. for ($i = 0; $i < $count; $i++) {
  131. $this->errors[] = [
  132. 'type' => HTML5_Tokenizer::PARSEERROR,
  133. 'data' => 'invalid-codepoint'
  134. ];
  135. }
  136. } else {
  137. // XXX: Need non-PCRE impl, probably using substr_count
  138. }
  139. $this->data = $data;
  140. $this->char = 0;
  141. $this->EOF = strlen($data);
  142. }
  143. /**
  144. * Returns the current line that the tokenizer is at.
  145. *
  146. * @return int
  147. */
  148. public function getCurrentLine() {
  149. // Check the string isn't empty
  150. if ($this->EOF) {
  151. // Add one to $this->char because we want the number for the next
  152. // byte to be processed.
  153. return substr_count($this->data, "\n", 0, min($this->char, $this->EOF)) + 1;
  154. } else {
  155. // If the string is empty, we are on the first line (sorta).
  156. return 1;
  157. }
  158. }
  159. /**
  160. * Returns the current column of the current line that the tokenizer is at.
  161. *
  162. * @return int
  163. */
  164. public function getColumnOffset() {
  165. // strrpos is weird, and the offset needs to be negative for what we
  166. // want (i.e., the last \n before $this->char). This needs to not have
  167. // one (to make it point to the next character, the one we want the
  168. // position of) added to it because strrpos's behaviour includes the
  169. // final offset byte.
  170. $lastLine = strrpos($this->data, "\n", $this->char - 1 - strlen($this->data));
  171. // However, for here we want the length up until the next byte to be
  172. // processed, so add one to the current byte ($this->char).
  173. if ($lastLine !== false) {
  174. $findLengthOf = substr($this->data, $lastLine + 1, $this->char - 1 - $lastLine);
  175. } else {
  176. $findLengthOf = substr($this->data, 0, $this->char);
  177. }
  178. // Get the length for the string we need.
  179. if (extension_loaded('iconv')) {
  180. return iconv_strlen($findLengthOf, 'utf-8');
  181. } elseif (extension_loaded('mbstring')) {
  182. return mb_strlen($findLengthOf, 'utf-8');
  183. } elseif (extension_loaded('xml')) {
  184. return strlen(utf8_decode($findLengthOf));
  185. } else {
  186. $count = count_chars($findLengthOf);
  187. // 0x80 = 0x7F - 0 + 1 (one added to get inclusive range)
  188. // 0x33 = 0xF4 - 0x2C + 1 (one added to get inclusive range)
  189. return array_sum(array_slice($count, 0, 0x80)) +
  190. array_sum(array_slice($count, 0xC2, 0x33));
  191. }
  192. }
  193. /**
  194. * Retrieve the currently consume character.
  195. * @note This performs bounds checking
  196. *
  197. * @return bool|string
  198. */
  199. public function char() {
  200. return ($this->char++ < $this->EOF)
  201. ? $this->data[$this->char - 1]
  202. : false;
  203. }
  204. /**
  205. * Get all characters until EOF.
  206. * @note This performs bounds checking
  207. *
  208. * @return string|bool
  209. */
  210. public function remainingChars() {
  211. if ($this->char < $this->EOF) {
  212. $data = substr($this->data, $this->char);
  213. $this->char = $this->EOF;
  214. return $data;
  215. } else {
  216. return false;
  217. }
  218. }
  219. /**
  220. * Matches as far as possible until we reach a certain set of bytes
  221. * and returns the matched substring.
  222. *
  223. * @param $bytes | Bytes to match.
  224. * @param null $max
  225. * @return bool|string
  226. */
  227. public function charsUntil($bytes, $max = null) {
  228. if ($this->char < $this->EOF) {
  229. if ($max === 0 || $max) {
  230. $len = strcspn($this->data, $bytes, $this->char, $max);
  231. } else {
  232. $len = strcspn($this->data, $bytes, $this->char);
  233. }
  234. $string = (string) substr($this->data, $this->char, $len);
  235. $this->char += $len;
  236. return $string;
  237. } else {
  238. return false;
  239. }
  240. }
  241. /**
  242. * Matches as far as possible with a certain set of bytes
  243. * and returns the matched substring.
  244. *
  245. * @param $bytes | Bytes to match.
  246. * @param null $max
  247. * @return bool|string
  248. */
  249. public function charsWhile($bytes, $max = null) {
  250. if ($this->char < $this->EOF) {
  251. if ($max === 0 || $max) {
  252. $len = strspn($this->data, $bytes, $this->char, $max);
  253. } else {
  254. $len = strspn($this->data, $bytes, $this->char);
  255. }
  256. $string = (string) substr($this->data, $this->char, $len);
  257. $this->char += $len;
  258. return $string;
  259. } else {
  260. return false;
  261. }
  262. }
  263. /**
  264. * Unconsume one character.
  265. */
  266. public function unget() {
  267. if ($this->char <= $this->EOF) {
  268. $this->char--;
  269. }
  270. }
  271. }