tools\Tokenizer.php

  1: <?php
  2: 
  3: /**
  4:  * This file is part of the Nette Framework (http://nette.org)
  5:  *
  6:  * Copyright (c) 2004, 2011 David Grudl (http://davidgrudl.com)
  7:  *
  8:  * For the full copyright and license information, please view
  9:  * the file license.txt that was distributed with this source code.
 10:  */
 11: 
 12: namespace Nette;
 13: 
 14: use Nette;
 15: 
 16: 
 17: 
 18: /**
 19:  * Simple lexical analyser.
 20:  *
 21:  * @author     David Grudl
 22:  */
 23: class Tokenizer extends Object
 24: {
 25:     /** @var string */
 26:     private $input;
 27: 
 28:     /** @var array */
 29:     public $tokens;
 30: 
 31:     /** @var string */
 32:     private $re;
 33: 
 34:     /** @var array */
 35:     private $names;
 36: 
 37: 
 38: 
 39:     /**
 40:      * @param  array of [symbol name => pattern]
 41:      * @param  string  regular expression flag
 42:      */
 43:     public function __construct(array $patterns, $flags = '')
 44:     {
 45:         $this->re = '~(' . implode(')|(', $patterns) . ')~A' . $flags;
 46:         $keys = array_keys($patterns);
 47:         $this->names = $keys === range(0, count($patterns) - 1) ? FALSE : $keys;
 48:     }
 49: 
 50: 
 51: 
 52:     /**
 53:      * Tokenize string.
 54:      * @param  string
 55:      * @return array
 56:      */
 57:     public function tokenize($input)
 58:     {
 59:         $this->input = $input;
 60:         if ($this->names) {
 61:             $this->tokens = String::matchAll($input, $this->re);
 62:             $len = 0;
 63:             foreach ($this->tokens as & $match) {
 64:                 $name = NULL;
 65:                 for ($i = 1; $i < count($this->names); $i++) {
 66:                     if (!isset($match[$i])) {
 67:                         break;
 68:                     } elseif ($match[$i] != NULL) {
 69:                         $name = $this->names[$i - 1]; break;
 70:                     }
 71:                 }
 72:                 $match = array($match[0], $name);
 73:                 $len += strlen($match[0]);
 74:             }
 75:             if ($len !== strlen($input)) {
 76:                 $errorOffset = $len;
 77:             }
 78: 
 79:         } else {
 80:             $this->tokens = String::split($input, $this->re, PREG_SPLIT_NO_EMPTY);
 81:             if ($this->tokens && !String::match(end($this->tokens), $this->re)) {
 82:                 $tmp = String::split($this->input, $this->re, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);
 83:                 list(, $errorOffset) = end($tmp);
 84:             }
 85:         }
 86: 
 87:         if (isset($errorOffset)) {
 88:             $line = $errorOffset ? substr_count($this->input, "\n", 0, $errorOffset) + 1 : 1;
 89:             $col = $errorOffset - strrpos(substr($this->input, 0, $errorOffset), "\n") + 1;
 90:             $token = str_replace("\n", '\n', substr($input, $errorOffset, 10));
 91:             throw new TokenizerException("Unexpected '$token' on line $line, column $col.");
 92:         }
 93:         return $this->tokens;
 94:     }
 95: 
 96: 
 97: 
 98:     /**
 99:      * Returns position of token in input string
100:      * @param  int token number
101:      * @return array [offset, line, column]
102:      */
103:     public function getOffset($i)
104:     {
105:         $tokens = String::split($this->input, $this->re, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);
106:         $offset = isset($tokens[$i]) ? $tokens[$i][1] : strlen($this->input);
107:         return array(
108:             $offset,
109:             ($offset ? substr_count($this->input, "\n", 0, $offset) + 1 : 1),
110:             $offset - strrpos(substr($this->input, 0, $offset), "\n"),
111:         );
112:     }
113: 
114: }
115:
Namespaces

Classes

Interfaces
