| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455 | 
							- var SyntaxError = require('./SyntaxError');
 
- var TAB = 9;
 
- var N = 10;
 
- var F = 12;
 
- var R = 13;
 
- var SPACE = 32;
 
- var Tokenizer = function(str) {
 
-     this.str = str;
 
-     this.pos = 0;
 
- };
 
- Tokenizer.prototype = {
 
-     charCodeAt: function(pos) {
 
-         return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
 
-     },
 
-     charCode: function() {
 
-         return this.charCodeAt(this.pos);
 
-     },
 
-     nextCharCode: function() {
 
-         return this.charCodeAt(this.pos + 1);
 
-     },
 
-     nextNonWsCode: function(pos) {
 
-         return this.charCodeAt(this.findWsEnd(pos));
 
-     },
 
-     findWsEnd: function(pos) {
 
-         for (; pos < this.str.length; pos++) {
 
-             var code = this.str.charCodeAt(pos);
 
-             if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
 
-                 break;
 
-             }
 
-         }
 
-         return pos;
 
-     },
 
-     substringToPos: function(end) {
 
-         return this.str.substring(this.pos, this.pos = end);
 
-     },
 
-     eat: function(code) {
 
-         if (this.charCode() !== code) {
 
-             this.error('Expect `' + String.fromCharCode(code) + '`');
 
-         }
 
-         this.pos++;
 
-     },
 
-     peek: function() {
 
-         return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
 
-     },
 
-     error: function(message) {
 
-         throw new SyntaxError(message, this.str, this.pos);
 
-     }
 
- };
 
- module.exports = Tokenizer;
 
 
  |