/*
 * Copyright (c) 1998-2006 Caucho Technology -- all rights reserved
 *
 * This file is part of Resin(R) Open Source
 *
 * Each copy or derived work must preserve the copyright notice and this
 * notice unmodified.
 *
 * Resin Open Source is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * Resin Open Source is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, or any warranty
 * of NON-INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Resin Open Source; if not, write to the
 *
 *   Free Software Foundation, Inc.
 *   59 Temple Place, Suite 330
 *   Boston, MA 02111-1307  USA
 *
 * @author Scott Ferguson
 */

package com.caucho.quercus.lib;

import java.util.logging.Logger;

import com.caucho.quercus.Quercus;
import com.caucho.quercus.QuercusException;
import com.caucho.quercus.QuercusModuleException;

import com.caucho.quercus.module.AbstractQuercusModule;
import com.caucho.quercus.module.Optional;
import com.caucho.quercus.module.Reference;

import com.caucho.quercus.env.ArrayValue;
import com.caucho.quercus.env.ArrayValueImpl;
import com.caucho.quercus.env.LongValue;
import com.caucho.quercus.env.StringValue;
import com.caucho.quercus.env.StringBuilderValue;

import com.caucho.util.L10N;
import com.caucho.util.IntMap;

/**
 * Quercus tokenizer 
 */
public class TokenModule extends AbstractQuercusModule {
  private static final L10N L = new L10N(TokenModule.class);
  private static final Logger log
    = Logger.getLogger(TokenModule.class.getName());

  public static final int T_ABSTRACT = 256;
  public static final int T_AND_EQUAL = 257;
  public static final int T_ARRAY = 258;
  public static final int T_ARRAY_CAST = 259;
  public static final int T_AS = 260;
  public static final int T_BAD_CHARACTER = 261;
  public static final int T_BOOLEAN_AND = 262;
  public static final int T_BOOLEAN_OR = 263;
  public static final int T_BOOL_CAST = 264;
  public static final int T_BREAK = 265;
  public static final int T_CASE = 266;
  public static final int T_CATCH = 267;
  public static final int T_CHARACTER = 268;
  public static final int T_CLASS = 269;
  public static final int T_CLONE = 270;
  public static final int T_CLOSE_TAG = 271;
  public static final int T_COMMENT = 272;
  public static final int T_CONCAT_EQUAL = 273;
  public static final int T_CONST = 274;
  public static final int T_CONSTANT_ENCAPSED_STRING = 275;
  public static final int T_CONTINUE = 276;
  public static final int T_CURLY_OPEN = 277;
  public static final int T_DEC = 278;
  public static final int T_DECLARE = 279;
  public static final int T_DEFAULT = 280;
  public static final int T_DIV_EQUAL = 281;
  public static final int T_DNUMBER = 282;
  public static final int T_DOC_COMMENT = 283;
  public static final int T_DO = 284;
  public static final int T_DOLLAR_OPEN_CURLY_BRACES = 285;
  public static final int T_DOUBLE_ARROW = 286;
  public static final int T_DOUBLE_CAST = 287;
  public static final int T_DOUBLE_COLON = 288;
  public static final int T_ECHO = 289;
  public static final int T_ELSE = 290;
  public static final int T_ELSEIF = 291;
  public static final int T_EMPTY = 292;
  public static final int T_ENCAPSED_AND_WHITESPACE = 293;
  public static final int T_ENDDECLARE = 294;
  public static final int T_ENDFOR = 295;
  public static final int T_ENDFOREACH = 296;
  public static final int T_ENDIF = 297;
  public static final int T_ENDSWITCH = 298;
  public static final int T_ENDWHILE = 299;
  public static final int T_END_HEREDOC = 300;
  public static final int T_EVAL = 301;
  public static final int T_EXIT = 302;
  public static final int T_EXTENDS = 303;
  public static final int T_FILE = 304;
  public static final int T_FINAL = 305;
  public static final int T_FOR = 306;
  public static final int T_FOREACH = 307;
  public static final int T_FUNCTION = 308;
  public static final int T_GLOBAL = 309;
  public static final int T_HALT_COMPILER = 310;
  public static final int T_IF = 311;
  public static final int T_IMPLEMENTS = 312;
  public static final int T_INC = 313;
  public static final int T_INCLUDE = 314;
  public static final int T_INCLUDE_ONCE = 315;
  public static final int T_INLINE_HTML = 316;
  public static final int T_INSTANCEOF = 317;
  public static final int T_INT_CAST = 318;
  public static final int T_INTERFACE = 319;
  public static final int T_ISSET = 320;
  public static final int T_IS_EQUAL = 321;
  public static final int T_IS_GREATER_OR_EQUAL = 322;
  public static final int T_IS_IDENTICAL = 323;
  public static final int T_IS_NOT_EQUAL = 324;
  public static final int T_IS_NOT_IDENTICAL = 325;
  public static final int T_IS_SMALLER_OR_EQUAL = 326;
  public static final int T_LINE = 327;
  public static final int T_LIST = 328;
  public static final int T_LNUMBER = 329;
  public static final int T_LOGICAL_AND = 330;
  public static final int T_LOGICAL_OR = 331;
  public static final int T_LOGICAL_XOR = 332;
  public static final int T_MINUS_EQUAL = 333;
  public static final int T_ML_COMMENT = 334;
  public static final int T_MOD_EQUAL = 335;
  public static final int T_MUL_EQUAL = 336;
  public static final int T_NEW = 337;
  public static final int T_NUM_STRING = 338;
  public static final int T_OBJECT_CAST = 339;
  public static final int T_OBJECT_OPERATOR = 340;
  public static final int T_OLD_FUNCTION = 341;
  public static final int T_OPEN_TAG = 342;
  public static final int T_OPEN_TAG_WITH_ECHO = 343;
  public static final int T_OR_EQUAL = 344;
  public static final int T_PAAMAYIM_NEKUDOTAYIM = T_DOUBLE_COLON;
  public static final int T_PLUS_EQUAL = 345;
  public static final int T_PRINT = 346;
  public static final int T_PRIVATE = 347;
  public static final int T_PUBLIC = 348;
  public static final int T_PROTECTED = 349;
  public static final int T_REQUIRE = 350;
  public static final int T_REQUIRE_ONCE = 351;
  public static final int T_RETURN = 352;
  public static final int T_SL = 353;
  public static final int T_SL_EQUAL = 354;
  public static final int T_SR = 355;
  public static final int T_SR_EQUAL = 356;
  public static final int T_START_HEREDOC = 357;
  public static final int T_STATIC = 358;
  public static final int T_STRING = 359;
  public static final int T_STRING_CAST = 360;
  public static final int T_STRING_VARNAME = 361;
  public static final int T_SWITCH = 362;
  public static final int T_THROW = 363;
  public static final int T_TRY = 364;
  public static final int T_UNSET = 365;
  public static final int T_UNSET_CAST = 366;
  public static final int T_USE = 367;
  public static final int T_VAR = 368;
  public static final int T_VARIABLE = 369;
  public static final int T_WHILE = 370;
  public static final int T_WHITESPACE = 371;
  public static final int T_XOR_EQUAL = 372;
  public static final int T_FUNC_C = 373;
  public static final int T_CLASS_C = 374;

  private static final IntMap _reservedMap = new IntMap();

  public String []getLoadedExtensions()
  {
    return new String[] { "tokenizer" };
  }

  /**
   * Parses the string.
   */
  public static ArrayValue token_get_all(StringValue s)
  {
    ArrayValue result = new ArrayValueImpl();

    Token lexer = new Token(s);
    int token;

    while ((token = lexer.nextToken()) >= 0) {
      if (0x20 <= token && token <= 0x7f) {
	result.put(StringValue.create((char) token));
      }
      else {
	result.put(new ArrayValueImpl()
		   .append(LongValue.create(token))
		   .append(lexer.getLexeme()));
      }
    }

    return result;
  }
  
  /**
   * Returns the token name for th egiven token.
   */
  public static String token_name(int token)
  {
    switch (token) {
    case T_ABSTRACT: return "T_ABSTRACT";
    case T_AND_EQUAL: return "T_AND_EQUAL";
    case T_ARRAY: return "T_ARRAY";
    case T_ARRAY_CAST: return "T_ARRAY_CAST";
    case T_AS: return "T_AS";
    case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
    case T_BOOLEAN_AND: return "T_BOOLEAN_AND";
    case T_BOOLEAN_OR: return "T_BOOLEAN_OR";
    case T_BOOL_CAST: return "T_BOOL_CAST";
    case T_BREAK: return "T_BREAK";
    case T_CASE: return "T_CASE";
    case T_CATCH: return "T_CATCH";
    case T_CHARACTER: return "T_CHARACTER";
    case T_CLASS: return "T_CLASS";
    case T_CLONE: return "T_CLONE";
    case T_CLOSE_TAG: return "T_CLOSE_TAG";
    case T_COMMENT: return "T_COMMENT";
    case T_CONCAT_EQUAL: return "T_CONCAT_EQUAL";
    case T_CONST: return "T_CONST";
    case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING";
    case T_CONTINUE: return "T_CONTINUE";
    case T_CURLY_OPEN: return "T_CURLY_OPEN";
    case T_DEC: return "T_DEC";
    case T_DECLARE: return "T_DECLARE";
    case T_DEFAULT: return "T_DEFAULT";
    case T_DIV_EQUAL: return "T_DIV_EQUAL";
    case T_DNUMBER: return "T_DNUMBER";
    case T_DOC_COMMENT: return "T_DOC_COMMENT";
    case T_DO: return "T_DO";
    case T_DOLLAR_OPEN_CURLY_BRACES: return "T_DOLLAR_OPEN_CURLY_BRACES";
    case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
    case T_DOUBLE_CAST: return "T_DOUBLE_CAST";
    case T_DOUBLE_COLON: return "T_DOUBLE_COLON";
    case T_ECHO: return "T_ECHO";
    case T_ELSE: return "T_ELSE";
    case T_ELSEIF: return "T_ELSEIF";
    case T_EMPTY: return "T_EMPTY";
    case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";
    case T_ENDDECLARE: return "T_ENDDECLARE";
    case T_ENDFOR: return "T_ENDFOR";
    case T_ENDFOREACH: return "T_ENDFOREACH";
    case T_ENDIF: return "T_ENDIF";
    case T_ENDSWITCH: return "T_ENDSWITCH";
    case T_ENDWHILE: return "T_ENDWHILE";
    case T_END_HEREDOC: return "T_END_HEREDOC";
    case T_EVAL: return "T_EVAL";
    case T_EXIT: return "T_EXIT";
    case T_EXTENDS: return "T_EXTENDS";
    case T_FILE: return "T_FILE";
    case T_FINAL: return "T_FINAL";
    case T_FOR: return "T_FOR";
    case T_FOREACH: return "T_FOREACH";
    case T_FUNCTION: return "T_FUNCTION";
    case T_GLOBAL: return "T_GLOBAL";
    case T_HALT_COMPILER: return "T_HALT_COMPILER";
    case T_IF: return "T_IF";
    case T_IMPLEMENTS: return "T_IMPLEMENTS";
    case T_INC: return "T_INC";
    case T_INCLUDE: return "T_INCLUDE";
    case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";
    case T_INLINE_HTML: return "T_INLINE_HTML";
    case T_INSTANCEOF: return "T_INSTANCEOF";
    case T_INT_CAST: return "T_INT_CAST";
    case T_INTERFACE: return "T_INTERFACE";
    case T_ISSET: return "T_ISSET";
    case T_IS_EQUAL: return "T_IS_EQUAL";
    case T_IS_GREATER_OR_EQUAL: return "T_IS_GREATER_OR_EQUAL";
    case T_IS_IDENTICAL: return "T_IS_IDENTICAL";
    case T_IS_NOT_EQUAL: return "T_IS_NOT_EQUAL";
    case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL";
    case T_IS_SMALLER_OR_EQUAL: return "T_IS_SMALLER_OR_EQUAL";
    case T_LINE: return "T_LINE";
    case T_LIST: return "T_LIST";
    case T_LNUMBER: return "T_LNUMBER";
    case T_LOGICAL_AND: return "T_LOGICAL_AND";
    case T_LOGICAL_OR: return "T_LOGICAL_OR";
    case T_LOGICAL_XOR: return "T_LOGICAL_XOR";
    case T_MINUS_EQUAL: return "T_MINUS_EQUAL";
    case T_ML_COMMENT: return "T_ML_COMMENT";
    case T_MOD_EQUAL: return "T_MOD_EQUAL";
    case T_MUL_EQUAL: return "T_MUL_EQUAL";
    case T_NEW: return "T_NEW";
    case T_NUM_STRING: return "T_NUM_STRING";
    case T_OBJECT_CAST: return "T_OBJECT_CAST";
    case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR";
    case T_OLD_FUNCTION: return "T_OLD_FUNCTION";
    case T_OPEN_TAG: return "T_OPEN_TAG";
    case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO";
    case T_OR_EQUAL: return "T_OR_EQUAL";
    case T_PLUS_EQUAL: return "T_PLUS_EQUAL";
    case T_PRINT: return "T_PRINT";
    case T_PRIVATE: return "T_PRIVATE";
    case T_PUBLIC: return "T_PUBLIC";
    case T_PROTECTED: return "T_PROTECTED";
    case T_REQUIRE: return "T_REQUIRE";
    case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE";
    case T_RETURN: return "T_RETURN";
    case T_SL: return "T_SL";
    case T_SL_EQUAL: return "T_SL_EQUAL";
    case T_SR: return "T_SR";
    case T_SR_EQUAL: return "T_SR_EQUAL";
    case T_START_HEREDOC: return "T_START_HEREDOC";
    case T_STATIC: return "T_STATIC";
    case T_STRING: return "T_STRING";
    case T_STRING_CAST: return "T_STRING_CAST";
    case T_STRING_VARNAME: return "T_STRING_VARNAME";
    case T_SWITCH: return "T_SWITCH";
    case T_THROW: return "T_THROW";
    case T_TRY: return "T_TRY";
    case T_UNSET: return "T_UNSET";
    case T_UNSET_CAST: return "T_UNSET_CAST";
    case T_USE: return "T_USE";
    case T_VAR: return "T_VAR";
    case T_VARIABLE: return "T_VARIABLE";
    case T_WHILE: return "T_WHILE";
    case T_WHITESPACE: return "T_WHITESPACE";
    case T_XOR_EQUAL: return "T_XOR_EQUAL";
    case T_FUNC_C: return "T_FUNC_C";
    case T_CLASS_C: return "T_CLASS_C";
      
    default:
      return "UNDEFINED";
    }
  }

  static class Token {
    private final StringValue _s;
    private final int _length;
    private int _i;
    private boolean _inPhp;

    private StringBuilderValue _lexeme;

    Token(StringValue s)
    {
      _s = s;
      _length = s.length();
    }

    int nextToken()
    {
      _lexeme = new StringBuilderValue();

      if (! _inPhp) {
	_inPhp = true;

	if (parseHtml()) {
	  return T_INLINE_HTML;
	}
      }
      
      int ch = read();

      switch (ch) {
      case -1:
	return -1;

      case ' ': case '\t': case '\r': case '\n':
	_lexeme.append((char) ch);
	while (Character.isWhitespace((ch = read()))) {
	  _lexeme.append((char) ch);
	}
	unread();
	return T_WHITESPACE;

      case '{': case '}': case ';': case '[': case ']': case ',':
      case '@': case '(': case ')':
	return ch;

      case '&':
	if ((ch = read()) == '&') {
	  _lexeme.append("&&");
	  return T_BOOLEAN_AND;
	}
	else if (ch == '=') {
	  _lexeme.append("&=");
	  return T_AND_EQUAL;
	}
	else {
	  unread();
	  return '&';
	}

      case '|':
	if ((ch = read()) == '|') {
	  _lexeme.append("||");
	  return T_BOOLEAN_OR;
	}
	else if (ch == '=') {
	  _lexeme.append("|=");
	  return T_OR_EQUAL;
	}
	else {
	  unread();
	  return '|';
	}

      case '?':
	if ((ch = read()) == '>') {
	  _lexeme.append("?>");
	  _inPhp = false;
	  return T_CLOSE_TAG;
	}
	else {
	  unread();
	  return '?';
	}

      case '/':
	if ((ch = read()) == '/') {
	  _lexeme.append("//");
	  
	  while ((ch = read()) >= 0 && ch != '\r' && ch != '\n') {
	    _lexeme.append((char) ch);
	  }
	  unread();
	  
	  return T_COMMENT;
	}
	else if (ch == '=') {
	  _lexeme.append("/=");
	  return T_DIV_EQUAL;
	}
	else if (ch == '*') {
	  int token = T_COMMENT;
	  _lexeme.append("/*");

	  if ((ch = read()) == '*') {
	    token = T_DOC_COMMENT;
	    _lexeme.append("*");
	  }
	  else
	    unread();

	  while ((ch = read()) >= 0) {
	    _lexeme.append((char) ch);

	    if (ch != '*') {
	    }
	    else if ((ch = read()) == '/') {
	      _lexeme.append((char) ch);
	      return token;
	    }
	    else
	      unread();
	  }

	  return token;
	}
	else {
	  unread();
	  return '/';
	}
	
      case '#':
	_lexeme.append((char) '#');
	while ((ch = read()) >= 0 && ch != '\r' && ch != '\n') {
	  _lexeme.append((char) ch);
	}
	unread();
	return T_COMMENT;

      case '.':
	if ((ch = read()) == '=') {
	  _lexeme.append(".=");
	  return T_CONCAT_EQUAL;
	}
	else {
	  unread();
	  return '.';
	}

      case '\'': case '"': case '`':
	{
	  int end = ch;

	  _lexeme.append((char) ch);

	  while ((ch = read()) >= 0 && ch != end) {
	    _lexeme.append((char) ch);

	    if (ch == '\\') {
	      _lexeme.append((char) read());
	    }
	  }

	  if (ch > 0)
	    _lexeme.append((char) ch);

	  return T_CONSTANT_ENCAPSED_STRING;
	}

      case '-':
	if ((ch = read()) == '-') {
	  _lexeme.append("--");
	  return T_DEC;
	}
	else if (ch == '=') {
	  _lexeme.append("-=");
	  return T_MINUS_EQUAL;
	}
	else if (ch == '>') {
	  _lexeme.append("->");
	  return T_OBJECT_OPERATOR;
	}
	else {
	  unread();
	  return '-';
	}

      case '+':
	if ((ch = read()) == '+') {
	  _lexeme.append("++");
	  return T_INC;
	}
	else if (ch == '=') {
	  _lexeme.append("+=");
	  return T_PLUS_EQUAL;
	}
	else {
	  unread();
	  return '+';
	}

      case '>':
	if ((ch = read()) == '>') {
	  if ((ch = read()) == '=') {
	    _lexeme.append(">>=");
	    return T_SR_EQUAL;
	  }
	  else {
	    unread();
	    _lexeme.append(">>");
	    return T_SR;
	  }
	}
	else if (ch == '=') {
	  _lexeme.append(">=");
	  return T_IS_GREATER_OR_EQUAL;
	}
	else {
	  unread();
	  return '>';
	}

      case '$':
	if ((ch = read()) == '{') {
	  _lexeme.append("${");
	  return T_DOLLAR_OPEN_CURLY_BRACES;
	}
	else if (ch == '$') {
	  unread();
	  return '$';
	}
	else if (Character.isJavaIdentifierStart(ch)) {
	  unread();
	  _lexeme.append("$");
	  readIdentifier();
	  return T_VARIABLE;
	}
	else {
	  unread();
	  return '$';
	}

      case '=':
	if ((ch = read()) == '=') {
	  if ((ch = read()) == '=') {
	    _lexeme.append("===");
	    return T_IS_IDENTICAL;
	  }
	  else {
	    unread();
	    _lexeme.append("==");
	    return T_IS_EQUAL;
	  }
	}
	else if (ch == '>') {
	  _lexeme.append("=>");
	  return T_DOUBLE_ARROW;
	}
	else {
	  unread();
	  return '=';
	}

      case '!':
	if ((ch = read()) == '=') {
	  if ((ch = read()) == '=') {
	    _lexeme.append("!==");
	    return T_IS_NOT_IDENTICAL;
	  }
	  else {
	    unread();
	    _lexeme.append("!=");
	    return T_IS_NOT_EQUAL;
	  }
	}
	else {
	  unread();
	  return '!';
	}

      case ':':
	if ((ch = read()) == ':') {
	  _lexeme.append("::");
	  return T_DOUBLE_COLON;
	}
	else {
	  unread();
	  return ':';
	}

      case '<':
	if ((ch = read()) == '?') {
	  if ((ch = read()) == '=') {
	    _lexeme.append("<?=");
	    return T_OPEN_TAG_WITH_ECHO;
	  }
	  else if (ch != 'p') {
	    unread();
	    _lexeme.append("<?");
	    return T_OPEN_TAG;
	  }
	  else if ((ch = read()) != 'h') {
	    unread();
	    unread();
	    _lexeme.append("<?");
	    return T_OPEN_TAG;
	  }
	  else if ((ch = read()) != 'p') {
	    unread();
	    unread();
	    unread();
	    _lexeme.append("<?");
	    return T_OPEN_TAG;
	  }
	  else {
	    _lexeme.append("<?php");
	    return T_OPEN_TAG;
	  }
	}
	else if (ch == '%') {
	  if ((ch = read()) == '=') {
	    _lexeme.append("<%=");
	    return T_OPEN_TAG_WITH_ECHO;
	  }
	  else {
	    unread();
	    _lexeme.append("<%");
	    return T_OPEN_TAG;
	  }
	}
	else if (ch == '<') {
	  if ((ch = read()) == '=') {
	    _lexeme.append("<<=");
	    return T_SL_EQUAL;
	  }
	  else if (ch == '<') {
	    _lexeme.append("<<<");
	    return T_START_HEREDOC;
	  }
	  else {
	    unread();
	    _lexeme.append("<<");
	    return T_SL;
	  }
	}
	else if (ch == '=') {
	  _lexeme.append("<=");
	  return T_IS_SMALLER_OR_EQUAL;
	}
	else if (ch == '>') {
	  _lexeme.append("<>");
	  return T_IS_NOT_EQUAL;
	}
	else {
	  unread();
	  return '<';
	}

      case '*':
	if ((ch = read()) == '=') {
	  _lexeme.append("*=");
	  return T_MUL_EQUAL;
	}
	else {
	  unread();
	  return '*';
	}

      case '%':
	if ((ch = read()) == '=') {
	  _lexeme.append("%=");
	  return T_MOD_EQUAL;
	}
	else {
	  unread();
	  return '%';
	}

      case '^':
	if ((ch = read()) == '=') {
	  _lexeme.append("^=");
	  return T_XOR_EQUAL;
	}
	else {
	  unread();
	  return '^';
	}

      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9':
	unread();
	return parseNumber();
	
      default:
	if (Character.isJavaIdentifierStart(ch)) {
	  unread();

	  readIdentifier();

	  int lexeme = _reservedMap.get(_lexeme.toString().toLowerCase());

	  if (lexeme > 0)
	    return lexeme;

	  return T_STRING;
	}
	
	_lexeme.append((char) ch);
	return T_BAD_CHARACTER;
      }
    }

    StringBuilderValue getLexeme()
    {
      return _lexeme;
    }
    
    private boolean parseHtml()
    {
      int ch;

      while ((ch = read()) >= 0) {
	if (ch != '<')
	  _lexeme.append((char) ch);
	else if ((ch = read()) == '?' || ch == '%') {
	  unread();
	  unread();

	  return _lexeme.length() > 0;
	}
	else {
	  _lexeme.append((char) '<');
	  
	  unread();
	}
      }

      return _lexeme.length() > 0;
    }

    private void readIdentifier()
    {
      int ch;
      
      while (Character.isJavaIdentifierPart((ch = read()))) {
	_lexeme.append((char) ch);
      }

      unread();
    }

    private int parseNumber()
    {
      boolean isInt = false;
      int ch;
      
      while ('0' <= (ch = read()) && ch <= '9'
	     || ch == '.'
	     || ch == 'x' || ch == 'X'
	     || 'a' <= ch && ch <= 'f'
	     || 'A' <= ch && ch <= 'F') {
	_lexeme.append((char) ch);

	if ('a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'f'
	    || ch == 'x' || ch == 'X')
	  isInt = true;
      }

      unread();
      
      return T_LNUMBER;
    }

    private int read()
    {
      if (_i < _length)
	return _s.charAt(_i++);
      else {
	_i++;
	return -1;
      }
    }
    
    private void unread()
    {
      if (_i <= _length)
	_i--;
    }
  }

  static {
    _reservedMap.put("abstract", T_ABSTRACT);
    _reservedMap.put("array", T_ARRAY);
    _reservedMap.put("as", T_AS);
    _reservedMap.put("break", T_BREAK);
    _reservedMap.put("case", T_CASE);
    _reservedMap.put("catch", T_CATCH);
    _reservedMap.put("class", T_CLASS);
    _reservedMap.put("clone", T_CLONE);
    _reservedMap.put("const", T_CONST);
    _reservedMap.put("continue", T_CONTINUE);
    _reservedMap.put("declare", T_DECLARE);
    _reservedMap.put("default", T_DEFAULT);
    _reservedMap.put("do", T_DO);
    _reservedMap.put("echo", T_ECHO);
    _reservedMap.put("else", T_ELSE);
    _reservedMap.put("elseif", T_ELSEIF);
    _reservedMap.put("empty", T_EMPTY);
    _reservedMap.put("enddeclare", T_ENDDECLARE);
    _reservedMap.put("endfor", T_ENDFOR);
    _reservedMap.put("endforeach", T_ENDFOREACH);
    _reservedMap.put("endif", T_ENDIF);
    _reservedMap.put("endswitch", T_ENDSWITCH);
    _reservedMap.put("eval", T_EVAL);
    _reservedMap.put("exit", T_EXIT);
    _reservedMap.put("die", T_EXIT);
    _reservedMap.put("extends", T_EXTENDS);
    _reservedMap.put("__FILE__", T_FILE);
    _reservedMap.put("final", T_FINAL);
    _reservedMap.put("for", T_FOR);
    _reservedMap.put("foreach", T_FOREACH);
    _reservedMap.put("function", T_FUNCTION);
    _reservedMap.put("cfunction", T_FUNCTION);
    _reservedMap.put("global", T_GLOBAL);
    _reservedMap.put("__halt_compiler", T_HALT_COMPILER);
    _reservedMap.put("if", T_IF);
    _reservedMap.put("implements", T_IMPLEMENTS);
    _reservedMap.put("include", T_INCLUDE);
    _reservedMap.put("include_once", T_INCLUDE_ONCE);
    _reservedMap.put("instanceof", T_INSTANCEOF);
    _reservedMap.put("isset", T_ISSET);
    _reservedMap.put("list", T_LIST);
    _reservedMap.put("and", T_LOGICAL_AND);
    _reservedMap.put("or", T_LOGICAL_OR);
    _reservedMap.put("xor", T_LOGICAL_XOR);
    _reservedMap.put("new", T_NEW);
    _reservedMap.put("old_function", T_OLD_FUNCTION);
    _reservedMap.put("print", T_PRINT);
    _reservedMap.put("private", T_PRIVATE);
    _reservedMap.put("public", T_PUBLIC);
    _reservedMap.put("protected", T_PROTECTED);
    _reservedMap.put("require", T_REQUIRE);
    _reservedMap.put("require_once", T_REQUIRE_ONCE);
    _reservedMap.put("return", T_RETURN);
    _reservedMap.put("static", T_STATIC);
    _reservedMap.put("switch", T_SWITCH);
    _reservedMap.put("throw", T_THROW);
    _reservedMap.put("try", T_TRY);
    _reservedMap.put("unset", T_UNSET);
    _reservedMap.put("var", T_VAR);
    _reservedMap.put("while", T_WHILE);
    _reservedMap.put("__FUNCTION__", T_FUNC_C);
    _reservedMap.put("__CLASS__", T_CLASS_C);
  }
}
