/*
 * Copyright 2008-2010 the T2 Project ant the Others.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
//package org.t2framework.commons.util;

/**
 * 
 * <#if locale="en">
 * <p>
 * JSON tokenizer.It makes tokens for parsing json data.
 * </p>
 * <#else>
 * <p>
 * 
 * </p>
 * </#if>
 * 
 * @author shot
 * 
 *         TODO : unicode, quotation("), reverse solidus(\), solidus(/),
 * 
 */
public class JSONTokenizer {

  public static final char COMMA = ',';

  public static final char COLON = ':';

  public static final char SINGLE_QUOTE = '\'';

  public static final char DOUBLE_QUOTE = '\"';

  public static final String NULL_STRING = "null";

  public static final String TRUE_STRING = "true";

  public static final String FALSE_STRING = "false";

  public static final char START_BRACKET = '[';

  public static final char END_BRACKET = ']';

  public static final char START_BRACE = '{';

  public static final char END_BRACE = '}';

  // public static final char UTF8_BOM = 0xFEFF;

  protected final String orgString;

  protected final int orgLength;

  protected String tokenString;

  protected int pos;

  protected int nextPos;

  protected TokenType type = TokenType.EOF;

  public JSONTokenizer(String s) {
    // TODO : check grammer first.
    this.orgString = (s != null) ? s : "null";
    this.orgLength = this.orgString.length();
    this.tokenString = this.orgString;
    prepare(0);
  }

  protected void prepare(int i) {
    if (i < orgLength) {
      char ch = orgString.charAt(i);
      if (ch == START_BRACE) {
        type = TokenType.START_BRACE;
      } else if (ch == END_BRACE) {
        type = TokenType.END_BRACE;
      } else if (ch == START_BRACKET) {
        type = TokenType.START_BRACKET;
      } else if (ch == END_BRACKET) {
        type = TokenType.END_BRACKET;
      }
      this.pos = i;
      this.nextPos = i + 1;
    } else {
      type = TokenType.EOF;
    }
  }

  public TokenType nextToken() {
    if (type == TokenType.EOF) {
      return TokenType.EOF;
    }
    String s = this.tokenString;
    TokenType retType = TokenType.EOF;
    boolean key = true;
    for (int i = this.nextPos; i < this.orgLength; i++) {
      char ch = this.orgString.charAt(i);
      if (isIgnorable(ch)) {
        continue;
      }
      if (ch == START_BRACE) {
        s = getElement(nextPos, this.orgLength - 1);
        retType = TokenType.START_BRACE;
        prepare(i);
        key = true;
        break;
      } else if (ch == START_BRACKET) {
        s = getElement(nextPos, this.orgLength - 1);
        retType = TokenType.START_BRACKET;
        prepare(i);
        key = true;
        break;
      } else if (ch == COLON) {
        if (i == this.orgLength - 1) {
          throw new IllegalStateException();
        }
        s = getElement(nextPos, i - 1);
        this.type = retType = TokenType.COLON;
        prepare(i);
        key = true;
        break;
      } else if (ch == COMMA) {
        if (i == this.orgLength - 1) {
          throw new IllegalArgumentException();
        }
        s = getElement(nextPos, i - 1);
        this.type = retType = (isObjectOrArrayEnd(i - 1)) ? TokenType.END_COMMA
            : TokenType.COMMA;
        prepare(i);
        key = false;
        break;
      } else if (ch == END_BRACKET) {
        this.type = (i == this.orgLength - 1) ? TokenType.EOF
            : TokenType.END_BRACKET;
        retType = TokenType.END_BRACKET;
        s = getElement(nextPos, i - 1);
        prepare(i);
        key = false;
        break;
      } else if (ch == END_BRACE) {
        this.type = (i == this.orgLength - 1) ? TokenType.EOF
            : TokenType.END_BRACE;
        retType = TokenType.END_BRACE;
        s = getElement(this.nextPos, i - 1);
        prepare(i);
        key = false;
        break;
      }
    }
    s = removeIgnorable(s);
    this.tokenString = (key) ? unquote(s) : s;
    return retType;
  }

  protected boolean isObjectOrArrayEnd(int pos) {
    for (int i = pos; 0 < i; i--) {
      char c = this.orgString.charAt(i);
      if (isIgnorable(c)) {
        continue;
      }
      if (c == END_BRACE || c == END_BRACKET) {
        return true;
      } else {
        return false;
      }
    }
    return false;
  }

  /**
   * <#if locale="en">
   * <p>
   * TODO refactor.
   * 
   * </p>
   * <#else>
   * <p>
   * 
   * </p>
   * </#if>
   * 
   * @param s
   * @return
   */
  protected String removeIgnorable(String s) {
    if (isEmpty(s)) {
      return s;
    }
    for (int pos1 = 0; pos1 < s.length(); pos1++) {
      if (isIgnorable(s.charAt(pos1)) == false) {
        return s.substring(pos1);
      }
    }
    return "";
  }

  public static boolean isEmpty(String text) {
    return text == null || text.length() == 0;
  }

  protected String getElement(final int orgStartPos, final int orgEndPos) {
    int startPos = orgStartPos;
    for (; startPos < orgEndPos; startPos++) {
      char ch = this.orgString.charAt(startPos);
      if (isIgnorable(ch) == false) {
        break;
      }
    }
    int endPos = orgEndPos;
    for (; startPos < endPos; endPos--) {
      char ch = this.orgString.charAt(endPos);
      if (isIgnorable(ch) == false) {
        break;
      }
    }
    return this.orgString.substring(startPos, endPos + 1);
  }

  protected static boolean isIgnorable(char ch) {
    boolean ret = false;
    switch (ch) {
    case '\b':
    case '\f':
    case '\n':
    case '\r':
    case '\t':
    case ' ':
      ret = true;
      break;
    default:
      ret = false;
      break;
    }
    return ret;
  }

  public static String unquote(String str) {
    if (str == null || str.length() == 0) {
      return str;
    }
    if (isQuoted(str, DOUBLE_QUOTE)) {
      return chopQuote(str);
    } else if (isQuoted(str, SINGLE_QUOTE)) {
      return chopQuote(str);
    } else {
      return str;
    }
  }

  public static String chopQuote(String str) {
    return str.substring(1, str.length() - 1);
  }

  protected static boolean isQuoted(String str, char quote) {
    return str.indexOf(quote) == 0 && str.lastIndexOf(quote) > 0;
  }

  public String getToken() {
    return this.tokenString;
  }

  public static enum TokenType {
    /**
     * <#if locale="en">
     * <p>
     * "{"
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    START_BRACE {
      public String toString() {
        return "{";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * "}"
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    END_BRACE {
      public String toString() {
        return "}";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * "["
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    START_BRACKET {
      public String toString() {
        return "[";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * "]"
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    END_BRACKET {
      public String toString() {
        return "]";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * ","
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    COMMA {
      public String toString() {
        return ",";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * ":"
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    COLON {
      public String toString() {
        return ":";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * "," and it is the end of {} or [].
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    END_COMMA {
      public String toString() {
        return ",";
      }
    },

    /**
     * <#if locale="en">
     * <p>
     * End of file.
     * </p>
     * <#else>
     * <p>
     * 
     * </p>
     * </#if>
     */
    EOF;

  }

  public TokenType currentTokenType() {
    return type;
  }
}
-------------------------
/*
 * Copyright 2008-2009 the T2 Project ant the Others.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.t2framework.commons.util;

import org.t2framework.commons.util.JSONTokenizer.TokenType;

import junit.framework.TestCase;

public class JSONTokenizerTest extends TestCase {

  public void test1_blank() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("");
    assertEquals(TokenType.EOF, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test2_null() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(null);
    assertEquals(TokenType.EOF, tokenizer.nextToken());
    assertEquals("null", tokenizer.getToken());
  }

  public void test3_keyvalue() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\"aaa\":123}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{\"aaa\":123}", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test4_array() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("[\"aaa\",123,\"b\"]");
    assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType());
    assertEquals("[\"aaa\",123,\"b\"]", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("\"aaa\"", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
    assertEquals("\"b\"", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test5_emptyobject() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{}", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test6_emptyarray() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("[]");
    assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType());
    assertEquals("[]", tokenizer.getToken());
    assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test7_multiplekeyvalue() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "{\"aaa\":123,\"bbb\":true}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{\"aaa\":123,\"bbb\":true}", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("true", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test8_keyvaluewithsinglequote() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{'aaa':'123'}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{'aaa':'123'}", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("'123'", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test9_blankwithtab() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\t}");
    assertEquals("{\t}", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test10_blankwithbackspace() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\b}");
    assertEquals("{\b}", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test11_blankwithformfeed() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\f}");
    assertEquals("{\f}", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test12_blankwithnewline() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\n}");
    assertEquals("{\n}", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test13_blankwithcarriagereturn() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\r}");
    assertEquals("{\r}", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test14_keyvalue_nest1() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "{\"aaa\":123, \"bbb\":{\"b1\":true}}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{\"aaa\":123, \"bbb\":{\"b1\":true}}", tokenizer
        .getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
      assertEquals("{\"b1\":true}}", tokenizer.getToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("b1", tokenizer.getToken());
      assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
      assertEquals("true", tokenizer.getToken());
    }
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
  }

  public void test15_ignorableSpaceShouldIgnoreAtObject() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\"aaa\"\r\t:\n123   }");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test16_ignorableSpaceShouldIgnoreAtArray() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "[ \"aaa\"\t,123\b,\f\'b\'     ]");
    assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("\"aaa\"", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
    assertEquals("\'b\'", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test17_blankwithlotsofignorables() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\t\r\n    \t}");
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("", tokenizer.getToken());
  }

  public void test18_keyvalue_nest3_array() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\", \"array1\":[\"1.1233333333000000000000001\"\r,\b1.1233333333000000000000001, \"3.0\"]}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("b1", tokenizer.getToken());
      assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
      assertEquals("true", tokenizer.getToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("ccc", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("\"fuga\"", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("array1", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACKET, tokenizer.nextToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("\"1.1233333333000000000000001\"", tokenizer
          .getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("1.1233333333000000000000001", tokenizer.getToken());
      assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
      assertEquals("\"3.0\"", tokenizer.getToken());
    }
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test18_stringEnquote() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\"a'aa\":\"?????\"}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("a'aa", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("\"?????\"", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test19_booleanarray() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("[true, false,true]");
    assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("true", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("false", tokenizer.getToken());
    assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
    assertEquals("true", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test20_nestarray() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("[1, [2, 3, 4, 5], 3]");
    assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("1", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACKET, tokenizer.nextToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("2", tokenizer.getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("3", tokenizer.getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("4", tokenizer.getToken());
      assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
      assertEquals("5", tokenizer.getToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
    assertEquals("3", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test21_nestarrayandobjects() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "[1, [2, 3, 4, 5], \"key\":{true, false, \"hoge\", \"array\":[0.001, 0.00001, 1.2E-7]   }, 3]");
    assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("1", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACKET, tokenizer.nextToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("2", tokenizer.getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("3", tokenizer.getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("4", tokenizer.getToken());
      assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
      assertEquals("5", tokenizer.getToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("key", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("true", tokenizer.getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("false", tokenizer.getToken());
      assertEquals(TokenType.COMMA, tokenizer.nextToken());
      assertEquals("\"hoge\"", tokenizer.getToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("array", tokenizer.getToken());
      {
        assertEquals(TokenType.START_BRACKET, tokenizer.nextToken());
        assertEquals(TokenType.COMMA, tokenizer.nextToken());
        assertEquals("0.001", tokenizer.getToken());
        assertEquals(TokenType.COMMA, tokenizer.nextToken());
        assertEquals("0.00001", tokenizer.getToken());
        assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
        assertEquals("1.2E-7", tokenizer.getToken());
      }
      assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.END_BRACKET, tokenizer.nextToken());
    assertEquals("3", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test22_stringSingleEnquote() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\'a'aa\':\"?????\"}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("a'aa", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("\"?????\"", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  /**
   * 
   * {@.en }
   * 
   * <br />
   * 
   * {@.ja ?????????key????String??????????.}
   * 
   * @throws Exception
   */
  public void test23_keyMustBeString() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{aaa:123}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test24_keyvalue_nestOnlyNestObject() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer("{\"bbb\":{\"b1\":true}}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{\"bbb\":{\"b1\":true}}", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
    assertEquals("{\"b1\":true}}", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("b1", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("true", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
  }

  public void test25_keyvalue_nestOnlyNestObject2() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "{\"bbb\":{\"b1\":true}, \"vvv\":null}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{\"bbb\":{\"b1\":true}, \"vvv\":null}", tokenizer
        .getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
      assertEquals("{\"b1\":true}, \"vvv\":null}", tokenizer.getToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("b1", tokenizer.getToken());
      assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
      assertEquals("true", tokenizer.getToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("vvv", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("null", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test26_keyvalue_deepNest1() throws Exception {
    final String json = "{\"bbb\":{\"dates\":{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}";
    JSONTokenizer tokenizer = new JSONTokenizer(json);
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals(json, tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
      assertEquals(
          "{\"dates\":{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}",
          tokenizer.getToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("dates", tokenizer.getToken());
      {
        assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
        assertEquals(
            "{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}",
            tokenizer.getToken());
        assertEquals(TokenType.COLON, tokenizer.nextToken());
        assertEquals("from", tokenizer.getToken());
        assertEquals(TokenType.COMMA, tokenizer.nextToken());
        assertEquals("20090101", tokenizer.getToken());
        assertEquals(TokenType.COLON, tokenizer.nextToken());
        assertEquals("to", tokenizer.getToken());
        assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
        assertEquals("20091231", tokenizer.getToken());
      }
      assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("b1", tokenizer.getToken());
      assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
      assertEquals("true", tokenizer.getToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("vvv", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("null", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

  public void test27_keyvalue_nest2() throws Exception {
    JSONTokenizer tokenizer = new JSONTokenizer(
        "{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\"}");
    assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType());
    assertEquals("{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\"}",
        tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("aaa", tokenizer.getToken());
    assertEquals(TokenType.COMMA, tokenizer.nextToken());
    assertEquals("123", tokenizer.getToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("bbb", tokenizer.getToken());
    {
      assertEquals(TokenType.START_BRACE, tokenizer.nextToken());
      assertEquals("{\"b1\":true},\t\"ccc\":\"fuga\"}", tokenizer
          .getToken());
      assertEquals(TokenType.COLON, tokenizer.nextToken());
      assertEquals("b1", tokenizer.getToken());
      assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
      assertEquals("true", tokenizer.getToken());
    }
    assertEquals(TokenType.END_COMMA, tokenizer.nextToken());
    assertEquals(TokenType.COLON, tokenizer.nextToken());
    assertEquals("ccc", tokenizer.getToken());
    assertEquals(TokenType.END_BRACE, tokenizer.nextToken());
    assertEquals("\"fuga\"", tokenizer.getToken());
    assertEquals(TokenType.EOF, tokenizer.nextToken());
  }

}