1 | /* |
2 | * Copyright 2008-2010 the T2 Project ant the Others. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | //package org.t2framework.commons.util; |
17 | |
18 | /** |
19 | * |
20 | * <#if locale="en"> |
21 | * <p> |
22 | * JSON tokenizer.It makes tokens for parsing json data. |
23 | * </p> |
24 | * <#else> |
25 | * <p> |
26 | * |
27 | * </p> |
28 | * </#if> |
29 | * |
30 | * @author shot |
31 | * |
32 | * TODO : unicode, quotation("), reverse solidus(\), solidus(/), |
33 | * |
34 | */ |
35 | public class JSONTokenizer { |
36 | |
37 | public static final char COMMA = ','; |
38 | |
39 | public static final char COLON = ':'; |
40 | |
41 | public static final char SINGLE_QUOTE = '\''; |
42 | |
43 | public static final char DOUBLE_QUOTE = '\"'; |
44 | |
45 | public static final String NULL_STRING = "null"; |
46 | |
47 | public static final String TRUE_STRING = "true"; |
48 | |
49 | public static final String FALSE_STRING = "false"; |
50 | |
51 | public static final char START_BRACKET = '['; |
52 | |
53 | public static final char END_BRACKET = ']'; |
54 | |
55 | public static final char START_BRACE = '{'; |
56 | |
57 | public static final char END_BRACE = '}'; |
58 | |
59 | // public static final char UTF8_BOM = 0xFEFF; |
60 | |
61 | protected final String orgString; |
62 | |
63 | protected final int orgLength; |
64 | |
65 | protected String tokenString; |
66 | |
67 | protected int pos; |
68 | |
69 | protected int nextPos; |
70 | |
71 | protected TokenType type = TokenType.EOF; |
72 | |
73 | public JSONTokenizer(String s) { |
74 | // TODO : check grammer first. |
75 | this.orgString = (s != null) ? s : "null"; |
76 | this.orgLength = this.orgString.length(); |
77 | this.tokenString = this.orgString; |
78 | prepare(0); |
79 | } |
80 | |
81 | protected void prepare(int i) { |
82 | if (i < orgLength) { |
83 | char ch = orgString.charAt(i); |
84 | if (ch == START_BRACE) { |
85 | type = TokenType.START_BRACE; |
86 | } else if (ch == END_BRACE) { |
87 | type = TokenType.END_BRACE; |
88 | } else if (ch == START_BRACKET) { |
89 | type = TokenType.START_BRACKET; |
90 | } else if (ch == END_BRACKET) { |
91 | type = TokenType.END_BRACKET; |
92 | } |
93 | this.pos = i; |
94 | this.nextPos = i + 1; |
95 | } else { |
96 | type = TokenType.EOF; |
97 | } |
98 | } |
99 | |
100 | public TokenType nextToken() { |
101 | if (type == TokenType.EOF) { |
102 | return TokenType.EOF; |
103 | } |
104 | String s = this.tokenString; |
105 | TokenType retType = TokenType.EOF; |
106 | boolean key = true; |
107 | for (int i = this.nextPos; i < this.orgLength; i++) { |
108 | char ch = this.orgString.charAt(i); |
109 | if (isIgnorable(ch)) { |
110 | continue; |
111 | } |
112 | if (ch == START_BRACE) { |
113 | s = getElement(nextPos, this.orgLength - 1); |
114 | retType = TokenType.START_BRACE; |
115 | prepare(i); |
116 | key = true; |
117 | break; |
118 | } else if (ch == START_BRACKET) { |
119 | s = getElement(nextPos, this.orgLength - 1); |
120 | retType = TokenType.START_BRACKET; |
121 | prepare(i); |
122 | key = true; |
123 | break; |
124 | } else if (ch == COLON) { |
125 | if (i == this.orgLength - 1) { |
126 | throw new IllegalStateException(); |
127 | } |
128 | s = getElement(nextPos, i - 1); |
129 | this.type = retType = TokenType.COLON; |
130 | prepare(i); |
131 | key = true; |
132 | break; |
133 | } else if (ch == COMMA) { |
134 | if (i == this.orgLength - 1) { |
135 | throw new IllegalArgumentException(); |
136 | } |
137 | s = getElement(nextPos, i - 1); |
138 | this.type = retType = (isObjectOrArrayEnd(i - 1)) ? TokenType.END_COMMA |
139 | : TokenType.COMMA; |
140 | prepare(i); |
141 | key = false; |
142 | break; |
143 | } else if (ch == END_BRACKET) { |
144 | this.type = (i == this.orgLength - 1) ? TokenType.EOF |
145 | : TokenType.END_BRACKET; |
146 | retType = TokenType.END_BRACKET; |
147 | s = getElement(nextPos, i - 1); |
148 | prepare(i); |
149 | key = false; |
150 | break; |
151 | } else if (ch == END_BRACE) { |
152 | this.type = (i == this.orgLength - 1) ? TokenType.EOF |
153 | : TokenType.END_BRACE; |
154 | retType = TokenType.END_BRACE; |
155 | s = getElement(this.nextPos, i - 1); |
156 | prepare(i); |
157 | key = false; |
158 | break; |
159 | } |
160 | } |
161 | s = removeIgnorable(s); |
162 | this.tokenString = (key) ? unquote(s) : s; |
163 | return retType; |
164 | } |
165 | |
166 | protected boolean isObjectOrArrayEnd(int pos) { |
167 | for (int i = pos; 0 < i; i--) { |
168 | char c = this.orgString.charAt(i); |
169 | if (isIgnorable(c)) { |
170 | continue; |
171 | } |
172 | if (c == END_BRACE || c == END_BRACKET) { |
173 | return true; |
174 | } else { |
175 | return false; |
176 | } |
177 | } |
178 | return false; |
179 | } |
180 | |
181 | /** |
182 | * <#if locale="en"> |
183 | * <p> |
184 | * TODO refactor. |
185 | * |
186 | * </p> |
187 | * <#else> |
188 | * <p> |
189 | * |
190 | * </p> |
191 | * </#if> |
192 | * |
193 | * @param s |
194 | * @return |
195 | */ |
196 | protected String removeIgnorable(String s) { |
197 | if (isEmpty(s)) { |
198 | return s; |
199 | } |
200 | for (int pos1 = 0; pos1 < s.length(); pos1++) { |
201 | if (isIgnorable(s.charAt(pos1)) == false) { |
202 | return s.substring(pos1); |
203 | } |
204 | } |
205 | return ""; |
206 | } |
207 | |
208 | public static boolean isEmpty(String text) { |
209 | return text == null || text.length() == 0; |
210 | } |
211 | |
212 | protected String getElement(final int orgStartPos, final int orgEndPos) { |
213 | int startPos = orgStartPos; |
214 | for (; startPos < orgEndPos; startPos++) { |
215 | char ch = this.orgString.charAt(startPos); |
216 | if (isIgnorable(ch) == false) { |
217 | break; |
218 | } |
219 | } |
220 | int endPos = orgEndPos; |
221 | for (; startPos < endPos; endPos--) { |
222 | char ch = this.orgString.charAt(endPos); |
223 | if (isIgnorable(ch) == false) { |
224 | break; |
225 | } |
226 | } |
227 | return this.orgString.substring(startPos, endPos + 1); |
228 | } |
229 | |
230 | protected static boolean isIgnorable(char ch) { |
231 | boolean ret = false; |
232 | switch (ch) { |
233 | case '\b': |
234 | case '\f': |
235 | case '\n': |
236 | case '\r': |
237 | case '\t': |
238 | case ' ': |
239 | ret = true; |
240 | break; |
241 | default: |
242 | ret = false; |
243 | break; |
244 | } |
245 | return ret; |
246 | } |
247 | |
248 | public static String unquote(String str) { |
249 | if (str == null || str.length() == 0) { |
250 | return str; |
251 | } |
252 | if (isQuoted(str, DOUBLE_QUOTE)) { |
253 | return chopQuote(str); |
254 | } else if (isQuoted(str, SINGLE_QUOTE)) { |
255 | return chopQuote(str); |
256 | } else { |
257 | return str; |
258 | } |
259 | } |
260 | |
261 | public static String chopQuote(String str) { |
262 | return str.substring(1, str.length() - 1); |
263 | } |
264 | |
265 | protected static boolean isQuoted(String str, char quote) { |
266 | return str.indexOf(quote) == 0 && str.lastIndexOf(quote) > 0; |
267 | } |
268 | |
269 | public String getToken() { |
270 | return this.tokenString; |
271 | } |
272 | |
273 | public static enum TokenType { |
274 | /** |
275 | * <#if locale="en"> |
276 | * <p> |
277 | * "{" |
278 | * </p> |
279 | * <#else> |
280 | * <p> |
281 | * |
282 | * </p> |
283 | * </#if> |
284 | */ |
285 | START_BRACE { |
286 | public String toString() { |
287 | return "{"; |
288 | } |
289 | }, |
290 | |
291 | /** |
292 | * <#if locale="en"> |
293 | * <p> |
294 | * "}" |
295 | * </p> |
296 | * <#else> |
297 | * <p> |
298 | * |
299 | * </p> |
300 | * </#if> |
301 | */ |
302 | END_BRACE { |
303 | public String toString() { |
304 | return "}"; |
305 | } |
306 | }, |
307 | |
308 | /** |
309 | * <#if locale="en"> |
310 | * <p> |
311 | * "[" |
312 | * </p> |
313 | * <#else> |
314 | * <p> |
315 | * |
316 | * </p> |
317 | * </#if> |
318 | */ |
319 | START_BRACKET { |
320 | public String toString() { |
321 | return "["; |
322 | } |
323 | }, |
324 | |
325 | /** |
326 | * <#if locale="en"> |
327 | * <p> |
328 | * "]" |
329 | * </p> |
330 | * <#else> |
331 | * <p> |
332 | * |
333 | * </p> |
334 | * </#if> |
335 | */ |
336 | END_BRACKET { |
337 | public String toString() { |
338 | return "]"; |
339 | } |
340 | }, |
341 | |
342 | /** |
343 | * <#if locale="en"> |
344 | * <p> |
345 | * "," |
346 | * </p> |
347 | * <#else> |
348 | * <p> |
349 | * |
350 | * </p> |
351 | * </#if> |
352 | */ |
353 | COMMA { |
354 | public String toString() { |
355 | return ","; |
356 | } |
357 | }, |
358 | |
359 | /** |
360 | * <#if locale="en"> |
361 | * <p> |
362 | * ":" |
363 | * </p> |
364 | * <#else> |
365 | * <p> |
366 | * |
367 | * </p> |
368 | * </#if> |
369 | */ |
370 | COLON { |
371 | public String toString() { |
372 | return ":"; |
373 | } |
374 | }, |
375 | |
376 | /** |
377 | * <#if locale="en"> |
378 | * <p> |
379 | * "," and it is the end of {} or []. |
380 | * </p> |
381 | * <#else> |
382 | * <p> |
383 | * |
384 | * </p> |
385 | * </#if> |
386 | */ |
387 | END_COMMA { |
388 | public String toString() { |
389 | return ","; |
390 | } |
391 | }, |
392 | |
393 | /** |
394 | * <#if locale="en"> |
395 | * <p> |
396 | * End of file. |
397 | * </p> |
398 | * <#else> |
399 | * <p> |
400 | * |
401 | * </p> |
402 | * </#if> |
403 | */ |
404 | EOF; |
405 | |
406 | } |
407 | |
408 | public TokenType currentTokenType() { |
409 | return type; |
410 | } |
411 | } |
412 | ------------------------- |
413 | /* |
414 | * Copyright 2008-2009 the T2 Project ant the Others. |
415 | * |
416 | * Licensed under the Apache License, Version 2.0 (the "License"); |
417 | * you may not use this file except in compliance with the License. |
418 | * You may obtain a copy of the License at |
419 | * |
420 | * http://www.apache.org/licenses/LICENSE-2.0 |
421 | * |
422 | * Unless required by applicable law or agreed to in writing, software |
423 | * distributed under the License is distributed on an "AS IS" BASIS, |
424 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
425 | * See the License for the specific language governing permissions and |
426 | * limitations under the License. |
427 | */ |
428 | package org.t2framework.commons.util; |
429 | |
430 | import org.t2framework.commons.util.JSONTokenizer.TokenType; |
431 | |
432 | import junit.framework.TestCase; |
433 | |
434 | public class JSONTokenizerTest extends TestCase { |
435 | |
436 | public void test1_blank() throws Exception { |
437 | JSONTokenizer tokenizer = new JSONTokenizer(""); |
438 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
439 | assertEquals("", tokenizer.getToken()); |
440 | } |
441 | |
442 | public void test2_null() throws Exception { |
443 | JSONTokenizer tokenizer = new JSONTokenizer(null); |
444 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
445 | assertEquals("null", tokenizer.getToken()); |
446 | } |
447 | |
448 | public void test3_keyvalue() throws Exception { |
449 | JSONTokenizer tokenizer = new JSONTokenizer("{\"aaa\":123}"); |
450 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
451 | assertEquals("{\"aaa\":123}", tokenizer.getToken()); |
452 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
453 | assertEquals("aaa", tokenizer.getToken()); |
454 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
455 | assertEquals("123", tokenizer.getToken()); |
456 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
457 | } |
458 | |
459 | public void test4_array() throws Exception { |
460 | JSONTokenizer tokenizer = new JSONTokenizer("[\"aaa\",123,\"b\"]"); |
461 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
462 | assertEquals("[\"aaa\",123,\"b\"]", tokenizer.getToken()); |
463 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
464 | assertEquals("\"aaa\"", tokenizer.getToken()); |
465 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
466 | assertEquals("123", tokenizer.getToken()); |
467 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
468 | assertEquals("\"b\"", tokenizer.getToken()); |
469 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
470 | } |
471 | |
472 | public void test5_emptyobject() throws Exception { |
473 | JSONTokenizer tokenizer = new JSONTokenizer("{}"); |
474 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
475 | assertEquals("{}", tokenizer.getToken()); |
476 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
477 | assertEquals("", tokenizer.getToken()); |
478 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
479 | } |
480 | |
481 | public void test6_emptyarray() throws Exception { |
482 | JSONTokenizer tokenizer = new JSONTokenizer("[]"); |
483 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
484 | assertEquals("[]", tokenizer.getToken()); |
485 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
486 | assertEquals("", tokenizer.getToken()); |
487 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
488 | } |
489 | |
490 | public void test7_multiplekeyvalue() throws Exception { |
491 | JSONTokenizer tokenizer = new JSONTokenizer( |
492 | "{\"aaa\":123,\"bbb\":true}"); |
493 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
494 | assertEquals("{\"aaa\":123,\"bbb\":true}", tokenizer.getToken()); |
495 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
496 | assertEquals("aaa", tokenizer.getToken()); |
497 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
498 | assertEquals("123", tokenizer.getToken()); |
499 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
500 | assertEquals("bbb", tokenizer.getToken()); |
501 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
502 | assertEquals("true", tokenizer.getToken()); |
503 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
504 | } |
505 | |
506 | public void test8_keyvaluewithsinglequote() throws Exception { |
507 | JSONTokenizer tokenizer = new JSONTokenizer("{'aaa':'123'}"); |
508 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
509 | assertEquals("{'aaa':'123'}", tokenizer.getToken()); |
510 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
511 | assertEquals("aaa", tokenizer.getToken()); |
512 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
513 | assertEquals("'123'", tokenizer.getToken()); |
514 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
515 | } |
516 | |
517 | public void test9_blankwithtab() throws Exception { |
518 | JSONTokenizer tokenizer = new JSONTokenizer("{\t}"); |
519 | assertEquals("{\t}", tokenizer.getToken()); |
520 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
521 | assertEquals("", tokenizer.getToken()); |
522 | } |
523 | |
524 | public void test10_blankwithbackspace() throws Exception { |
525 | JSONTokenizer tokenizer = new JSONTokenizer("{\b}"); |
526 | assertEquals("{\b}", tokenizer.getToken()); |
527 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
528 | assertEquals("", tokenizer.getToken()); |
529 | } |
530 | |
531 | public void test11_blankwithformfeed() throws Exception { |
532 | JSONTokenizer tokenizer = new JSONTokenizer("{\f}"); |
533 | assertEquals("{\f}", tokenizer.getToken()); |
534 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
535 | assertEquals("", tokenizer.getToken()); |
536 | } |
537 | |
538 | public void test12_blankwithnewline() throws Exception { |
539 | JSONTokenizer tokenizer = new JSONTokenizer("{\n}"); |
540 | assertEquals("{\n}", tokenizer.getToken()); |
541 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
542 | assertEquals("", tokenizer.getToken()); |
543 | } |
544 | |
545 | public void test13_blankwithcarriagereturn() throws Exception { |
546 | JSONTokenizer tokenizer = new JSONTokenizer("{\r}"); |
547 | assertEquals("{\r}", tokenizer.getToken()); |
548 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
549 | assertEquals("", tokenizer.getToken()); |
550 | } |
551 | |
552 | public void test14_keyvalue_nest1() throws Exception { |
553 | JSONTokenizer tokenizer = new JSONTokenizer( |
554 | "{\"aaa\":123, \"bbb\":{\"b1\":true}}"); |
555 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
556 | assertEquals("{\"aaa\":123, \"bbb\":{\"b1\":true}}", tokenizer |
557 | .getToken()); |
558 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
559 | assertEquals("aaa", tokenizer.getToken()); |
560 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
561 | assertEquals("123", tokenizer.getToken()); |
562 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
563 | assertEquals("bbb", tokenizer.getToken()); |
564 | { |
565 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
566 | assertEquals("{\"b1\":true}}", tokenizer.getToken()); |
567 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
568 | assertEquals("b1", tokenizer.getToken()); |
569 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
570 | assertEquals("true", tokenizer.getToken()); |
571 | } |
572 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
573 | } |
574 | |
575 | public void test15_ignorableSpaceShouldIgnoreAtObject() throws Exception { |
576 | JSONTokenizer tokenizer = new JSONTokenizer("{\"aaa\"\r\t:\n123 }"); |
577 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
578 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
579 | assertEquals("aaa", tokenizer.getToken()); |
580 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
581 | assertEquals("123", tokenizer.getToken()); |
582 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
583 | } |
584 | |
585 | public void test16_ignorableSpaceShouldIgnoreAtArray() throws Exception { |
586 | JSONTokenizer tokenizer = new JSONTokenizer( |
587 | "[ \"aaa\"\t,123\b,\f\'b\' ]"); |
588 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
589 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
590 | assertEquals("\"aaa\"", tokenizer.getToken()); |
591 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
592 | assertEquals("123", tokenizer.getToken()); |
593 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
594 | assertEquals("\'b\'", tokenizer.getToken()); |
595 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
596 | } |
597 | |
598 | public void test17_blankwithlotsofignorables() throws Exception { |
599 | JSONTokenizer tokenizer = new JSONTokenizer("{\t\r\n \t}"); |
600 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
601 | assertEquals("", tokenizer.getToken()); |
602 | } |
603 | |
604 | public void test18_keyvalue_nest3_array() throws Exception { |
605 | JSONTokenizer tokenizer = new JSONTokenizer( |
606 | "{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\", \"array1\":[\"1.1233333333000000000000001\"\r,\b1.1233333333000000000000001, \"3.0\"]}"); |
607 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
608 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
609 | assertEquals("aaa", tokenizer.getToken()); |
610 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
611 | assertEquals("123", tokenizer.getToken()); |
612 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
613 | assertEquals("bbb", tokenizer.getToken()); |
614 | { |
615 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
616 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
617 | assertEquals("b1", tokenizer.getToken()); |
618 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
619 | assertEquals("true", tokenizer.getToken()); |
620 | } |
621 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
622 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
623 | assertEquals("ccc", tokenizer.getToken()); |
624 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
625 | assertEquals("\"fuga\"", tokenizer.getToken()); |
626 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
627 | assertEquals("array1", tokenizer.getToken()); |
628 | { |
629 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
630 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
631 | assertEquals("\"1.1233333333000000000000001\"", tokenizer |
632 | .getToken()); |
633 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
634 | assertEquals("1.1233333333000000000000001", tokenizer.getToken()); |
635 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
636 | assertEquals("\"3.0\"", tokenizer.getToken()); |
637 | } |
638 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
639 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
640 | } |
641 | |
642 | public void test18_stringEnquote() throws Exception { |
643 | JSONTokenizer tokenizer = new JSONTokenizer("{\"a'aa\":\"?????\"}"); |
644 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
645 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
646 | assertEquals("a'aa", tokenizer.getToken()); |
647 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
648 | assertEquals("\"?????\"", tokenizer.getToken()); |
649 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
650 | } |
651 | |
652 | public void test19_booleanarray() throws Exception { |
653 | JSONTokenizer tokenizer = new JSONTokenizer("[true, false,true]"); |
654 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
655 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
656 | assertEquals("true", tokenizer.getToken()); |
657 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
658 | assertEquals("false", tokenizer.getToken()); |
659 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
660 | assertEquals("true", tokenizer.getToken()); |
661 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
662 | } |
663 | |
664 | public void test20_nestarray() throws Exception { |
665 | JSONTokenizer tokenizer = new JSONTokenizer("[1, [2, 3, 4, 5], 3]"); |
666 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
667 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
668 | assertEquals("1", tokenizer.getToken()); |
669 | { |
670 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
671 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
672 | assertEquals("2", tokenizer.getToken()); |
673 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
674 | assertEquals("3", tokenizer.getToken()); |
675 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
676 | assertEquals("4", tokenizer.getToken()); |
677 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
678 | assertEquals("5", tokenizer.getToken()); |
679 | } |
680 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
681 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
682 | assertEquals("3", tokenizer.getToken()); |
683 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
684 | } |
685 | |
686 | public void test21_nestarrayandobjects() throws Exception { |
687 | JSONTokenizer tokenizer = new JSONTokenizer( |
688 | "[1, [2, 3, 4, 5], \"key\":{true, false, \"hoge\", \"array\":[0.001, 0.00001, 1.2E-7] }, 3]"); |
689 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
690 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
691 | assertEquals("1", tokenizer.getToken()); |
692 | { |
693 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
694 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
695 | assertEquals("2", tokenizer.getToken()); |
696 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
697 | assertEquals("3", tokenizer.getToken()); |
698 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
699 | assertEquals("4", tokenizer.getToken()); |
700 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
701 | assertEquals("5", tokenizer.getToken()); |
702 | } |
703 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
704 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
705 | assertEquals("key", tokenizer.getToken()); |
706 | { |
707 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
708 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
709 | assertEquals("true", tokenizer.getToken()); |
710 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
711 | assertEquals("false", tokenizer.getToken()); |
712 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
713 | assertEquals("\"hoge\"", tokenizer.getToken()); |
714 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
715 | assertEquals("array", tokenizer.getToken()); |
716 | { |
717 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
718 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
719 | assertEquals("0.001", tokenizer.getToken()); |
720 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
721 | assertEquals("0.00001", tokenizer.getToken()); |
722 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
723 | assertEquals("1.2E-7", tokenizer.getToken()); |
724 | } |
725 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
726 | } |
727 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
728 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
729 | assertEquals("3", tokenizer.getToken()); |
730 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
731 | } |
732 | |
733 | public void test22_stringSingleEnquote() throws Exception { |
734 | JSONTokenizer tokenizer = new JSONTokenizer("{\'a'aa\':\"?????\"}"); |
735 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
736 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
737 | assertEquals("a'aa", tokenizer.getToken()); |
738 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
739 | assertEquals("\"?????\"", tokenizer.getToken()); |
740 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
741 | } |
742 | |
743 | /** |
744 | * |
745 | * {@.en } |
746 | * |
747 | * <br /> |
748 | * |
749 | * {@.ja ?????????key????String??????????.} |
750 | * |
751 | * @throws Exception |
752 | */ |
753 | public void test23_keyMustBeString() throws Exception { |
754 | JSONTokenizer tokenizer = new JSONTokenizer("{aaa:123}"); |
755 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
756 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
757 | assertEquals("aaa", tokenizer.getToken()); |
758 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
759 | assertEquals("123", tokenizer.getToken()); |
760 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
761 | } |
762 | |
763 | public void test24_keyvalue_nestOnlyNestObject() throws Exception { |
764 | JSONTokenizer tokenizer = new JSONTokenizer("{\"bbb\":{\"b1\":true}}"); |
765 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
766 | assertEquals("{\"bbb\":{\"b1\":true}}", tokenizer.getToken()); |
767 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
768 | assertEquals("bbb", tokenizer.getToken()); |
769 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
770 | assertEquals("{\"b1\":true}}", tokenizer.getToken()); |
771 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
772 | assertEquals("b1", tokenizer.getToken()); |
773 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
774 | assertEquals("true", tokenizer.getToken()); |
775 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
776 | } |
777 | |
778 | public void test25_keyvalue_nestOnlyNestObject2() throws Exception { |
779 | JSONTokenizer tokenizer = new JSONTokenizer( |
780 | "{\"bbb\":{\"b1\":true}, \"vvv\":null}"); |
781 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
782 | assertEquals("{\"bbb\":{\"b1\":true}, \"vvv\":null}", tokenizer |
783 | .getToken()); |
784 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
785 | assertEquals("bbb", tokenizer.getToken()); |
786 | { |
787 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
788 | assertEquals("{\"b1\":true}, \"vvv\":null}", tokenizer.getToken()); |
789 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
790 | assertEquals("b1", tokenizer.getToken()); |
791 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
792 | assertEquals("true", tokenizer.getToken()); |
793 | } |
794 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
795 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
796 | assertEquals("vvv", tokenizer.getToken()); |
797 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
798 | assertEquals("null", tokenizer.getToken()); |
799 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
800 | } |
801 | |
802 | public void test26_keyvalue_deepNest1() throws Exception { |
803 | final String json = "{\"bbb\":{\"dates\":{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}"; |
804 | JSONTokenizer tokenizer = new JSONTokenizer(json); |
805 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
806 | assertEquals(json, tokenizer.getToken()); |
807 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
808 | assertEquals("bbb", tokenizer.getToken()); |
809 | { |
810 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
811 | assertEquals( |
812 | "{\"dates\":{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}", |
813 | tokenizer.getToken()); |
814 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
815 | assertEquals("dates", tokenizer.getToken()); |
816 | { |
817 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
818 | assertEquals( |
819 | "{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}", |
820 | tokenizer.getToken()); |
821 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
822 | assertEquals("from", tokenizer.getToken()); |
823 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
824 | assertEquals("20090101", tokenizer.getToken()); |
825 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
826 | assertEquals("to", tokenizer.getToken()); |
827 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
828 | assertEquals("20091231", tokenizer.getToken()); |
829 | } |
830 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
831 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
832 | assertEquals("b1", tokenizer.getToken()); |
833 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
834 | assertEquals("true", tokenizer.getToken()); |
835 | } |
836 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
837 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
838 | assertEquals("vvv", tokenizer.getToken()); |
839 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
840 | assertEquals("null", tokenizer.getToken()); |
841 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
842 | } |
843 | |
844 | public void test27_keyvalue_nest2() throws Exception { |
845 | JSONTokenizer tokenizer = new JSONTokenizer( |
846 | "{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\"}"); |
847 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
848 | assertEquals("{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\"}", |
849 | tokenizer.getToken()); |
850 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
851 | assertEquals("aaa", tokenizer.getToken()); |
852 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
853 | assertEquals("123", tokenizer.getToken()); |
854 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
855 | assertEquals("bbb", tokenizer.getToken()); |
856 | { |
857 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
858 | assertEquals("{\"b1\":true},\t\"ccc\":\"fuga\"}", tokenizer |
859 | .getToken()); |
860 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
861 | assertEquals("b1", tokenizer.getToken()); |
862 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
863 | assertEquals("true", tokenizer.getToken()); |
864 | } |
865 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
866 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
867 | assertEquals("ccc", tokenizer.getToken()); |
868 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
869 | assertEquals("\"fuga\"", tokenizer.getToken()); |
870 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
871 | } |
872 | |
873 | } |
Snippet is not live.
Travelled to 12 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt
No comments. add comment
Snippet ID: | #2000398 |
Snippet name: | JSON tokenizer by T2 project |
Eternal ID of this version: | #2000398/1 |
Text MD5: | 257a5e3c94fdfb04ef521088840bdd00 |
Author: | stefan |
Category: | javax |
Type: | New Tinybrain snippet |
Public (visible to everyone): | Yes |
Archived (hidden from active list): | No |
Created/modified: | 2015-06-28 18:11:53 |
Source code size: | 30606 bytes / 873 lines |
Pitched / IR pitched: | No / Yes |
Views / Downloads: | 637 / 151 |
Referenced in: | [show references] |