1 | /* |
2 | * Copyright 2008-2010 the T2 Project ant the Others. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | //package org.t2framework.commons.util; |
17 | |
18 | /** |
19 | * |
20 | * <#if locale="en"> |
21 | * <p> |
22 | * JSON tokenizer.It makes tokens for parsing json data. |
23 | * </p> |
24 | * <#else> |
25 | * <p> |
26 | * |
27 | * </p> |
28 | * </#if> |
29 | * |
30 | * @author shot |
31 | * |
32 | * TODO : unicode, quotation("), reverse solidus(\), solidus(/),
|
33 | * |
34 | */ |
35 | public class JSONTokenizer {
|
36 | |
37 | public static final char COMMA = ','; |
38 | |
39 | public static final char COLON = ':'; |
40 | |
41 | public static final char SINGLE_QUOTE = '\''; |
42 | |
43 | public static final char DOUBLE_QUOTE = '\"'; |
44 | |
45 | public static final String NULL_STRING = "null"; |
46 | |
47 | public static final String TRUE_STRING = "true"; |
48 | |
49 | public static final String FALSE_STRING = "false"; |
50 | |
51 | public static final char START_BRACKET = '['; |
52 | |
53 | public static final char END_BRACKET = ']'; |
54 | |
55 | public static final char START_BRACE = '{';
|
56 | |
57 | public static final char END_BRACE = '}'; |
58 | |
59 | // public static final char UTF8_BOM = 0xFEFF; |
60 | |
61 | protected final String orgString; |
62 | |
63 | protected final int orgLength; |
64 | |
65 | protected String tokenString; |
66 | |
67 | protected int pos; |
68 | |
69 | protected int nextPos; |
70 | |
71 | protected TokenType type = TokenType.EOF; |
72 | |
73 | public JSONTokenizer(String s) {
|
74 | // TODO : check grammer first. |
75 | this.orgString = (s != null) ? s : "null"; |
76 | this.orgLength = this.orgString.length(); |
77 | this.tokenString = this.orgString; |
78 | prepare(0); |
79 | } |
80 | |
81 | protected void prepare(int i) {
|
82 | if (i < orgLength) {
|
83 | char ch = orgString.charAt(i); |
84 | if (ch == START_BRACE) {
|
85 | type = TokenType.START_BRACE; |
86 | } else if (ch == END_BRACE) {
|
87 | type = TokenType.END_BRACE; |
88 | } else if (ch == START_BRACKET) {
|
89 | type = TokenType.START_BRACKET; |
90 | } else if (ch == END_BRACKET) {
|
91 | type = TokenType.END_BRACKET; |
92 | } |
93 | this.pos = i; |
94 | this.nextPos = i + 1; |
95 | } else {
|
96 | type = TokenType.EOF; |
97 | } |
98 | } |
99 | |
100 | public TokenType nextToken() {
|
101 | if (type == TokenType.EOF) {
|
102 | return TokenType.EOF; |
103 | } |
104 | String s = this.tokenString; |
105 | TokenType retType = TokenType.EOF; |
106 | boolean key = true; |
107 | for (int i = this.nextPos; i < this.orgLength; i++) {
|
108 | char ch = this.orgString.charAt(i); |
109 | if (isIgnorable(ch)) {
|
110 | continue; |
111 | } |
112 | if (ch == START_BRACE) {
|
113 | s = getElement(nextPos, this.orgLength - 1); |
114 | retType = TokenType.START_BRACE; |
115 | prepare(i); |
116 | key = true; |
117 | break; |
118 | } else if (ch == START_BRACKET) {
|
119 | s = getElement(nextPos, this.orgLength - 1); |
120 | retType = TokenType.START_BRACKET; |
121 | prepare(i); |
122 | key = true; |
123 | break; |
124 | } else if (ch == COLON) {
|
125 | if (i == this.orgLength - 1) {
|
126 | throw new IllegalStateException(); |
127 | } |
128 | s = getElement(nextPos, i - 1); |
129 | this.type = retType = TokenType.COLON; |
130 | prepare(i); |
131 | key = true; |
132 | break; |
133 | } else if (ch == COMMA) {
|
134 | if (i == this.orgLength - 1) {
|
135 | throw new IllegalArgumentException(); |
136 | } |
137 | s = getElement(nextPos, i - 1); |
138 | this.type = retType = (isObjectOrArrayEnd(i - 1)) ? TokenType.END_COMMA |
139 | : TokenType.COMMA; |
140 | prepare(i); |
141 | key = false; |
142 | break; |
143 | } else if (ch == END_BRACKET) {
|
144 | this.type = (i == this.orgLength - 1) ? TokenType.EOF |
145 | : TokenType.END_BRACKET; |
146 | retType = TokenType.END_BRACKET; |
147 | s = getElement(nextPos, i - 1); |
148 | prepare(i); |
149 | key = false; |
150 | break; |
151 | } else if (ch == END_BRACE) {
|
152 | this.type = (i == this.orgLength - 1) ? TokenType.EOF |
153 | : TokenType.END_BRACE; |
154 | retType = TokenType.END_BRACE; |
155 | s = getElement(this.nextPos, i - 1); |
156 | prepare(i); |
157 | key = false; |
158 | break; |
159 | } |
160 | } |
161 | s = removeIgnorable(s); |
162 | this.tokenString = (key) ? unquote(s) : s; |
163 | return retType; |
164 | } |
165 | |
166 | protected boolean isObjectOrArrayEnd(int pos) {
|
167 | for (int i = pos; 0 < i; i--) {
|
168 | char c = this.orgString.charAt(i); |
169 | if (isIgnorable(c)) {
|
170 | continue; |
171 | } |
172 | if (c == END_BRACE || c == END_BRACKET) {
|
173 | return true; |
174 | } else {
|
175 | return false; |
176 | } |
177 | } |
178 | return false; |
179 | } |
180 | |
181 | /** |
182 | * <#if locale="en"> |
183 | * <p> |
184 | * TODO refactor. |
185 | * |
186 | * </p> |
187 | * <#else> |
188 | * <p> |
189 | * |
190 | * </p> |
191 | * </#if> |
192 | * |
193 | * @param s |
194 | * @return |
195 | */ |
196 | protected String removeIgnorable(String s) {
|
197 | if (isEmpty(s)) {
|
198 | return s; |
199 | } |
200 | for (int pos1 = 0; pos1 < s.length(); pos1++) {
|
201 | if (isIgnorable(s.charAt(pos1)) == false) {
|
202 | return s.substring(pos1); |
203 | } |
204 | } |
205 | return ""; |
206 | } |
207 | |
208 | public static boolean isEmpty(String text) {
|
209 | return text == null || text.length() == 0; |
210 | } |
211 | |
212 | protected String getElement(final int orgStartPos, final int orgEndPos) {
|
213 | int startPos = orgStartPos; |
214 | for (; startPos < orgEndPos; startPos++) {
|
215 | char ch = this.orgString.charAt(startPos); |
216 | if (isIgnorable(ch) == false) {
|
217 | break; |
218 | } |
219 | } |
220 | int endPos = orgEndPos; |
221 | for (; startPos < endPos; endPos--) {
|
222 | char ch = this.orgString.charAt(endPos); |
223 | if (isIgnorable(ch) == false) {
|
224 | break; |
225 | } |
226 | } |
227 | return this.orgString.substring(startPos, endPos + 1); |
228 | } |
229 | |
230 | protected static boolean isIgnorable(char ch) {
|
231 | boolean ret = false; |
232 | switch (ch) {
|
233 | case '\b': |
234 | case '\f': |
235 | case '\n': |
236 | case '\r': |
237 | case '\t': |
238 | case ' ': |
239 | ret = true; |
240 | break; |
241 | default: |
242 | ret = false; |
243 | break; |
244 | } |
245 | return ret; |
246 | } |
247 | |
248 | public static String unquote(String str) {
|
249 | if (str == null || str.length() == 0) {
|
250 | return str; |
251 | } |
252 | if (isQuoted(str, DOUBLE_QUOTE)) {
|
253 | return chopQuote(str); |
254 | } else if (isQuoted(str, SINGLE_QUOTE)) {
|
255 | return chopQuote(str); |
256 | } else {
|
257 | return str; |
258 | } |
259 | } |
260 | |
261 | public static String chopQuote(String str) {
|
262 | return str.substring(1, str.length() - 1); |
263 | } |
264 | |
265 | protected static boolean isQuoted(String str, char quote) {
|
266 | return str.indexOf(quote) == 0 && str.lastIndexOf(quote) > 0; |
267 | } |
268 | |
269 | public String getToken() {
|
270 | return this.tokenString; |
271 | } |
272 | |
273 | public static enum TokenType {
|
274 | /** |
275 | * <#if locale="en"> |
276 | * <p> |
277 | * "{"
|
278 | * </p> |
279 | * <#else> |
280 | * <p> |
281 | * |
282 | * </p> |
283 | * </#if> |
284 | */ |
285 | START_BRACE {
|
286 | public String toString() {
|
287 | return "{";
|
288 | } |
289 | }, |
290 | |
291 | /** |
292 | * <#if locale="en"> |
293 | * <p> |
294 | * "}" |
295 | * </p> |
296 | * <#else> |
297 | * <p> |
298 | * |
299 | * </p> |
300 | * </#if> |
301 | */ |
302 | END_BRACE {
|
303 | public String toString() {
|
304 | return "}"; |
305 | } |
306 | }, |
307 | |
308 | /** |
309 | * <#if locale="en"> |
310 | * <p> |
311 | * "[" |
312 | * </p> |
313 | * <#else> |
314 | * <p> |
315 | * |
316 | * </p> |
317 | * </#if> |
318 | */ |
319 | START_BRACKET {
|
320 | public String toString() {
|
321 | return "["; |
322 | } |
323 | }, |
324 | |
325 | /** |
326 | * <#if locale="en"> |
327 | * <p> |
328 | * "]" |
329 | * </p> |
330 | * <#else> |
331 | * <p> |
332 | * |
333 | * </p> |
334 | * </#if> |
335 | */ |
336 | END_BRACKET {
|
337 | public String toString() {
|
338 | return "]"; |
339 | } |
340 | }, |
341 | |
342 | /** |
343 | * <#if locale="en"> |
344 | * <p> |
345 | * "," |
346 | * </p> |
347 | * <#else> |
348 | * <p> |
349 | * |
350 | * </p> |
351 | * </#if> |
352 | */ |
353 | COMMA {
|
354 | public String toString() {
|
355 | return ","; |
356 | } |
357 | }, |
358 | |
359 | /** |
360 | * <#if locale="en"> |
361 | * <p> |
362 | * ":" |
363 | * </p> |
364 | * <#else> |
365 | * <p> |
366 | * |
367 | * </p> |
368 | * </#if> |
369 | */ |
370 | COLON {
|
371 | public String toString() {
|
372 | return ":"; |
373 | } |
374 | }, |
375 | |
376 | /** |
377 | * <#if locale="en"> |
378 | * <p> |
379 | * "," and it is the end of {} or [].
|
380 | * </p> |
381 | * <#else> |
382 | * <p> |
383 | * |
384 | * </p> |
385 | * </#if> |
386 | */ |
387 | END_COMMA {
|
388 | public String toString() {
|
389 | return ","; |
390 | } |
391 | }, |
392 | |
393 | /** |
394 | * <#if locale="en"> |
395 | * <p> |
396 | * End of file. |
397 | * </p> |
398 | * <#else> |
399 | * <p> |
400 | * |
401 | * </p> |
402 | * </#if> |
403 | */ |
404 | EOF; |
405 | |
406 | } |
407 | |
408 | public TokenType currentTokenType() {
|
409 | return type; |
410 | } |
411 | } |
412 | ------------------------- |
413 | /* |
414 | * Copyright 2008-2009 the T2 Project ant the Others. |
415 | * |
416 | * Licensed under the Apache License, Version 2.0 (the "License"); |
417 | * you may not use this file except in compliance with the License. |
418 | * You may obtain a copy of the License at |
419 | * |
420 | * http://www.apache.org/licenses/LICENSE-2.0 |
421 | * |
422 | * Unless required by applicable law or agreed to in writing, software |
423 | * distributed under the License is distributed on an "AS IS" BASIS, |
424 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
425 | * See the License for the specific language governing permissions and |
426 | * limitations under the License. |
427 | */ |
428 | package org.t2framework.commons.util; |
429 | |
430 | import org.t2framework.commons.util.JSONTokenizer.TokenType; |
431 | |
432 | import junit.framework.TestCase; |
433 | |
434 | public class JSONTokenizerTest extends TestCase {
|
435 | |
436 | public void test1_blank() throws Exception {
|
437 | JSONTokenizer tokenizer = new JSONTokenizer("");
|
438 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
439 | assertEquals("", tokenizer.getToken());
|
440 | } |
441 | |
442 | public void test2_null() throws Exception {
|
443 | JSONTokenizer tokenizer = new JSONTokenizer(null); |
444 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
445 | assertEquals("null", tokenizer.getToken());
|
446 | } |
447 | |
448 | public void test3_keyvalue() throws Exception {
|
449 | JSONTokenizer tokenizer = new JSONTokenizer("{\"aaa\":123}");
|
450 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
451 | assertEquals("{\"aaa\":123}", tokenizer.getToken());
|
452 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
453 | assertEquals("aaa", tokenizer.getToken());
|
454 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
455 | assertEquals("123", tokenizer.getToken());
|
456 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
457 | } |
458 | |
459 | public void test4_array() throws Exception {
|
460 | JSONTokenizer tokenizer = new JSONTokenizer("[\"aaa\",123,\"b\"]");
|
461 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
462 | assertEquals("[\"aaa\",123,\"b\"]", tokenizer.getToken());
|
463 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
464 | assertEquals("\"aaa\"", tokenizer.getToken());
|
465 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
466 | assertEquals("123", tokenizer.getToken());
|
467 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
468 | assertEquals("\"b\"", tokenizer.getToken());
|
469 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
470 | } |
471 | |
472 | public void test5_emptyobject() throws Exception {
|
473 | JSONTokenizer tokenizer = new JSONTokenizer("{}");
|
474 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
475 | assertEquals("{}", tokenizer.getToken());
|
476 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
477 | assertEquals("", tokenizer.getToken());
|
478 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
479 | } |
480 | |
481 | public void test6_emptyarray() throws Exception {
|
482 | JSONTokenizer tokenizer = new JSONTokenizer("[]");
|
483 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
484 | assertEquals("[]", tokenizer.getToken());
|
485 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
486 | assertEquals("", tokenizer.getToken());
|
487 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
488 | } |
489 | |
490 | public void test7_multiplekeyvalue() throws Exception {
|
491 | JSONTokenizer tokenizer = new JSONTokenizer( |
492 | "{\"aaa\":123,\"bbb\":true}");
|
493 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
494 | assertEquals("{\"aaa\":123,\"bbb\":true}", tokenizer.getToken());
|
495 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
496 | assertEquals("aaa", tokenizer.getToken());
|
497 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
498 | assertEquals("123", tokenizer.getToken());
|
499 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
500 | assertEquals("bbb", tokenizer.getToken());
|
501 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
502 | assertEquals("true", tokenizer.getToken());
|
503 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
504 | } |
505 | |
506 | public void test8_keyvaluewithsinglequote() throws Exception {
|
507 | JSONTokenizer tokenizer = new JSONTokenizer("{'aaa':'123'}");
|
508 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
509 | assertEquals("{'aaa':'123'}", tokenizer.getToken());
|
510 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
511 | assertEquals("aaa", tokenizer.getToken());
|
512 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
513 | assertEquals("'123'", tokenizer.getToken());
|
514 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
515 | } |
516 | |
517 | public void test9_blankwithtab() throws Exception {
|
518 | JSONTokenizer tokenizer = new JSONTokenizer("{\t}");
|
519 | assertEquals("{\t}", tokenizer.getToken());
|
520 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
521 | assertEquals("", tokenizer.getToken());
|
522 | } |
523 | |
524 | public void test10_blankwithbackspace() throws Exception {
|
525 | JSONTokenizer tokenizer = new JSONTokenizer("{\b}");
|
526 | assertEquals("{\b}", tokenizer.getToken());
|
527 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
528 | assertEquals("", tokenizer.getToken());
|
529 | } |
530 | |
531 | public void test11_blankwithformfeed() throws Exception {
|
532 | JSONTokenizer tokenizer = new JSONTokenizer("{\f}");
|
533 | assertEquals("{\f}", tokenizer.getToken());
|
534 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
535 | assertEquals("", tokenizer.getToken());
|
536 | } |
537 | |
538 | public void test12_blankwithnewline() throws Exception {
|
539 | JSONTokenizer tokenizer = new JSONTokenizer("{\n}");
|
540 | assertEquals("{\n}", tokenizer.getToken());
|
541 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
542 | assertEquals("", tokenizer.getToken());
|
543 | } |
544 | |
545 | public void test13_blankwithcarriagereturn() throws Exception {
|
546 | JSONTokenizer tokenizer = new JSONTokenizer("{\r}");
|
547 | assertEquals("{\r}", tokenizer.getToken());
|
548 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
549 | assertEquals("", tokenizer.getToken());
|
550 | } |
551 | |
552 | public void test14_keyvalue_nest1() throws Exception {
|
553 | JSONTokenizer tokenizer = new JSONTokenizer( |
554 | "{\"aaa\":123, \"bbb\":{\"b1\":true}}");
|
555 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
556 | assertEquals("{\"aaa\":123, \"bbb\":{\"b1\":true}}", tokenizer
|
557 | .getToken()); |
558 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
559 | assertEquals("aaa", tokenizer.getToken());
|
560 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
561 | assertEquals("123", tokenizer.getToken());
|
562 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
563 | assertEquals("bbb", tokenizer.getToken());
|
564 | {
|
565 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
566 | assertEquals("{\"b1\":true}}", tokenizer.getToken());
|
567 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
568 | assertEquals("b1", tokenizer.getToken());
|
569 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
570 | assertEquals("true", tokenizer.getToken());
|
571 | } |
572 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
573 | } |
574 | |
575 | public void test15_ignorableSpaceShouldIgnoreAtObject() throws Exception {
|
576 | JSONTokenizer tokenizer = new JSONTokenizer("{\"aaa\"\r\t:\n123 }");
|
577 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
578 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
579 | assertEquals("aaa", tokenizer.getToken());
|
580 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
581 | assertEquals("123", tokenizer.getToken());
|
582 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
583 | } |
584 | |
585 | public void test16_ignorableSpaceShouldIgnoreAtArray() throws Exception {
|
586 | JSONTokenizer tokenizer = new JSONTokenizer( |
587 | "[ \"aaa\"\t,123\b,\f\'b\' ]"); |
588 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
589 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
590 | assertEquals("\"aaa\"", tokenizer.getToken());
|
591 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
592 | assertEquals("123", tokenizer.getToken());
|
593 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
594 | assertEquals("\'b\'", tokenizer.getToken());
|
595 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
596 | } |
597 | |
598 | public void test17_blankwithlotsofignorables() throws Exception {
|
599 | JSONTokenizer tokenizer = new JSONTokenizer("{\t\r\n \t}");
|
600 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
601 | assertEquals("", tokenizer.getToken());
|
602 | } |
603 | |
604 | public void test18_keyvalue_nest3_array() throws Exception {
|
605 | JSONTokenizer tokenizer = new JSONTokenizer( |
606 | "{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\", \"array1\":[\"1.1233333333000000000000001\"\r,\b1.1233333333000000000000001, \"3.0\"]}");
|
607 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
608 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
609 | assertEquals("aaa", tokenizer.getToken());
|
610 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
611 | assertEquals("123", tokenizer.getToken());
|
612 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
613 | assertEquals("bbb", tokenizer.getToken());
|
614 | {
|
615 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
616 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
617 | assertEquals("b1", tokenizer.getToken());
|
618 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
619 | assertEquals("true", tokenizer.getToken());
|
620 | } |
621 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
622 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
623 | assertEquals("ccc", tokenizer.getToken());
|
624 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
625 | assertEquals("\"fuga\"", tokenizer.getToken());
|
626 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
627 | assertEquals("array1", tokenizer.getToken());
|
628 | {
|
629 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
630 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
631 | assertEquals("\"1.1233333333000000000000001\"", tokenizer
|
632 | .getToken()); |
633 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
634 | assertEquals("1.1233333333000000000000001", tokenizer.getToken());
|
635 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
636 | assertEquals("\"3.0\"", tokenizer.getToken());
|
637 | } |
638 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
639 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
640 | } |
641 | |
642 | public void test18_stringEnquote() throws Exception {
|
643 | JSONTokenizer tokenizer = new JSONTokenizer("{\"a'aa\":\"?????\"}");
|
644 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
645 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
646 | assertEquals("a'aa", tokenizer.getToken());
|
647 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
648 | assertEquals("\"?????\"", tokenizer.getToken());
|
649 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
650 | } |
651 | |
652 | public void test19_booleanarray() throws Exception {
|
653 | JSONTokenizer tokenizer = new JSONTokenizer("[true, false,true]");
|
654 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
655 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
656 | assertEquals("true", tokenizer.getToken());
|
657 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
658 | assertEquals("false", tokenizer.getToken());
|
659 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
660 | assertEquals("true", tokenizer.getToken());
|
661 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
662 | } |
663 | |
664 | public void test20_nestarray() throws Exception {
|
665 | JSONTokenizer tokenizer = new JSONTokenizer("[1, [2, 3, 4, 5], 3]");
|
666 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
667 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
668 | assertEquals("1", tokenizer.getToken());
|
669 | {
|
670 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
671 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
672 | assertEquals("2", tokenizer.getToken());
|
673 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
674 | assertEquals("3", tokenizer.getToken());
|
675 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
676 | assertEquals("4", tokenizer.getToken());
|
677 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
678 | assertEquals("5", tokenizer.getToken());
|
679 | } |
680 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
681 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
682 | assertEquals("3", tokenizer.getToken());
|
683 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
684 | } |
685 | |
686 | public void test21_nestarrayandobjects() throws Exception {
|
687 | JSONTokenizer tokenizer = new JSONTokenizer( |
688 | "[1, [2, 3, 4, 5], \"key\":{true, false, \"hoge\", \"array\":[0.001, 0.00001, 1.2E-7] }, 3]");
|
689 | assertEquals(TokenType.START_BRACKET, tokenizer.currentTokenType()); |
690 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
691 | assertEquals("1", tokenizer.getToken());
|
692 | {
|
693 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
694 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
695 | assertEquals("2", tokenizer.getToken());
|
696 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
697 | assertEquals("3", tokenizer.getToken());
|
698 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
699 | assertEquals("4", tokenizer.getToken());
|
700 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
701 | assertEquals("5", tokenizer.getToken());
|
702 | } |
703 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
704 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
705 | assertEquals("key", tokenizer.getToken());
|
706 | {
|
707 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
708 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
709 | assertEquals("true", tokenizer.getToken());
|
710 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
711 | assertEquals("false", tokenizer.getToken());
|
712 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
713 | assertEquals("\"hoge\"", tokenizer.getToken());
|
714 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
715 | assertEquals("array", tokenizer.getToken());
|
716 | {
|
717 | assertEquals(TokenType.START_BRACKET, tokenizer.nextToken()); |
718 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
719 | assertEquals("0.001", tokenizer.getToken());
|
720 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
721 | assertEquals("0.00001", tokenizer.getToken());
|
722 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
723 | assertEquals("1.2E-7", tokenizer.getToken());
|
724 | } |
725 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
726 | } |
727 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
728 | assertEquals(TokenType.END_BRACKET, tokenizer.nextToken()); |
729 | assertEquals("3", tokenizer.getToken());
|
730 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
731 | } |
732 | |
733 | public void test22_stringSingleEnquote() throws Exception {
|
734 | JSONTokenizer tokenizer = new JSONTokenizer("{\'a'aa\':\"?????\"}");
|
735 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
736 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
737 | assertEquals("a'aa", tokenizer.getToken());
|
738 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
739 | assertEquals("\"?????\"", tokenizer.getToken());
|
740 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
741 | } |
742 | |
743 | /** |
744 | * |
745 | * {@.en }
|
746 | * |
747 | * <br /> |
748 | * |
749 | * {@.ja ?????????key????String??????????.}
|
750 | * |
751 | * @throws Exception |
752 | */ |
753 | public void test23_keyMustBeString() throws Exception {
|
754 | JSONTokenizer tokenizer = new JSONTokenizer("{aaa:123}");
|
755 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
756 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
757 | assertEquals("aaa", tokenizer.getToken());
|
758 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
759 | assertEquals("123", tokenizer.getToken());
|
760 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
761 | } |
762 | |
763 | public void test24_keyvalue_nestOnlyNestObject() throws Exception {
|
764 | JSONTokenizer tokenizer = new JSONTokenizer("{\"bbb\":{\"b1\":true}}");
|
765 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
766 | assertEquals("{\"bbb\":{\"b1\":true}}", tokenizer.getToken());
|
767 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
768 | assertEquals("bbb", tokenizer.getToken());
|
769 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
770 | assertEquals("{\"b1\":true}}", tokenizer.getToken());
|
771 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
772 | assertEquals("b1", tokenizer.getToken());
|
773 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
774 | assertEquals("true", tokenizer.getToken());
|
775 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
776 | } |
777 | |
778 | public void test25_keyvalue_nestOnlyNestObject2() throws Exception {
|
779 | JSONTokenizer tokenizer = new JSONTokenizer( |
780 | "{\"bbb\":{\"b1\":true}, \"vvv\":null}");
|
781 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
782 | assertEquals("{\"bbb\":{\"b1\":true}, \"vvv\":null}", tokenizer
|
783 | .getToken()); |
784 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
785 | assertEquals("bbb", tokenizer.getToken());
|
786 | {
|
787 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
788 | assertEquals("{\"b1\":true}, \"vvv\":null}", tokenizer.getToken());
|
789 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
790 | assertEquals("b1", tokenizer.getToken());
|
791 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
792 | assertEquals("true", tokenizer.getToken());
|
793 | } |
794 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
795 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
796 | assertEquals("vvv", tokenizer.getToken());
|
797 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
798 | assertEquals("null", tokenizer.getToken());
|
799 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
800 | } |
801 | |
802 | public void test26_keyvalue_deepNest1() throws Exception {
|
803 | final String json = "{\"bbb\":{\"dates\":{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}";
|
804 | JSONTokenizer tokenizer = new JSONTokenizer(json); |
805 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
806 | assertEquals(json, tokenizer.getToken()); |
807 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
808 | assertEquals("bbb", tokenizer.getToken());
|
809 | {
|
810 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
811 | assertEquals( |
812 | "{\"dates\":{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}",
|
813 | tokenizer.getToken()); |
814 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
815 | assertEquals("dates", tokenizer.getToken());
|
816 | {
|
817 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
818 | assertEquals( |
819 | "{\"from\":20090101,\n \"to\":20091231},\t\"b1\":true}, \"vvv\":null}",
|
820 | tokenizer.getToken()); |
821 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
822 | assertEquals("from", tokenizer.getToken());
|
823 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
824 | assertEquals("20090101", tokenizer.getToken());
|
825 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
826 | assertEquals("to", tokenizer.getToken());
|
827 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
828 | assertEquals("20091231", tokenizer.getToken());
|
829 | } |
830 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
831 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
832 | assertEquals("b1", tokenizer.getToken());
|
833 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
834 | assertEquals("true", tokenizer.getToken());
|
835 | } |
836 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
837 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
838 | assertEquals("vvv", tokenizer.getToken());
|
839 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
840 | assertEquals("null", tokenizer.getToken());
|
841 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
842 | } |
843 | |
844 | public void test27_keyvalue_nest2() throws Exception {
|
845 | JSONTokenizer tokenizer = new JSONTokenizer( |
846 | "{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\"}");
|
847 | assertEquals(TokenType.START_BRACE, tokenizer.currentTokenType()); |
848 | assertEquals("{\"aaa\":123, \"bbb\":{\"b1\":true},\t\"ccc\":\"fuga\"}",
|
849 | tokenizer.getToken()); |
850 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
851 | assertEquals("aaa", tokenizer.getToken());
|
852 | assertEquals(TokenType.COMMA, tokenizer.nextToken()); |
853 | assertEquals("123", tokenizer.getToken());
|
854 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
855 | assertEquals("bbb", tokenizer.getToken());
|
856 | {
|
857 | assertEquals(TokenType.START_BRACE, tokenizer.nextToken()); |
858 | assertEquals("{\"b1\":true},\t\"ccc\":\"fuga\"}", tokenizer
|
859 | .getToken()); |
860 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
861 | assertEquals("b1", tokenizer.getToken());
|
862 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
863 | assertEquals("true", tokenizer.getToken());
|
864 | } |
865 | assertEquals(TokenType.END_COMMA, tokenizer.nextToken()); |
866 | assertEquals(TokenType.COLON, tokenizer.nextToken()); |
867 | assertEquals("ccc", tokenizer.getToken());
|
868 | assertEquals(TokenType.END_BRACE, tokenizer.nextToken()); |
869 | assertEquals("\"fuga\"", tokenizer.getToken());
|
870 | assertEquals(TokenType.EOF, tokenizer.nextToken()); |
871 | } |
872 | |
873 | } |
Snippet is not live.
Travelled to 12 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt
No comments. add comment
| Snippet ID: | #2000398 |
| Snippet name: | JSON tokenizer by T2 project |
| Eternal ID of this version: | #2000398/1 |
| Text MD5: | 257a5e3c94fdfb04ef521088840bdd00 |
| Author: | stefan |
| Category: | javax |
| Type: | New Tinybrain snippet |
| Public (visible to everyone): | Yes |
| Archived (hidden from active list): | No |
| Created/modified: | 2015-06-28 18:11:53 |
| Source code size: | 30606 bytes / 873 lines |
| Pitched / IR pitched: | No / Yes |
| Views / Downloads: | 637 / 151 |
| Referenced in: | [show references] |