Try to make a JavaX tokenizer for RSTA [dev.] [1014799]

!7

sS snippetID;
sS savedText, remoteText;
static EditorFrame e;

import org.fife.ui.rsyntaxtextarea.*;

sclass JavaXTokenizer extends AbstractTokenMaker {
  public TokenMap getWordsToHighlight() {
	ret new TokenMap;
  }

  int currentTokenStart, currentTokenType, startTokenType;
  
  /**
 * Returns a list of tokens representing the given text.
 *
 * @param text The text to break into tokens.
 * @param startTokenType The token with which to start tokenizing.
 * @param startOffset The offset at which the line of tokens begins.
 * @return A linked list of tokens representing <code>text</code>.
 */
  public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
     resetTokenList();

   char[] array = text.array;
   int offset = text.offset;
   int count = text.count;
   int end = offset + count;

   // Token starting offsets are always of the form:
   // 'startOffset + (currentTokenStart-offset)', but since startOffset and
   // offset are constant, tokens' starting positions become:
   // 'newStartOffset+currentTokenStart'.
   int newStartOffset = startOffset - offset;

   currentTokenStart = offset;
   currentTokenType  = startTokenType;

   for (int i=offset; i<end; i++) {

      char c = array[i];

      switch (currentTokenType) {

         case Token.NULL:

            currentTokenStart = i;   // Starting a new token here.

            switch (c) {

               case ' ':
               case '\t':
                  currentTokenType = Token.WHITESPACE;
                  break;

               case '"':
                  currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
                  break;

               case '#':
                  currentTokenType = Token.COMMENT_EOL;
                  break;

               default:
                  if (RSyntaxUtilities.isDigit(c)) {
                     currentTokenType = Token.LITERAL_NUMBER_DECIMAL_INT;
                     break;
                  }
                  else if (RSyntaxUtilities.isLetter(c) || c=='/' || c=='_') {
                     currentTokenType = Token.IDENTIFIER;
                     break;
                  }
                  
                  // Anything not currently handled - mark as an identifier
                  currentTokenType = Token.IDENTIFIER;
                  break;

            } // End of switch (c).

            break;

         case Token.WHITESPACE:

            switch (c) {

               case ' ':
               case '\t':
                  break;   // Still whitespace.

               case '"':
                  addToken(text, currentTokenStart,i-1, Token.WHITESPACE, newStartOffset+currentTokenStart);
                  currentTokenStart = i;
                  currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
                  break;

               case '#':
                  addToken(text, currentTokenStart,i-1, Token.WHITESPACE, newStartOffset+currentTokenStart);
                  currentTokenStart = i;
                  currentTokenType = Token.COMMENT_EOL;
                  break;

               default:   // Add the whitespace token and start anew.

                  addToken(text, currentTokenStart,i-1, Token.WHITESPACE, newStartOffset+currentTokenStart);
                  currentTokenStart = i;

                  if (RSyntaxUtilities.isDigit(c)) {
                     currentTokenType = Token.LITERAL_NUMBER_DECIMAL_INT;
                     break;
                  }
                  else if (RSyntaxUtilities.isLetter(c) || c=='/' || c=='_') {
                     currentTokenType = Token.IDENTIFIER;
                     break;
                  }

                  // Anything not currently handled - mark as identifier
                  currentTokenType = Token.IDENTIFIER;

            } // End of switch (c).

            break;

         default: // Should never happen
         case Token.IDENTIFIER:

            switch (c) {

               case ' ':
               case '\t':
                  addToken(text, currentTokenStart,i-1, Token.IDENTIFIER, newStartOffset+currentTokenStart);
                  currentTokenStart = i;
                  currentTokenType = Token.WHITESPACE;
                  break;

               case '"':
                  addToken(text, currentTokenStart,i-1, Token.IDENTIFIER, newStartOffset+currentTokenStart);
                  currentTokenStart = i;
                  currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
                  break;

               default:
                  if (RSyntaxUtilities.isLetterOrDigit(c) || c=='/' || c=='_') {
                     break;   // Still an identifier of some type.
                  }
                  // Otherwise, we're still an identifier (?).

            } // End of switch (c).

            break;

         case Token.LITERAL_NUMBER_DECIMAL_INT:

            switch (c) {

               case ' ':
               case '\t':
                  addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_DECIMAL_INT, newStartOffset+currentTokenStart);
                  currentTokenStart = i;
                  currentTokenType = Token.WHITESPACE;
                  break;

               case '"':
                  addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_DECIMAL_INT, newStartOffset+currentTokenStart);
                  currentTokenStart = i;
                  currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
                  break;

               default:

                  if (RSyntaxUtilities.isDigit(c)) {
                     break;   // Still a literal number.
                  }

                  // Otherwise, remember this was a number and start over.
                  addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_DECIMAL_INT, newStartOffset+currentTokenStart);
                  i--;
                  currentTokenType = Token.NULL;

            } // End of switch (c).

            break;

         case Token.COMMENT_EOL:
            i = end - 1;
            addToken(text, currentTokenStart,i, currentTokenType, newStartOffset+currentTokenStart);
            // We need to set token type to null so at the bottom we don't add one more token.
            currentTokenType = Token.NULL;
            break;

         case Token.LITERAL_STRING_DOUBLE_QUOTE:
            if (c=='"') {
               addToken(text, currentTokenStart,i, Token.LITERAL_STRING_DOUBLE_QUOTE, newStartOffset+currentTokenStart);
               currentTokenType = Token.NULL;
            }
            break;

      } // End of switch (currentTokenType).

   } // End of for (int i=offset; i<end; i++).

   switch (currentTokenType) {

      // Remember what token type to begin the next line with.
      case Token.LITERAL_STRING_DOUBLE_QUOTE:
         addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
         break;

      // Do nothing if everything was okay.
      case Token.NULL:
         addNullToken();
         break;

      // All other token types don't continue to the next line...
      default:
         addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
         addNullToken();

   }

   // Return the first token in our linked list.
   return firstToken;

  }
}


p-substance {
  autoRestart();
  load('savedText);
  
  e = new EditorFrame;
  e.getDocument().setSyntaxStyle(new JavaXTokenizer);
  e._setFont(deriveFont(e._getFont(), 16));
  e.setText(savedText);
  showFrame(e);
  frameIcon(e, #1004655);
  
  addMenu(e, "New", "New text", f newText);

  addMenu(e, "Load", "Load snippet...", r {
    selectSnippetID_v2(vf1(f _loadSnippet));
  });
  
  addMenu(e, "Save", "Save snippet", r { saveSnippetThen(null) });
  
  addMenu(e, "Transpile",
    "Save & Quick Transpile", r { transpile(false) },
    "Save & Medium Transpile", r { transpile(true) });
  addMenu(e, "Activate", "Butter-Reload", f _butterReload);
  
  S snippetID = cast readPersistentVar('snippetID);
  if (snippetID != null) _loadSnippet(snippetID);

  hideConsole();
  
  awtCalcEvery(e, 1000, f update);
  onFrameDeactivated(e, f fullUpdate);
}

svoid update { update(windowActive(e)); }
svoid fullUpdate { update(true); }
svoid update(bool full) {
  S text = e.getText();
  setAndSaveIfChanged(savedText := text);
    
  if (!full) ret;
  
  Pair<Int, S> p = testBracketHygieneExt(text);
  S status = "";
  if (remoteText != null && neq(remoteText, text))
    status = "Changes. ";
  if (p == null)
    status += "Hygienic";
  else {
    status += p.b;
    //setCaretPosition(ta, p.a);
  }
  silentStatus(trim(status));
}

svoid status(S s) {
  silentStatus(infoBoxAndReturn(s));
}

svoid silentStatus(S s) {
  e.setStatus(s);
}

svoid saveSnippetThen(final Runnable next) {
  if (empty(snippetID)) ret;
  fS text = getText(e.textArea);
  status("Saving...");
  thread {
    fS page = editSnippet(snippetID, text);
    remoteText = text; // TODO: check save result
    status("Saved snippet: " + page);
    awtCallF(next);
  }
}

svoid _butterReload() {
  time { butterReload(snippetID); }
  infoBox("Reloaded in " + lastTiming_format100ms());
}

svoid transpile(final bool medium) {
  saveSnippetThen(r { _transpile(medium) });
}

svoid _transpile(final bool medium) {
  status("Transpiling...");
  //jdisable(btnEleuReload);
  
  thread "Transpiling" {
    try {
      final Pair<Bool, S> p = transpileOnServer(snippetID, medium ? "medium" : "quick");
      awt {
        if (p.a)
          status("Transpiled OK!");
        else {
          status("Transpilation error. " + p.b);
          showText("Transpilation Error", p.b);
        }
      }
    } catch print e {
      status("Transpilation problem. " + e);
    }
    //jenable(btnEleuReload);
  }
}

svoid _loadSnippet(S snippetID) {
  e.textArea.setText(remoteText = loadSnippet(snippetID));
  setAndSave('snippetID, snippetID);
  frameTitle(e, snippetID);
}

svoid newText {
  setAndSave(snippetID := null);
  remoteText = null;
  S lastText = e.getText();
  if (nempty(lastText)) logQuotedWithDate("saved-texts.log", lastText);
  e.setText("");
  frameTitle(e, programTitle());
}

svoid cleanMeUp {
  setAndSaveIfChanged(savedText := e.getText());
}

Travelled to 13 computer(s): aoiabmzegqzx, bhatertpkbcr, cbybwowwnfue, cfunsshuasjs, gwrvuhgaqvyk, ishqpsrjomds, lpdgvwnxivlt, mqqgnosmbjvj, pyentgdyhuwx, pzhvpgtvlbxg, tslmcundralx, tvejysmllsmz, vouqrxazstgt

Snippet ID:	#1014799
Snippet name:	Try to make a JavaX tokenizer for RSTA [dev.]
Eternal ID of this version:	#1014799/8
Text MD5:	eea7eac202bfa483846d944cd328e6b1
Transpilation MD5:	c470766a357981dc0b56c271a1aa5eca
Author:	stefan
Category:	javax / gui
Type:	JavaX source code (desktop)
Public (visible to everyone):	Yes
Archived (hidden from active list):	No
Created/modified:	2018-05-02 21:55:17
Source code size:	10363 bytes / 351 lines
Pitched / IR pitched:	No / No
Views / Downloads:	304 / 807
Version history:	7 change(s)
Referenced in:	[show references]

< > BotCompany Repo | #1014799 // Try to make a JavaX tokenizer for RSTA [dev.]

JavaX source code (desktop) [tags: use-pretranspiled] - run with: x30.jar

Author comment

1	!7
2
3	sS snippetID;
4	sS savedText, remoteText;
5	static EditorFrame e;
6
7	import org.fife.ui.rsyntaxtextarea.*;
8
9	sclass JavaXTokenizer extends AbstractTokenMaker {
10	public TokenMap getWordsToHighlight() {
11	ret new TokenMap;
12	}
13
14	int currentTokenStart, currentTokenType, startTokenType;
15
16	/**
17	* Returns a list of tokens representing the given text.
18	*
19	* @param text The text to break into tokens.
20	* @param startTokenType The token with which to start tokenizing.
21	* @param startOffset The offset at which the line of tokens begins.
22	* @return A linked list of tokens representing <code>text</code>.
23	*/
24	public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
25	resetTokenList();
26
27	char[] array = text.array;
28	int offset = text.offset;
29	int count = text.count;
30	int end = offset + count;
31
32	// Token starting offsets are always of the form:
33	// 'startOffset + (currentTokenStart-offset)', but since startOffset and
34	// offset are constant, tokens' starting positions become:
35	// 'newStartOffset+currentTokenStart'.
36	int newStartOffset = startOffset - offset;
37
38	currentTokenStart = offset;
39	currentTokenType = startTokenType;
40
41	for (int i=offset; i<end; i++) {
42
43	char c = array[i];
44
45	switch (currentTokenType) {
46
47	case Token.NULL:
48
49	currentTokenStart = i; // Starting a new token here.
50
51	switch (c) {
52
53	case ' ':
54	case '\t':
55	currentTokenType = Token.WHITESPACE;
56	break;
57
58	case '"':
59	currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
60	break;
61
62	case '#':
63	currentTokenType = Token.COMMENT_EOL;
64	break;
65
66	default:
67	if (RSyntaxUtilities.isDigit(c)) {
68	currentTokenType = Token.LITERAL_NUMBER_DECIMAL_INT;
69	break;
70	}
71	else if (RSyntaxUtilities.isLetter(c) \|\| c=='/' \|\| c=='_') {
72	currentTokenType = Token.IDENTIFIER;
73	break;
74	}
75
76	// Anything not currently handled - mark as an identifier
77	currentTokenType = Token.IDENTIFIER;
78	break;
79
80	} // End of switch (c).
81
82	break;
83
84	case Token.WHITESPACE:
85
86	switch (c) {
87
88	case ' ':
89	case '\t':
90	break; // Still whitespace.
91
92	case '"':
93	addToken(text, currentTokenStart,i-1, Token.WHITESPACE, newStartOffset+currentTokenStart);
94	currentTokenStart = i;
95	currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
96	break;
97
98	case '#':
99	addToken(text, currentTokenStart,i-1, Token.WHITESPACE, newStartOffset+currentTokenStart);
100	currentTokenStart = i;
101	currentTokenType = Token.COMMENT_EOL;
102	break;
103
104	default: // Add the whitespace token and start anew.
105
106	addToken(text, currentTokenStart,i-1, Token.WHITESPACE, newStartOffset+currentTokenStart);
107	currentTokenStart = i;
108
109	if (RSyntaxUtilities.isDigit(c)) {
110	currentTokenType = Token.LITERAL_NUMBER_DECIMAL_INT;
111	break;
112	}
113	else if (RSyntaxUtilities.isLetter(c) \|\| c=='/' \|\| c=='_') {
114	currentTokenType = Token.IDENTIFIER;
115	break;
116	}
117
118	// Anything not currently handled - mark as identifier
119	currentTokenType = Token.IDENTIFIER;
120
121	} // End of switch (c).
122
123	break;
124
125	default: // Should never happen
126	case Token.IDENTIFIER:
127
128	switch (c) {
129
130	case ' ':
131	case '\t':
132	addToken(text, currentTokenStart,i-1, Token.IDENTIFIER, newStartOffset+currentTokenStart);
133	currentTokenStart = i;
134	currentTokenType = Token.WHITESPACE;
135	break;
136
137	case '"':
138	addToken(text, currentTokenStart,i-1, Token.IDENTIFIER, newStartOffset+currentTokenStart);
139	currentTokenStart = i;
140	currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
141	break;
142
143	default:
144	if (RSyntaxUtilities.isLetterOrDigit(c) \|\| c=='/' \|\| c=='_') {
145	break; // Still an identifier of some type.
146	}
147	// Otherwise, we're still an identifier (?).
148
149	} // End of switch (c).
150
151	break;
152
153	case Token.LITERAL_NUMBER_DECIMAL_INT:
154
155	switch (c) {
156
157	case ' ':
158	case '\t':
159	addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_DECIMAL_INT, newStartOffset+currentTokenStart);
160	currentTokenStart = i;
161	currentTokenType = Token.WHITESPACE;
162	break;
163
164	case '"':
165	addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_DECIMAL_INT, newStartOffset+currentTokenStart);
166	currentTokenStart = i;
167	currentTokenType = Token.LITERAL_STRING_DOUBLE_QUOTE;
168	break;
169
170	default:
171
172	if (RSyntaxUtilities.isDigit(c)) {
173	break; // Still a literal number.
174	}
175
176	// Otherwise, remember this was a number and start over.
177	addToken(text, currentTokenStart,i-1, Token.LITERAL_NUMBER_DECIMAL_INT, newStartOffset+currentTokenStart);
178	i--;
179	currentTokenType = Token.NULL;
180
181	} // End of switch (c).
182
183	break;
184
185	case Token.COMMENT_EOL:
186	i = end - 1;
187	addToken(text, currentTokenStart,i, currentTokenType, newStartOffset+currentTokenStart);
188	// We need to set token type to null so at the bottom we don't add one more token.
189	currentTokenType = Token.NULL;
190	break;
191
192	case Token.LITERAL_STRING_DOUBLE_QUOTE:
193	if (c=='"') {
194	addToken(text, currentTokenStart,i, Token.LITERAL_STRING_DOUBLE_QUOTE, newStartOffset+currentTokenStart);
195	currentTokenType = Token.NULL;
196	}
197	break;
198
199	} // End of switch (currentTokenType).
200
201	} // End of for (int i=offset; i<end; i++).
202
203	switch (currentTokenType) {
204
205	// Remember what token type to begin the next line with.
206	case Token.LITERAL_STRING_DOUBLE_QUOTE:
207	addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
208	break;
209
210	// Do nothing if everything was okay.
211	case Token.NULL:
212	addNullToken();
213	break;
214
215	// All other token types don't continue to the next line...
216	default:
217	addToken(text, currentTokenStart,end-1, currentTokenType, newStartOffset+currentTokenStart);
218	addNullToken();
219
220	}
221
222	// Return the first token in our linked list.
223	return firstToken;
224
225	}
226	}
227
228
229	p-substance {
230	autoRestart();
231	load('savedText);
232
233	e = new EditorFrame;
234	e.getDocument().setSyntaxStyle(new JavaXTokenizer);
235	e._setFont(deriveFont(e._getFont(), 16));
236	e.setText(savedText);
237	showFrame(e);
238	frameIcon(e, #1004655);
239
240	addMenu(e, "New", "New text", f newText);
241
242	addMenu(e, "Load", "Load snippet...", r {
243	selectSnippetID_v2(vf1(f _loadSnippet));
244	});
245
246	addMenu(e, "Save", "Save snippet", r { saveSnippetThen(null) });
247
248	addMenu(e, "Transpile",
249	"Save & Quick Transpile", r { transpile(false) },
250	"Save & Medium Transpile", r { transpile(true) });
251	addMenu(e, "Activate", "Butter-Reload", f _butterReload);
252
253	S snippetID = cast readPersistentVar('snippetID);
254	if (snippetID != null) _loadSnippet(snippetID);
255
256	hideConsole();
257
258	awtCalcEvery(e, 1000, f update);
259	onFrameDeactivated(e, f fullUpdate);
260	}
261
262	svoid update { update(windowActive(e)); }
263	svoid fullUpdate { update(true); }
264	svoid update(bool full) {
265	S text = e.getText();
266	setAndSaveIfChanged(savedText := text);
267
268	if (!full) ret;
269
270	Pair<Int, S> p = testBracketHygieneExt(text);
271	S status = "";
272	if (remoteText != null && neq(remoteText, text))
273	status = "Changes. ";
274	if (p == null)
275	status += "Hygienic";
276	else {
277	status += p.b;
278	//setCaretPosition(ta, p.a);
279	}
280	silentStatus(trim(status));
281	}
282
283	svoid status(S s) {
284	silentStatus(infoBoxAndReturn(s));
285	}
286
287	svoid silentStatus(S s) {
288	e.setStatus(s);
289	}
290
291	svoid saveSnippetThen(final Runnable next) {
292	if (empty(snippetID)) ret;
293	fS text = getText(e.textArea);
294	status("Saving...");
295	thread {
296	fS page = editSnippet(snippetID, text);
297	remoteText = text; // TODO: check save result
298	status("Saved snippet: " + page);
299	awtCallF(next);
300	}
301	}
302
303	svoid _butterReload() {
304	time { butterReload(snippetID); }
305	infoBox("Reloaded in " + lastTiming_format100ms());
306	}
307
308	svoid transpile(final bool medium) {
309	saveSnippetThen(r { _transpile(medium) });
310	}
311
312	svoid _transpile(final bool medium) {
313	status("Transpiling...");
314	//jdisable(btnEleuReload);
315
316	thread "Transpiling" {
317	try {
318	final Pair<Bool, S> p = transpileOnServer(snippetID, medium ? "medium" : "quick");
319	awt {
320	if (p.a)
321	status("Transpiled OK!");
322	else {
323	status("Transpilation error. " + p.b);
324	showText("Transpilation Error", p.b);
325	}
326	}
327	} catch print e {
328	status("Transpilation problem. " + e);
329	}
330	//jenable(btnEleuReload);
331	}
332	}
333
334	svoid _loadSnippet(S snippetID) {
335	e.textArea.setText(remoteText = loadSnippet(snippetID));
336	setAndSave('snippetID, snippetID);
337	frameTitle(e, snippetID);
338	}
339
340	svoid newText {
341	setAndSave(snippetID := null);
342	remoteText = null;
343	S lastText = e.getText();
344	if (nempty(lastText)) logQuotedWithDate("saved-texts.log", lastText);
345	e.setText("");
346	frameTitle(e, programTitle());
347	}
348
349	svoid cleanMeUp {
350	setAndSaveIfChanged(savedText := e.getText());
351	}