001 /*
002 * Copyright 1995-2005 Sun Microsystems, Inc. All Rights Reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation. Sun designates this
008 * particular file as subject to the "Classpath" exception as provided
009 * by Sun in the LICENSE file that accompanied this code.
010 *
011 * This code is distributed in the hope that it will be useful, but WITHOUT
012 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
013 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
014 * version 2 for more details (a copy is included in the LICENSE file that
015 * accompanied this code).
016 *
017 * You should have received a copy of the GNU General Public License version
018 * 2 along with this work; if not, write to the Free Software Foundation,
019 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
020 *
021 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
022 * CA 95054 USA or visit www.sun.com if you need additional information or
023 * have any questions.
024 */
025
026 package java.io;
027
028 import java.util.Arrays;
029
030 /**
031 * The <code>StreamTokenizer</code> class takes an input stream and
032 * parses it into "tokens", allowing the tokens to be
033 * read one at a time. The parsing process is controlled by a table
034 * and a number of flags that can be set to various states. The
035 * stream tokenizer can recognize identifiers, numbers, quoted
036 * strings, and various comment styles.
037 * <p>
038 * Each byte read from the input stream is regarded as a character
039 * in the range <code>'\u0000'</code> through <code>'\u00FF'</code>.
040 * The character value is used to look up five possible attributes of
041 * the character: <i>white space</i>, <i>alphabetic</i>,
042 * <i>numeric</i>, <i>string quote</i>, and <i>comment character</i>.
043 * Each character can have zero or more of these attributes.
044 * <p>
045 * In addition, an instance has four flags. These flags indicate:
046 * <ul>
047 * <li>Whether line terminators are to be returned as tokens or treated
048 * as white space that merely separates tokens.
049 * <li>Whether C-style comments are to be recognized and skipped.
050 * <li>Whether C++-style comments are to be recognized and skipped.
051 * <li>Whether the characters of identifiers are converted to lowercase.
052 * </ul>
053 * <p>
054 * A typical application first constructs an instance of this class,
055 * sets up the syntax tables, and then repeatedly loops calling the
056 * <code>nextToken</code> method in each iteration of the loop until
057 * it returns the value <code>TT_EOF</code>.
058 *
059 * @author James Gosling
060 * @version 1.53, 05/05/07
061 * @see java.io.StreamTokenizer#nextToken()
062 * @see java.io.StreamTokenizer#TT_EOF
063 * @since JDK1.0
064 */
065
066 public class StreamTokenizer {
067
068 /* Only one of these will be non-null */
069 private Reader reader = null;
070 private InputStream input = null;
071
072 private char buf[] = new char[20];
073
074 /**
075 * The next character to be considered by the nextToken method. May also
076 * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
077 * to indicate that a new character should be read and, if it is a '\n'
078 * character, it should be discarded and a second new character should be
079 * read.
080 */
081 private int peekc = NEED_CHAR;
082
083 private static final int NEED_CHAR = Integer.MAX_VALUE;
084 private static final int SKIP_LF = Integer.MAX_VALUE - 1;
085
086 private boolean pushedBack;
087 private boolean forceLower;
088 /** The line number of the last token read */
089 private int LINENO = 1;
090
091 private boolean eolIsSignificantP = false;
092 private boolean slashSlashCommentsP = false;
093 private boolean slashStarCommentsP = false;
094
095 private byte ctype[] = new byte[256];
096 private static final byte CT_WHITESPACE = 1;
097 private static final byte CT_DIGIT = 2;
098 private static final byte CT_ALPHA = 4;
099 private static final byte CT_QUOTE = 8;
100 private static final byte CT_COMMENT = 16;
101
102 /**
103 * After a call to the <code>nextToken</code> method, this field
104 * contains the type of the token just read. For a single character
105 * token, its value is the single character, converted to an integer.
106 * For a quoted string token, its value is the quote character.
107 * Otherwise, its value is one of the following:
108 * <ul>
109 * <li><code>TT_WORD</code> indicates that the token is a word.
110 * <li><code>TT_NUMBER</code> indicates that the token is a number.
111 * <li><code>TT_EOL</code> indicates that the end of line has been read.
112 * The field can only have this value if the
113 * <code>eolIsSignificant</code> method has been called with the
114 * argument <code>true</code>.
115 * <li><code>TT_EOF</code> indicates that the end of the input stream
116 * has been reached.
117 * </ul>
118 * <p>
119 * The initial value of this field is -4.
120 *
121 * @see java.io.StreamTokenizer#eolIsSignificant(boolean)
122 * @see java.io.StreamTokenizer#nextToken()
123 * @see java.io.StreamTokenizer#quoteChar(int)
124 * @see java.io.StreamTokenizer#TT_EOF
125 * @see java.io.StreamTokenizer#TT_EOL
126 * @see java.io.StreamTokenizer#TT_NUMBER
127 * @see java.io.StreamTokenizer#TT_WORD
128 */
129 public int ttype = TT_NOTHING;
130
131 /**
132 * A constant indicating that the end of the stream has been read.
133 */
134 public static final int TT_EOF = -1;
135
136 /**
137 * A constant indicating that the end of the line has been read.
138 */
139 public static final int TT_EOL = '\n';
140
141 /**
142 * A constant indicating that a number token has been read.
143 */
144 public static final int TT_NUMBER = -2;
145
146 /**
147 * A constant indicating that a word token has been read.
148 */
149 public static final int TT_WORD = -3;
150
151 /* A constant indicating that no token has been read, used for
152 * initializing ttype. FIXME This could be made public and
153 * made available as the part of the API in a future release.
154 */
155 private static final int TT_NOTHING = -4;
156
157 /**
158 * If the current token is a word token, this field contains a
159 * string giving the characters of the word token. When the current
160 * token is a quoted string token, this field contains the body of
161 * the string.
162 * <p>
163 * The current token is a word when the value of the
164 * <code>ttype</code> field is <code>TT_WORD</code>. The current token is
165 * a quoted string token when the value of the <code>ttype</code> field is
166 * a quote character.
167 * <p>
168 * The initial value of this field is null.
169 *
170 * @see java.io.StreamTokenizer#quoteChar(int)
171 * @see java.io.StreamTokenizer#TT_WORD
172 * @see java.io.StreamTokenizer#ttype
173 */
174 public String sval;
175
176 /**
177 * If the current token is a number, this field contains the value
178 * of that number. The current token is a number when the value of
179 * the <code>ttype</code> field is <code>TT_NUMBER</code>.
180 * <p>
181 * The initial value of this field is 0.0.
182 *
183 * @see java.io.StreamTokenizer#TT_NUMBER
184 * @see java.io.StreamTokenizer#ttype
185 */
186 public double nval;
187
188 /** Private constructor that initializes everything except the streams. */
189 private StreamTokenizer() {
190 wordChars('a', 'z');
191 wordChars('A', 'Z');
192 wordChars(128 + 32, 255);
193 whitespaceChars(0, ' ');
194 commentChar('/');
195 quoteChar('"');
196 quoteChar('\'');
197 parseNumbers();
198 }
199
200 /**
201 * Creates a stream tokenizer that parses the specified input
202 * stream. The stream tokenizer is initialized to the following
203 * default state:
204 * <ul>
205 * <li>All byte values <code>'A'</code> through <code>'Z'</code>,
206 * <code>'a'</code> through <code>'z'</code>, and
207 * <code>'\u00A0'</code> through <code>'\u00FF'</code> are
208 * considered to be alphabetic.
209 * <li>All byte values <code>'\u0000'</code> through
210 * <code>'\u0020'</code> are considered to be white space.
211 * <li><code>'/'</code> is a comment character.
212 * <li>Single quote <code>'\''</code> and double quote <code>'"'</code>
213 * are string quote characters.
214 * <li>Numbers are parsed.
215 * <li>Ends of lines are treated as white space, not as separate tokens.
216 * <li>C-style and C++-style comments are not recognized.
217 * </ul>
218 *
219 * @deprecated As of JDK version 1.1, the preferred way to tokenize an
220 * input stream is to convert it into a character stream, for example:
221 * <blockquote><pre>
222 * Reader r = new BufferedReader(new InputStreamReader(is));
223 * StreamTokenizer st = new StreamTokenizer(r);
224 * </pre></blockquote>
225 *
226 * @param is an input stream.
227 * @see java.io.BufferedReader
228 * @see java.io.InputStreamReader
229 * @see java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
230 */
231 @Deprecated
232 public StreamTokenizer(InputStream is) {
233 this ();
234 if (is == null) {
235 throw new NullPointerException();
236 }
237 input = is;
238 }
239
240 /**
241 * Create a tokenizer that parses the given character stream.
242 *
243 * @param r a Reader object providing the input stream.
244 * @since JDK1.1
245 */
246 public StreamTokenizer(Reader r) {
247 this ();
248 if (r == null) {
249 throw new NullPointerException();
250 }
251 reader = r;
252 }
253
254 /**
255 * Resets this tokenizer's syntax table so that all characters are
256 * "ordinary." See the <code>ordinaryChar</code> method
257 * for more information on a character being ordinary.
258 *
259 * @see java.io.StreamTokenizer#ordinaryChar(int)
260 */
261 public void resetSyntax() {
262 for (int i = ctype.length; --i >= 0;)
263 ctype[i] = 0;
264 }
265
266 /**
267 * Specifies that all characters <i>c</i> in the range
268 * <code>low <= <i>c</i> <= high</code>
269 * are word constituents. A word token consists of a word constituent
270 * followed by zero or more word constituents or number constituents.
271 *
272 * @param low the low end of the range.
273 * @param hi the high end of the range.
274 */
275 public void wordChars(int low, int hi) {
276 if (low < 0)
277 low = 0;
278 if (hi >= ctype.length)
279 hi = ctype.length - 1;
280 while (low <= hi)
281 ctype[low++] |= CT_ALPHA;
282 }
283
284 /**
285 * Specifies that all characters <i>c</i> in the range
286 * <code>low <= <i>c</i> <= high</code>
287 * are white space characters. White space characters serve only to
288 * separate tokens in the input stream.
289 *
290 * <p>Any other attribute settings for the characters in the specified
291 * range are cleared.
292 *
293 * @param low the low end of the range.
294 * @param hi the high end of the range.
295 */
296 public void whitespaceChars(int low, int hi) {
297 if (low < 0)
298 low = 0;
299 if (hi >= ctype.length)
300 hi = ctype.length - 1;
301 while (low <= hi)
302 ctype[low++] = CT_WHITESPACE;
303 }
304
305 /**
306 * Specifies that all characters <i>c</i> in the range
307 * <code>low <= <i>c</i> <= high</code>
308 * are "ordinary" in this tokenizer. See the
309 * <code>ordinaryChar</code> method for more information on a
310 * character being ordinary.
311 *
312 * @param low the low end of the range.
313 * @param hi the high end of the range.
314 * @see java.io.StreamTokenizer#ordinaryChar(int)
315 */
316 public void ordinaryChars(int low, int hi) {
317 if (low < 0)
318 low = 0;
319 if (hi >= ctype.length)
320 hi = ctype.length - 1;
321 while (low <= hi)
322 ctype[low++] = 0;
323 }
324
325 /**
326 * Specifies that the character argument is "ordinary"
327 * in this tokenizer. It removes any special significance the
328 * character has as a comment character, word component, string
329 * delimiter, white space, or number character. When such a character
330 * is encountered by the parser, the parser treats it as a
331 * single-character token and sets <code>ttype</code> field to the
332 * character value.
333 *
334 * <p>Making a line terminator character "ordinary" may interfere
335 * with the ability of a <code>StreamTokenizer</code> to count
336 * lines. The <code>lineno</code> method may no longer reflect
337 * the presence of such terminator characters in its line count.
338 *
339 * @param ch the character.
340 * @see java.io.StreamTokenizer#ttype
341 */
342 public void ordinaryChar(int ch) {
343 if (ch >= 0 && ch < ctype.length)
344 ctype[ch] = 0;
345 }
346
347 /**
348 * Specified that the character argument starts a single-line
349 * comment. All characters from the comment character to the end of
350 * the line are ignored by this stream tokenizer.
351 *
352 * <p>Any other attribute settings for the specified character are cleared.
353 *
354 * @param ch the character.
355 */
356 public void commentChar(int ch) {
357 if (ch >= 0 && ch < ctype.length)
358 ctype[ch] = CT_COMMENT;
359 }
360
361 /**
362 * Specifies that matching pairs of this character delimit string
363 * constants in this tokenizer.
364 * <p>
365 * When the <code>nextToken</code> method encounters a string
366 * constant, the <code>ttype</code> field is set to the string
367 * delimiter and the <code>sval</code> field is set to the body of
368 * the string.
369 * <p>
370 * If a string quote character is encountered, then a string is
371 * recognized, consisting of all characters after (but not including)
372 * the string quote character, up to (but not including) the next
373 * occurrence of that same string quote character, or a line
374 * terminator, or end of file. The usual escape sequences such as
375 * <code>"\n"</code> and <code>"\t"</code> are recognized and
376 * converted to single characters as the string is parsed.
377 *
378 * <p>Any other attribute settings for the specified character are cleared.
379 *
380 * @param ch the character.
381 * @see java.io.StreamTokenizer#nextToken()
382 * @see java.io.StreamTokenizer#sval
383 * @see java.io.StreamTokenizer#ttype
384 */
385 public void quoteChar(int ch) {
386 if (ch >= 0 && ch < ctype.length)
387 ctype[ch] = CT_QUOTE;
388 }
389
390 /**
391 * Specifies that numbers should be parsed by this tokenizer. The
392 * syntax table of this tokenizer is modified so that each of the twelve
393 * characters:
394 * <blockquote><pre>
395 * 0 1 2 3 4 5 6 7 8 9 . -
396 * </pre></blockquote>
397 * <p>
398 * has the "numeric" attribute.
399 * <p>
400 * When the parser encounters a word token that has the format of a
401 * double precision floating-point number, it treats the token as a
402 * number rather than a word, by setting the <code>ttype</code>
403 * field to the value <code>TT_NUMBER</code> and putting the numeric
404 * value of the token into the <code>nval</code> field.
405 *
406 * @see java.io.StreamTokenizer#nval
407 * @see java.io.StreamTokenizer#TT_NUMBER
408 * @see java.io.StreamTokenizer#ttype
409 */
410 public void parseNumbers() {
411 for (int i = '0'; i <= '9'; i++)
412 ctype[i] |= CT_DIGIT;
413 ctype['.'] |= CT_DIGIT;
414 ctype['-'] |= CT_DIGIT;
415 }
416
417 /**
418 * Determines whether or not ends of line are treated as tokens.
419 * If the flag argument is true, this tokenizer treats end of lines
420 * as tokens; the <code>nextToken</code> method returns
421 * <code>TT_EOL</code> and also sets the <code>ttype</code> field to
422 * this value when an end of line is read.
423 * <p>
424 * A line is a sequence of characters ending with either a
425 * carriage-return character (<code>'\r'</code>) or a newline
426 * character (<code>'\n'</code>). In addition, a carriage-return
427 * character followed immediately by a newline character is treated
428 * as a single end-of-line token.
429 * <p>
430 * If the <code>flag</code> is false, end-of-line characters are
431 * treated as white space and serve only to separate tokens.
432 *
433 * @param flag <code>true</code> indicates that end-of-line characters
434 * are separate tokens; <code>false</code> indicates that
435 * end-of-line characters are white space.
436 * @see java.io.StreamTokenizer#nextToken()
437 * @see java.io.StreamTokenizer#ttype
438 * @see java.io.StreamTokenizer#TT_EOL
439 */
440 public void eolIsSignificant(boolean flag) {
441 eolIsSignificantP = flag;
442 }
443
444 /**
445 * Determines whether or not the tokenizer recognizes C-style comments.
446 * If the flag argument is <code>true</code>, this stream tokenizer
447 * recognizes C-style comments. All text between successive
448 * occurrences of <code>/*</code> and <code>*/</code> are discarded.
449 * <p>
450 * If the flag argument is <code>false</code>, then C-style comments
451 * are not treated specially.
452 *
453 * @param flag <code>true</code> indicates to recognize and ignore
454 * C-style comments.
455 */
456 public void slashStarComments(boolean flag) {
457 slashStarCommentsP = flag;
458 }
459
460 /**
461 * Determines whether or not the tokenizer recognizes C++-style comments.
462 * If the flag argument is <code>true</code>, this stream tokenizer
463 * recognizes C++-style comments. Any occurrence of two consecutive
464 * slash characters (<code>'/'</code>) is treated as the beginning of
465 * a comment that extends to the end of the line.
466 * <p>
467 * If the flag argument is <code>false</code>, then C++-style
468 * comments are not treated specially.
469 *
470 * @param flag <code>true</code> indicates to recognize and ignore
471 * C++-style comments.
472 */
473 public void slashSlashComments(boolean flag) {
474 slashSlashCommentsP = flag;
475 }
476
477 /**
478 * Determines whether or not word token are automatically lowercased.
479 * If the flag argument is <code>true</code>, then the value in the
480 * <code>sval</code> field is lowercased whenever a word token is
481 * returned (the <code>ttype</code> field has the
482 * value <code>TT_WORD</code> by the <code>nextToken</code> method
483 * of this tokenizer.
484 * <p>
485 * If the flag argument is <code>false</code>, then the
486 * <code>sval</code> field is not modified.
487 *
488 * @param fl <code>true</code> indicates that all word tokens should
489 * be lowercased.
490 * @see java.io.StreamTokenizer#nextToken()
491 * @see java.io.StreamTokenizer#ttype
492 * @see java.io.StreamTokenizer#TT_WORD
493 */
494 public void lowerCaseMode(boolean fl) {
495 forceLower = fl;
496 }
497
498 /** Read the next character */
499 private int read() throws IOException {
500 if (reader != null)
501 return reader.read();
502 else if (input != null)
503 return input.read();
504 else
505 throw new IllegalStateException();
506 }
507
508 /**
509 * Parses the next token from the input stream of this tokenizer.
510 * The type of the next token is returned in the <code>ttype</code>
511 * field. Additional information about the token may be in the
512 * <code>nval</code> field or the <code>sval</code> field of this
513 * tokenizer.
514 * <p>
515 * Typical clients of this
516 * class first set up the syntax tables and then sit in a loop
517 * calling nextToken to parse successive tokens until TT_EOF
518 * is returned.
519 *
520 * @return the value of the <code>ttype</code> field.
521 * @exception IOException if an I/O error occurs.
522 * @see java.io.StreamTokenizer#nval
523 * @see java.io.StreamTokenizer#sval
524 * @see java.io.StreamTokenizer#ttype
525 */
526 public int nextToken() throws IOException {
527 if (pushedBack) {
528 pushedBack = false;
529 return ttype;
530 }
531 byte ct[] = ctype;
532 sval = null;
533
534 int c = peekc;
535 if (c < 0)
536 c = NEED_CHAR;
537 if (c == SKIP_LF) {
538 c = read();
539 if (c < 0)
540 return ttype = TT_EOF;
541 if (c == '\n')
542 c = NEED_CHAR;
543 }
544 if (c == NEED_CHAR) {
545 c = read();
546 if (c < 0)
547 return ttype = TT_EOF;
548 }
549 ttype = c; /* Just to be safe */
550
551 /* Set peekc so that the next invocation of nextToken will read
552 * another character unless peekc is reset in this invocation
553 */
554 peekc = NEED_CHAR;
555
556 int ctype = c < 256 ? ct[c] : CT_ALPHA;
557 while ((ctype & CT_WHITESPACE) != 0) {
558 if (c == '\r') {
559 LINENO++;
560 if (eolIsSignificantP) {
561 peekc = SKIP_LF;
562 return ttype = TT_EOL;
563 }
564 c = read();
565 if (c == '\n')
566 c = read();
567 } else {
568 if (c == '\n') {
569 LINENO++;
570 if (eolIsSignificantP) {
571 return ttype = TT_EOL;
572 }
573 }
574 c = read();
575 }
576 if (c < 0)
577 return ttype = TT_EOF;
578 ctype = c < 256 ? ct[c] : CT_ALPHA;
579 }
580
581 if ((ctype & CT_DIGIT) != 0) {
582 boolean neg = false;
583 if (c == '-') {
584 c = read();
585 if (c != '.' && (c < '0' || c > '9')) {
586 peekc = c;
587 return ttype = '-';
588 }
589 neg = true;
590 }
591 double v = 0;
592 int decexp = 0;
593 int seendot = 0;
594 while (true) {
595 if (c == '.' && seendot == 0)
596 seendot = 1;
597 else if ('0' <= c && c <= '9') {
598 v = v * 10 + (c - '0');
599 decexp += seendot;
600 } else
601 break;
602 c = read();
603 }
604 peekc = c;
605 if (decexp != 0) {
606 double denom = 10;
607 decexp--;
608 while (decexp > 0) {
609 denom *= 10;
610 decexp--;
611 }
612 /* Do one division of a likely-to-be-more-accurate number */
613 v = v / denom;
614 }
615 nval = neg ? -v : v;
616 return ttype = TT_NUMBER;
617 }
618
619 if ((ctype & CT_ALPHA) != 0) {
620 int i = 0;
621 do {
622 if (i >= buf.length) {
623 buf = Arrays.copyOf(buf, buf.length * 2);
624 }
625 buf[i++] = (char) c;
626 c = read();
627 ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c]
628 : CT_ALPHA;
629 } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0);
630 peekc = c;
631 sval = String.copyValueOf(buf, 0, i);
632 if (forceLower)
633 sval = sval.toLowerCase();
634 return ttype = TT_WORD;
635 }
636
637 if ((ctype & CT_QUOTE) != 0) {
638 ttype = c;
639 int i = 0;
640 /* Invariants (because \Octal needs a lookahead):
641 * (i) c contains char value
642 * (ii) d contains the lookahead
643 */
644 int d = read();
645 while (d >= 0 && d != ttype && d != '\n' && d != '\r') {
646 if (d == '\\') {
647 c = read();
648 int first = c; /* To allow \377, but not \477 */
649 if (c >= '0' && c <= '7') {
650 c = c - '0';
651 int c2 = read();
652 if ('0' <= c2 && c2 <= '7') {
653 c = (c << 3) + (c2 - '0');
654 c2 = read();
655 if ('0' <= c2 && c2 <= '7' && first <= '3') {
656 c = (c << 3) + (c2 - '0');
657 d = read();
658 } else
659 d = c2;
660 } else
661 d = c2;
662 } else {
663 switch (c) {
664 case 'a':
665 c = 0x7;
666 break;
667 case 'b':
668 c = '\b';
669 break;
670 case 'f':
671 c = 0xC;
672 break;
673 case 'n':
674 c = '\n';
675 break;
676 case 'r':
677 c = '\r';
678 break;
679 case 't':
680 c = '\t';
681 break;
682 case 'v':
683 c = 0xB;
684 break;
685 }
686 d = read();
687 }
688 } else {
689 c = d;
690 d = read();
691 }
692 if (i >= buf.length) {
693 buf = Arrays.copyOf(buf, buf.length * 2);
694 }
695 buf[i++] = (char) c;
696 }
697
698 /* If we broke out of the loop because we found a matching quote
699 * character then arrange to read a new character next time
700 * around; otherwise, save the character.
701 */
702 peekc = (d == ttype) ? NEED_CHAR : d;
703
704 sval = String.copyValueOf(buf, 0, i);
705 return ttype;
706 }
707
708 if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) {
709 c = read();
710 if (c == '*' && slashStarCommentsP) {
711 int prevc = 0;
712 while ((c = read()) != '/' || prevc != '*') {
713 if (c == '\r') {
714 LINENO++;
715 c = read();
716 if (c == '\n') {
717 c = read();
718 }
719 } else {
720 if (c == '\n') {
721 LINENO++;
722 c = read();
723 }
724 }
725 if (c < 0)
726 return ttype = TT_EOF;
727 prevc = c;
728 }
729 return nextToken();
730 } else if (c == '/' && slashSlashCommentsP) {
731 while ((c = read()) != '\n' && c != '\r' && c >= 0)
732 ;
733 peekc = c;
734 return nextToken();
735 } else {
736 /* Now see if it is still a single line comment */
737 if ((ct['/'] & CT_COMMENT) != 0) {
738 while ((c = read()) != '\n' && c != '\r' && c >= 0)
739 ;
740 peekc = c;
741 return nextToken();
742 } else {
743 peekc = c;
744 return ttype = '/';
745 }
746 }
747 }
748
749 if ((ctype & CT_COMMENT) != 0) {
750 while ((c = read()) != '\n' && c != '\r' && c >= 0)
751 ;
752 peekc = c;
753 return nextToken();
754 }
755
756 return ttype = c;
757 }
758
759 /**
760 * Causes the next call to the <code>nextToken</code> method of this
761 * tokenizer to return the current value in the <code>ttype</code>
762 * field, and not to modify the value in the <code>nval</code> or
763 * <code>sval</code> field.
764 *
765 * @see java.io.StreamTokenizer#nextToken()
766 * @see java.io.StreamTokenizer#nval
767 * @see java.io.StreamTokenizer#sval
768 * @see java.io.StreamTokenizer#ttype
769 */
770 public void pushBack() {
771 if (ttype != TT_NOTHING) /* No-op if nextToken() not called */
772 pushedBack = true;
773 }
774
775 /**
776 * Return the current line number.
777 *
778 * @return the current line number of this stream tokenizer.
779 */
780 public int lineno() {
781 return LINENO;
782 }
783
784 /**
785 * Returns the string representation of the current stream token and
786 * the line number it occurs on.
787 *
788 * <p>The precise string returned is unspecified, although the following
789 * example can be considered typical:
790 *
791 * <blockquote><pre>Token['a'], line 10</pre></blockquote>
792 *
793 * @return a string representation of the token
794 * @see java.io.StreamTokenizer#nval
795 * @see java.io.StreamTokenizer#sval
796 * @see java.io.StreamTokenizer#ttype
797 */
798 public String toString() {
799 String ret;
800 switch (ttype) {
801 case TT_EOF:
802 ret = "EOF";
803 break;
804 case TT_EOL:
805 ret = "EOL";
806 break;
807 case TT_WORD:
808 ret = sval;
809 break;
810 case TT_NUMBER:
811 ret = "n=" + nval;
812 break;
813 case TT_NOTHING:
814 ret = "NOTHING";
815 break;
816 default: {
817 /*
818 * ttype is the first character of either a quoted string or
819 * is an ordinary character. ttype can definitely not be less
820 * than 0, since those are reserved values used in the previous
821 * case statements
822 */
823 if (ttype < 256 && ((ctype[ttype] & CT_QUOTE) != 0)) {
824 ret = sval;
825 break;
826 }
827
828 char s[] = new char[3];
829 s[0] = s[2] = '\'';
830 s[1] = (char) ttype;
831 ret = new String(s);
832 break;
833 }
834 }
835 return "Token[" + ret + "], line " + LINENO;
836 }
837
838 }
|