001: /*
002: * @(#)StreamTokenizer.java 1.46 06/10/10
003: *
004: * Copyright 1990-2006 Sun Microsystems, Inc. All Rights Reserved.
005: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER
006: *
007: * This program is free software; you can redistribute it and/or
008: * modify it under the terms of the GNU General Public License version
009: * 2 only, as published by the Free Software Foundation.
010: *
011: * This program is distributed in the hope that it will be useful, but
012: * WITHOUT ANY WARRANTY; without even the implied warranty of
013: * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014: * General Public License version 2 for more details (a copy is
015: * included at /legal/license.txt).
016: *
017: * You should have received a copy of the GNU General Public License
018: * version 2 along with this work; if not, write to the Free Software
019: * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
020: * 02110-1301 USA
021: *
022: * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
023: * Clara, CA 95054 or visit www.sun.com if you need additional
024: * information or have any questions.
025: *
026: */
027:
028: package java.io;
029:
030: import sun.misc.CVM;
031:
032: /**
033: * The <code>StreamTokenizer</code> class takes an input stream and
034: * parses it into "tokens", allowing the tokens to be
035: * read one at a time. The parsing process is controlled by a table
036: * and a number of flags that can be set to various states. The
037: * stream tokenizer can recognize identifiers, numbers, quoted
038: * strings, and various comment styles.
039: * <p>
040: * Each byte read from the input stream is regarded as a character
041: * in the range <code>'\u0000'</code> through <code>'\u00FF'</code>.
042: * The character value is used to look up five possible attributes of
043: * the character: <i>white space</i>, <i>alphabetic</i>,
044: * <i>numeric</i>, <i>string quote</i>, and <i>comment character</i>.
045: * Each character can have zero or more of these attributes.
046: * <p>
047: * In addition, an instance has four flags. These flags indicate:
048: * <ul>
049: * <li>Whether line terminators are to be returned as tokens or treated
050: * as white space that merely separates tokens.
051: * <li>Whether C-style comments are to be recognized and skipped.
052: * <li>Whether C++-style comments are to be recognized and skipped.
053: * <li>Whether the characters of identifiers are converted to lowercase.
054: * </ul>
055: * <p>
056: * A typical application first constructs an instance of this class,
057: * sets up the syntax tables, and then repeatedly loops calling the
058: * <code>nextToken</code> method in each iteration of the loop until
059: * it returns the value <code>TT_EOF</code>.
060: *
061: * @author James Gosling
062: * @version 1.37, 05/03/00
063: * @see java.io.StreamTokenizer#nextToken()
064: * @see java.io.StreamTokenizer#TT_EOF
065: * @since JDK1.0
066: */
067:
068: public class StreamTokenizer {
069:
070: /* Only one of these will be non-null */
071: private Reader reader = null;
072: private InputStream input = null;
073:
074: private char buf[] = new char[20];
075:
076: /**
077: * The next character to be considered by the nextToken method. May also
078: * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
079: * to indicate that a new character should be read and, if it is a '\n'
080: * character, it should be discarded and a second new character should be
081: * read.
082: */
083: private int peekc = NEED_CHAR;
084:
085: private static final int NEED_CHAR = Integer.MAX_VALUE;
086: private static final int SKIP_LF = Integer.MAX_VALUE - 1;
087:
088: private boolean pushedBack;
089: private boolean forceLower;
090: /** The line number of the last token read */
091: private int LINENO = 1;
092:
093: private boolean eolIsSignificantP = false;
094: private boolean slashSlashCommentsP = false;
095: private boolean slashStarCommentsP = false;
096:
097: private byte ctype[] = new byte[256];
098: private static final byte CT_WHITESPACE = 1;
099: private static final byte CT_DIGIT = 2;
100: private static final byte CT_ALPHA = 4;
101: private static final byte CT_QUOTE = 8;
102: private static final byte CT_COMMENT = 16;
103:
104: /**
105: * After a call to the <code>nextToken</code> method, this field
106: * contains the type of the token just read. For a single character
107: * token, its value is the single character, converted to an integer.
108: * For a quoted string token (see , its value is the quote character.
109: * Otherwise, its value is one of the following:
110: * <ul>
111: * <li><code>TT_WORD</code> indicates that the token is a word.
112: * <li><code>TT_NUMBER</code> indicates that the token is a number.
113: * <li><code>TT_EOL</code> indicates that the end of line has been read.
114: * The field can only have this value if the
115: * <code>eolIsSignificant</code> method has been called with the
116: * argument <code>true</code>.
117: * <li><code>TT_EOF</code> indicates that the end of the input stream
118: * has been reached.
119: * </ul>
120: * <p>
121: * The initial value of this field is -4.
122: *
123: * @see java.io.StreamTokenizer#eolIsSignificant(boolean)
124: * @see java.io.StreamTokenizer#nextToken()
125: * @see java.io.StreamTokenizer#quoteChar(int)
126: * @see java.io.StreamTokenizer#TT_EOF
127: * @see java.io.StreamTokenizer#TT_EOL
128: * @see java.io.StreamTokenizer#TT_NUMBER
129: * @see java.io.StreamTokenizer#TT_WORD
130: */
131: public int ttype = TT_NOTHING;
132:
133: /**
134: * A constant indicating that the end of the stream has been read.
135: */
136: public static final int TT_EOF = -1;
137:
138: /**
139: * A constant indicating that the end of the line has been read.
140: */
141: public static final int TT_EOL = '\n';
142:
143: /**
144: * A constant indicating that a number token has been read.
145: */
146: public static final int TT_NUMBER = -2;
147:
148: /**
149: * A constant indicating that a word token has been read.
150: */
151: public static final int TT_WORD = -3;
152:
153: /* A constant indicating that no token has been read, used for
154: * initializing ttype. (Perhaps this could be made public and
155: * made available as the part of the API in a future release?)
156: */
157: private static final int TT_NOTHING = -4;
158:
159: /**
160: * If the current token is a word token, this field contains a
161: * string giving the characters of the word token. When the current
162: * token is a quoted string token, this field contains the body of
163: * the string.
164: * <p>
165: * The current token is a word when the value of the
166: * <code>ttype</code> field is <code>TT_WORD</code>. The current token is
167: * a quoted string token when the value of the <code>ttype</code> field is
168: * a quote character.
169: * <p>
170: * The initial value of this field is null.
171: *
172: * @see java.io.StreamTokenizer#quoteChar(int)
173: * @see java.io.StreamTokenizer#TT_WORD
174: * @see java.io.StreamTokenizer#ttype
175: */
176: public String sval;
177:
178: /**
179: * If the current token is a number, this field contains the value
180: * of that number. The current token is a number when the value of
181: * the <code>ttype</code> field is <code>TT_NUMBER</code>.
182: * <p>
183: * The initial value of this field is 0.0.
184: *
185: * @see java.io.StreamTokenizer#TT_NUMBER
186: * @see java.io.StreamTokenizer#ttype
187: */
188: public double nval;
189:
190: /** Private constructor that initializes everything except the streams. */
191: private StreamTokenizer() {
192: wordChars('a', 'z');
193: wordChars('A', 'Z');
194: wordChars(128 + 32, 255);
195: whitespaceChars(0, ' ');
196: commentChar('/');
197: quoteChar('"');
198: quoteChar('\'');
199: parseNumbers();
200: }
201:
202: /**
203: * Creates a stream tokenizer that parses the specified input
204: * stream. The stream tokenizer is initialized to the following
205: * default state:
206: * <ul>
207: * <li>All byte values <code>'A'</code> through <code>'Z'</code>,
208: * <code>'a'</code> through <code>'z'</code>, and
209: * <code>'\u00A0'</code> through <code>'\u00FF'</code> are
210: * considered to be alphabetic.
211: * <li>All byte values <code>'\u0000'</code> through
212: * <code>'\u0020'</code> are considered to be white space.
213: * <li><code>'/'</code> is a comment character.
214: * <li>Single quote <code>'\''</code> and double quote <code>'"'</code>
215: * are string quote characters.
216: * <li>Numbers are parsed.
217: * <li>Ends of lines are treated as white space, not as separate tokens.
218: * <li>C-style and C++-style comments are not recognized.
219: * </ul>
220: *
221: * deprecated As of JDK version 1.1, the preferred way to tokenize an
222: * input stream is to convert it into a character stream, for example:
223: * <blockquote><pre>
224: * Reader r = new BufferedReader(new InputStreamReader(is));
225: * StreamTokenizer st = new StreamTokenizer(r);
226: * </pre></blockquote>
227: *
228: * @param is an input stream.
229: * @see java.io.BufferedReader
230: * @see java.io.InputStreamReader
231: * @see java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
232: public StreamTokenizer(InputStream is) {
233: this();
234: if (is == null) {
235: throw new NullPointerException();
236: }
237: input = is;
238: }
239: */
240:
241: /**
242: * Create a tokenizer that parses the given character stream.
243: *
244: * @param r a Reader object providing the input stream.
245: * @since JDK1.1
246: */
247: public StreamTokenizer(Reader r) {
248: this ();
249: if (r == null) {
250: throw new NullPointerException();
251: }
252: reader = r;
253: }
254:
255: /**
256: * Resets this tokenizer's syntax table so that all characters are
257: * "ordinary." See the <code>ordinaryChar</code> method
258: * for more information on a character being ordinary.
259: *
260: * @see java.io.StreamTokenizer#ordinaryChar(int)
261: */
262: public void resetSyntax() {
263: for (int i = ctype.length; --i >= 0;)
264: ctype[i] = 0;
265: }
266:
267: /**
268: * Specifies that all characters <i>c</i> in the range
269: * <code>low <= <i>c</i> <= high</code>
270: * are word constituents. A word token consists of a word constituent
271: * followed by zero or more word constituents or number constituents.
272: *
273: * @param low the low end of the range.
274: * @param hi the high end of the range.
275: */
276: public void wordChars(int low, int hi) {
277: if (low < 0)
278: low = 0;
279: if (hi >= ctype.length)
280: hi = ctype.length - 1;
281: while (low <= hi)
282: ctype[low++] |= CT_ALPHA;
283: }
284:
285: /**
286: * Specifies that all characters <i>c</i> in the range
287: * <code>low <= <i>c</i> <= high</code>
288: * are white space characters. White space characters serve only to
289: * separate tokens in the input stream.
290: *
291: * <p>Any other attribute settings for the characters in the specified
292: * range are cleared.
293: *
294: * @param low the low end of the range.
295: * @param hi the high end of the range.
296: */
297: public void whitespaceChars(int low, int hi) {
298: if (low < 0)
299: low = 0;
300: if (hi >= ctype.length)
301: hi = ctype.length - 1;
302: while (low <= hi)
303: ctype[low++] = CT_WHITESPACE;
304: }
305:
306: /**
307: * Specifies that all characters <i>c</i> in the range
308: * <code>low <= <i>c</i> <= high</code>
309: * are "ordinary" in this tokenizer. See the
310: * <code>ordinaryChar</code> method for more information on a
311: * character being ordinary.
312: *
313: * @param low the low end of the range.
314: * @param hi the high end of the range.
315: * @see java.io.StreamTokenizer#ordinaryChar(int)
316: */
317: public void ordinaryChars(int low, int hi) {
318: if (low < 0)
319: low = 0;
320: if (hi >= ctype.length)
321: hi = ctype.length - 1;
322: while (low <= hi)
323: ctype[low++] = 0;
324: }
325:
326: /**
327: * Specifies that the character argument is "ordinary"
328: * in this tokenizer. It removes any special significance the
329: * character has as a comment character, word component, string
330: * delimiter, white space, or number character. When such a character
331: * is encountered by the parser, the parser treates it as a
332: * single-character token and sets <code>ttype</code> field to the
333: * character value.
334: *
335: * @param ch the character.
336: * @see java.io.StreamTokenizer#ttype
337: */
338: public void ordinaryChar(int ch) {
339: if (ch >= 0 && ch < ctype.length)
340: ctype[ch] = 0;
341: }
342:
343: /**
344: * Specified that the character argument starts a single-line
345: * comment. All characters from the comment character to the end of
346: * the line are ignored by this stream tokenizer.
347: *
348: * <p>Any other attribute settings for the specified character are cleared.
349: *
350: * @param ch the character.
351: */
352: public void commentChar(int ch) {
353: if (ch >= 0 && ch < ctype.length)
354: ctype[ch] = CT_COMMENT;
355: }
356:
357: /**
358: * Specifies that matching pairs of this character delimit string
359: * constants in this tokenizer.
360: * <p>
361: * When the <code>nextToken</code> method encounters a string
362: * constant, the <code>ttype</code> field is set to the string
363: * delimiter and the <code>sval</code> field is set to the body of
364: * the string.
365: * <p>
366: * If a string quote character is encountered, then a string is
367: * recognized, consisting of all characters after (but not including)
368: * the string quote character, up to (but not including) the next
369: * occurrence of that same string quote character, or a line
370: * terminator, or end of file. The usual escape sequences such as
371: * <code>"\n"</code> and <code>"\t"</code> are recognized and
372: * converted to single characters as the string is parsed.
373: *
374: * <p>Any other attribute settings for the specified character are cleared.
375: *
376: * @param ch the character.
377: * @see java.io.StreamTokenizer#nextToken()
378: * @see java.io.StreamTokenizer#sval
379: * @see java.io.StreamTokenizer#ttype
380: */
381: public void quoteChar(int ch) {
382: if (ch >= 0 && ch < ctype.length)
383: ctype[ch] = CT_QUOTE;
384: }
385:
386: /**
387: * Specifies that numbers should be parsed by this tokenizer. The
388: * syntax table of this tokenizer is modified so that each of the twelve
389: * characters:
390: * <blockquote><pre>
391: * 0 1 2 3 4 5 6 7 8 9 . -
392: * </pre></blockquote>
393: * <p>
394: * has the "numeric" attribute.
395: * <p>
396: * When the parser encounters a word token that has the format of a
397: * double precision floating-point number, it treats the token as a
398: * number rather than a word, by setting the the <code>ttype</code>
399: * field to the value <code>TT_NUMBER</code> and putting the numeric
400: * value of the token into the <code>nval</code> field.
401: *
402: * @see java.io.StreamTokenizer#nval
403: * @see java.io.StreamTokenizer#TT_NUMBER
404: * @see java.io.StreamTokenizer#ttype
405: */
406: public void parseNumbers() {
407: for (int i = '0'; i <= '9'; i++)
408: ctype[i] |= CT_DIGIT;
409: ctype['.'] |= CT_DIGIT;
410: ctype['-'] |= CT_DIGIT;
411: }
412:
413: /**
414: * Determines whether or not ends of line are treated as tokens.
415: * If the flag argument is true, this tokenizer treats end of lines
416: * as tokens; the <code>nextToken</code> method returns
417: * <code>TT_EOL</code> and also sets the <code>ttype</code> field to
418: * this value when an end of line is read.
419: * <p>
420: * A line is a sequence of characters ending with either a
421: * carriage-return character (<code>'\r'</code>) or a newline
422: * character (<code>'\n'</code>). In addition, a carriage-return
423: * character followed immediately by a newline character is treated
424: * as a single end-of-line token.
425: * <p>
426: * If the <code>flag</code> is false, end-of-line characters are
427: * treated as white space and serve only to separate tokens.
428: *
429: * @param flag <code>true</code> indicates that end-of-line characters
430: * are separate tokens; <code>false</code> indicates that
431: * end-of-line characters are white space.
432: * @see java.io.StreamTokenizer#nextToken()
433: * @see java.io.StreamTokenizer#ttype
434: * @see java.io.StreamTokenizer#TT_EOL
435: */
436: public void eolIsSignificant(boolean flag) {
437: eolIsSignificantP = flag;
438: }
439:
440: /**
441: * Determines whether or not the tokenizer recognizes C-style comments.
442: * If the flag argument is <code>true</code>, this stream tokenizer
443: * recognizes C-style comments. All text between successive
444: * occurrences of <code>/*</code> and <code>*/</code> are discarded.
445: * <p>
446: * If the flag argument is <code>false</code>, then C-style comments
447: * are not treated specially.
448: *
449: * @param flag <code>true</code> indicates to recognize and ignore
450: * C-style comments.
451: */
452: public void slashStarComments(boolean flag) {
453: slashStarCommentsP = flag;
454: }
455:
456: /**
457: * Determines whether or not the tokenizer recognizes C++-style comments.
458: * If the flag argument is <code>true</code>, this stream tokenizer
459: * recognizes C++-style comments. Any occurrence of two consecutive
460: * slash characters (<code>'/'</code>) is treated as the beginning of
461: * a comment that extends to the end of the line.
462: * <p>
463: * If the flag argument is <code>false</code>, then C++-style
464: * comments are not treated specially.
465: *
466: * @param flag <code>true</code> indicates to recognize and ignore
467: * C++-style comments.
468: */
469: public void slashSlashComments(boolean flag) {
470: slashSlashCommentsP = flag;
471: }
472:
473: /**
474: * Determines whether or not word token are automatically lowercased.
475: * If the flag argument is <code>true</code>, then the value in the
476: * <code>sval</code> field is lowercased whenever a word token is
477: * returned (the <code>ttype</code> field has the
478: * value <code>TT_WORD</code> by the <code>nextToken</code> method
479: * of this tokenizer.
480: * <p>
481: * If the flag argument is <code>false</code>, then the
482: * <code>sval</code> field is not modified.
483: *
484: * @param fl <code>true</code> indicates that all word tokens should
485: * be lowercased.
486: * @see java.io.StreamTokenizer#nextToken()
487: * @see java.io.StreamTokenizer#ttype
488: * @see java.io.StreamTokenizer#TT_WORD
489: */
490: public void lowerCaseMode(boolean fl) {
491: forceLower = fl;
492: }
493:
494: /** Read the next character */
495: private int read() throws IOException {
496: if (reader != null)
497: return reader.read();
498: else if (input != null)
499: return input.read();
500: else
501: throw new IllegalStateException();
502: }
503:
504: /**
505: * Parses the next token from the input stream of this tokenizer.
506: * The type of the next token is returned in the <code>ttype</code>
507: * field. Additional information about the token may be in the
508: * <code>nval</code> field or the <code>sval</code> field of this
509: * tokenizer.
510: * <p>
511: * Typical clients of this
512: * class first set up the syntax tables and then sit in a loop
513: * calling nextToken to parse successive tokens until TT_EOF
514: * is returned.
515: *
516: * @return the value of the <code>ttype</code> field.
517: * @exception IOException if an I/O error occurs.
518: * @see java.io.StreamTokenizer#nval
519: * @see java.io.StreamTokenizer#sval
520: * @see java.io.StreamTokenizer#ttype
521: */
522: public int nextToken() throws IOException {
523: if (pushedBack) {
524: pushedBack = false;
525: return ttype;
526: }
527: byte ct[] = ctype;
528: sval = null;
529:
530: int c = peekc;
531: if (c < 0)
532: c = NEED_CHAR;
533: if (c == SKIP_LF) {
534: c = read();
535: if (c < 0)
536: return ttype = TT_EOF;
537: if (c == '\n')
538: c = NEED_CHAR;
539: }
540: if (c == NEED_CHAR) {
541: c = read();
542: if (c < 0)
543: return ttype = TT_EOF;
544: }
545: ttype = c; /* Just to be safe */
546:
547: /* Set peekc so that the next invocation of nextToken will read
548: * another character unless peekc is reset in this invocation
549: */
550: peekc = NEED_CHAR;
551:
552: int ctype = c < 256 ? ct[c] : CT_ALPHA;
553: while ((ctype & CT_WHITESPACE) != 0) {
554: if (c == '\r') {
555: LINENO++;
556: if (eolIsSignificantP) {
557: peekc = SKIP_LF;
558: return ttype = TT_EOL;
559: }
560: c = read();
561: if (c == '\n')
562: c = read();
563: } else {
564: if (c == '\n') {
565: LINENO++;
566: if (eolIsSignificantP) {
567: return ttype = TT_EOL;
568: }
569: }
570: c = read();
571: }
572: if (c < 0)
573: return ttype = TT_EOF;
574: ctype = c < 256 ? ct[c] : CT_ALPHA;
575: }
576:
577: if ((ctype & CT_DIGIT) != 0) {
578: boolean neg = false;
579: if (c == '-') {
580: c = read();
581: if (c != '.' && (c < '0' || c > '9')) {
582: peekc = c;
583: return ttype = '-';
584: }
585: neg = true;
586: }
587: double v = 0;
588: int decexp = 0;
589: int seendot = 0;
590: while (true) {
591: if (c == '.' && seendot == 0)
592: seendot = 1;
593: else if ('0' <= c && c <= '9') {
594: v = v * 10 + (c - '0');
595: decexp += seendot;
596: } else
597: break;
598: c = read();
599: }
600: peekc = c;
601: if (decexp != 0) {
602: double denom = 10;
603: decexp--;
604: while (decexp > 0) {
605: denom *= 10;
606: decexp--;
607: }
608: /* Do one division of a likely-to-be-more-accurate number */
609: v = v / denom;
610: }
611: nval = neg ? -v : v;
612: return ttype = TT_NUMBER;
613: }
614:
615: if ((ctype & CT_ALPHA) != 0) {
616: int i = 0;
617: do {
618: if (i >= buf.length) {
619: char nb[] = new char[buf.length * 2];
620: /* IAI - 15 */
621: CVM.copyCharArray(buf, 0, nb, 0, buf.length);
622: /* IAI - 15 */
623: buf = nb;
624: }
625: buf[i++] = (char) c;
626: c = read();
627: ctype = c < 0 ? CT_WHITESPACE : c < 256 ? ct[c]
628: : CT_ALPHA;
629: } while ((ctype & (CT_ALPHA | CT_DIGIT)) != 0);
630: peekc = c;
631: sval = String.copyValueOf(buf, 0, i);
632: if (forceLower)
633: sval = sval.toLowerCase();
634: return ttype = TT_WORD;
635: }
636:
637: if ((ctype & CT_QUOTE) != 0) {
638: ttype = c;
639: int i = 0;
640: /* Invariants (because \Octal needs a lookahead):
641: * (i) c contains char value
642: * (ii) d contains the lookahead
643: */
644: int d = read();
645: while (d >= 0 && d != ttype && d != '\n' && d != '\r') {
646: if (d == '\\') {
647: c = read();
648: int first = c; /* To allow \377, but not \477 */
649: if (c >= '0' && c <= '7') {
650: c = c - '0';
651: int c2 = read();
652: if ('0' <= c2 && c2 <= '7') {
653: c = (c << 3) + (c2 - '0');
654: c2 = read();
655: if ('0' <= c2 && c2 <= '7' && first <= '3') {
656: c = (c << 3) + (c2 - '0');
657: d = read();
658: } else
659: d = c2;
660: } else
661: d = c2;
662: } else {
663: switch (c) {
664: case 'a':
665: c = 0x7;
666: break;
667: case 'b':
668: c = '\b';
669: break;
670: case 'f':
671: c = 0xC;
672: break;
673: case 'n':
674: c = '\n';
675: break;
676: case 'r':
677: c = '\r';
678: break;
679: case 't':
680: c = '\t';
681: break;
682: case 'v':
683: c = 0xB;
684: break;
685: }
686: d = read();
687: }
688: } else {
689: c = d;
690: d = read();
691: }
692: if (i >= buf.length) {
693: char nb[] = new char[buf.length * 2];
694: /* IAI - 15 */
695: CVM.copyCharArray(buf, 0, nb, 0, buf.length);
696: /* IAI - 15 */
697: buf = nb;
698: }
699: buf[i++] = (char) c;
700: }
701:
702: /* If we broke out of the loop because we found a matching quote
703: * character then arrange to read a new character next time
704: * around; otherwise, save the character.
705: */
706: peekc = (d == ttype) ? NEED_CHAR : d;
707:
708: sval = String.copyValueOf(buf, 0, i);
709: return ttype;
710: }
711:
712: if (c == '/' && (slashSlashCommentsP || slashStarCommentsP)) {
713: c = read();
714: if (c == '*' && slashStarCommentsP) {
715: int prevc = 0;
716: while ((c = read()) != '/' || prevc != '*') {
717: if (c == '\r') {
718: LINENO++;
719: c = read();
720: if (c == '\n') {
721: c = read();
722: }
723: } else {
724: if (c == '\n') {
725: LINENO++;
726: c = read();
727: }
728: }
729: if (c < 0)
730: return ttype = TT_EOF;
731: prevc = c;
732: }
733: return nextToken();
734: } else if (c == '/' && slashSlashCommentsP) {
735: while ((c = read()) != '\n' && c != '\r' && c >= 0)
736: ;
737: peekc = c;
738: return nextToken();
739: } else {
740: /* Now see if it is still a single line comment */
741: if ((ct['/'] & CT_COMMENT) != 0) {
742: while ((c = read()) != '\n' && c != '\r' && c >= 0)
743: ;
744: peekc = c;
745: return nextToken();
746: } else {
747: peekc = c;
748: return ttype = '/';
749: }
750: }
751: }
752:
753: if ((ctype & CT_COMMENT) != 0) {
754: while ((c = read()) != '\n' && c != '\r' && c >= 0)
755: ;
756: peekc = c;
757: return nextToken();
758: }
759:
760: return ttype = c;
761: }
762:
763: /**
764: * Causes the next call to the <code>nextToken</code> method of this
765: * tokenizer to return the current value in the <code>ttype</code>
766: * field, and not to modify the value in the <code>nval</code> or
767: * <code>sval</code> field.
768: *
769: * @see java.io.StreamTokenizer#nextToken()
770: * @see java.io.StreamTokenizer#nval
771: * @see java.io.StreamTokenizer#sval
772: * @see java.io.StreamTokenizer#ttype
773: */
774: public void pushBack() {
775: if (ttype != TT_NOTHING) /* No-op if nextToken() not called */
776: pushedBack = true;
777: }
778:
779: /**
780: * Return the current line number.
781: *
782: * @return the current line number of this stream tokenizer.
783: */
784: public int lineno() {
785: return LINENO;
786: }
787:
788: /**
789: * Returns the string representation of the current stream token and
790: * the line number it occurs on.
791: *
792: * <p>The precise string returned is unspecified, although the following
793: * example can be considered typical:
794: *
795: * <blockquote><pre>Token['a'], line 10</pre></blockquote>
796: *
797: * @return a string representation of the token
798: * @see java.io.StreamTokenizer#nval
799: * @see java.io.StreamTokenizer#sval
800: * @see java.io.StreamTokenizer#ttype
801: */
802: public String toString() {
803: String ret;
804: switch (ttype) {
805: case TT_EOF:
806: ret = "EOF";
807: break;
808: case TT_EOL:
809: ret = "EOL";
810: break;
811: case TT_WORD:
812: ret = sval;
813: break;
814: case TT_NUMBER:
815: ret = "n=" + nval;
816: break;
817: case TT_NOTHING:
818: ret = "NOTHING";
819: break;
820: default: {
821: /*
822: * ttype is the first character of either a quoted string or
823: * is an ordinary character. ttype can definitely not be less
824: * than 0, since those are reserved values used in the previous
825: * case statements
826: */
827: if (ttype < 256 && ((ctype[ttype] & CT_QUOTE) != 0)) {
828: ret = sval;
829: break;
830: }
831:
832: char s[] = new char[3];
833: s[0] = s[2] = '\'';
834: s[1] = (char) ttype;
835: ret = new String(s);
836: break;
837: }
838: }
839: return "Token[" + ret + "], line " + LINENO;
840: }
841:
842: }
|