01: package fri.patterns.interpreter.parsergenerator;
02:
03: import java.util.List;
04: import java.util.Map;
05: import java.io.PrintStream;
06: import java.io.IOException;
07:
08: /**
09: A Lexer reads input bytes (InputStream) or characters (Reader), until
10: one of its terminals is fulfilled. This happens when the Parser calls
11: <i>getNextToken()</i>. The terminals will be set by the Parser on init.
12: <p>
13: Usage:
14: <pre>
15: SyntaxSeparation separation = new SyntaxSeparation(new Syntax(myRules));
16: LexerBuilder builder = new LexerBuilder(separation.getLexerSyntax(), separation.getIgnoredSymbols());
17: Lexer lexer = builder.getLexer(inputStream);
18: lexer.setTerminals(separation.getTokenSymbols());
19: Token token;
20: do {
21: token = lexer.getNextToken(null);
22: System.err.println("token.symbol="+token.symbol+", text >"+token.text+"<");
23: }
24: while (token.symbol != null && Token.isEpsilon(token) == false);
25: boolean error = token.symbol == null;
26: </pre>
27:
28: @see fri.patterns.interpreter.parsergenerator.lexer.LexerImpl
29: @author (c) 2000, Fritz Ritzberger
30: */
31:
32: public interface Lexer {
33: /**
34: Set the input to be processed by the Lexer.
35: @param text can be String, StringBuffer, File, InputStream, Reader.
36: */
37: public void setInput(Object text) throws IOException;
38:
39: /**
40: Tells the Lexer the terminals (tokens) to scan, called on init. Every terminal is a String
41: that satisfies the facts defined in <i>Token.isTerminal()</i> (EPSILON or delimited by quotes).
42: @param terminals List of String containing all terminals of the parser syntax.
43: */
44: public void setTerminals(List terminals);
45:
46: /**
47: Returns the next token from input. This is done trying to satisy the parser hints, or
48: by using contained character consumers, which are obtained by the lexer strategy.
49: @param tokenSymbols a Map that contains token symbols (in "key" field) expected by the Parser, can be null (slower).
50: @return a Token with a terminal symbol and its instance text, or a Token with null symbol for error.
51: */
52: public Token getNextToken(Map tokenSymbols) throws IOException;
53:
54: /** Reset the Lexer for another pass. */
55: public void clear();
56:
57: /**
58: * A way to receive every parsing syntax Token the Lexer reads, even it is ignored.
59: * The implementer can install itself by calling <i>addTokenListener()</i>.
60: */
61: public interface TokenListener {
62: /**
63: * The implementer receives every token the lexer reads, even if it is ignored.
64: * @param token the Token that was successfully scanned from input.
65: * @param ignored true when this Token will be thrown away (not passed to Parser), else false.
66: */
67: public void tokenReceived(Token token, boolean ignored);
68: }
69:
70: /**
71: * Installs a TokenListener that wants to know about every read Token, even it is ignored.
72: * @param listener the Lexer.Listener implementation to install.
73: */
74: public void addTokenListener(Lexer.TokenListener listener);
75:
76: /**
77: * Removes a TokenListener from this Lexer.
78: * @param listener the Lexer.Listener implementation to remove.
79: */
80: public void removeTokenListener(Lexer.TokenListener listener);
81:
82: // debug methods
83:
84: /** Dump the current text and the scan position. */
85: public void dump(PrintStream out);
86:
87: /** Turn on and off debug mode. */
88: public void setDebug(boolean debug);
89:
90: }
|