[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: Line-by-line evaluation or all at once?
On Monday, 16. April 2001 21:18, Etienne M. Gagnon wrote:
> Would you like to implement this? (Maybe you already have). If you
> send me the code (licensed under the LGPL, you retain copyright:-), I
> can take care of integrating it into the official SableCC development
> tree. Just be patient. I won't have much time to devote to SableCC
> until the summer.
>
> Etienne
Hi Etienne,
here is my implementation of the ideas I mentioned. Its just a little
modification of the "parser.txt" file. The changes are marked by
"// 18.04.2001 ml start" and "// 18.04.2001 ml end". I hope you and the
community find them helpful.
The impatient users may replace src/org/sablecc/sablecc/parser.txt with the
new file and run "ant jar".
The parser object can than be reused by providing a new Lexer with new method
parser.setLexer(lexer)
In a project where I need to parse a huge database of chess games I used the
following aproach to analyze the database game by game with the same lexer
(the TResult token marks the end of a game):
chess.game.parser.parser.Parser p =
new chess.game.parser.parser.Parser(
new Lexer(
new PushbackReader(
new InputStreamReader(
new FileInputStream(arg[0])), 1024))
{
boolean resultRead = false;
protected void filter() {
if (resultRead) {
try {
unread(token);
} catch (IOException e) {}
token = new EOF();
resultRead = false;
} else if (token instanceof TResult) {
resultRead = true;
}
}
}
);
Marco
--
mailto:Marco.Ladermann@gmx.de
Öffentlicher Schlüssel auf Nachfrage erhältlich
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* This file is part of SableCC. *
* See the file "LICENSE" for copyright information and the *
* terms and conditions for copying, distribution and *
* modification of SableCC. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
Macro:ParserHeader
/* This file was generated by SableCC (http://www.sablecc.org/). */
package $0$;
import $1$.*;
import $2$.*;
import $3$.*;
import java.util.*;
import java.io.DataInputStream;
import java.io.BufferedInputStream;
import java.io.IOException;
public class Parser
{
public final Analysis ignoredTokens = new AnalysisAdapter();
protected Node node;
// 18.04.2001 ml start
// private final Lexer lexer;
// private final ListIterator stack = new LinkedList().listIterator();
private Lexer lexer; // can be set anew by method setLexer
private ListIterator stack; // initialized in parse method
private final LinkedList parserStack = new LinkedList();
// 18.04.2001 ml end
private int last_shift;
private int last_pos;
private int last_line;
private Token last_token;
private final TokenIndex converter = new TokenIndex();
private final int[] action = new int[2];
private final static int SHIFT = 0;
private final static int REDUCE = 1;
private final static int ACCEPT = 2;
private final static int ERROR = 3;
protected void filter() throws ParserException, LexerException, IOException
{
}
public Parser(Lexer lexer)
{
this.lexer = lexer;
if(actionTable == null)
{
try
{
DataInputStream s = new DataInputStream(
new BufferedInputStream(
Parser.class.getResourceAsStream("parser.dat")));
// read actionTable
int length = s.readInt();
actionTable = new int[length][][];
for(int i = 0; i < actionTable.length; i++)
{
length = s.readInt();
actionTable[i] = new int[length][3];
for(int j = 0; j < actionTable[i].length; j++)
{
for(int k = 0; k < 3; k++)
{
actionTable[i][j][k] = s.readInt();
}
}
}
// read gotoTable
length = s.readInt();
gotoTable = new int[length][][];
for(int i = 0; i < gotoTable.length; i++)
{
length = s.readInt();
gotoTable[i] = new int[length][2];
for(int j = 0; j < gotoTable[i].length; j++)
{
for(int k = 0; k < 2; k++)
{
gotoTable[i][j][k] = s.readInt();
}
}
}
// read errorMessages
length = s.readInt();
errorMessages = new String[length];
for(int i = 0; i < errorMessages.length; i++)
{
length = s.readInt();
StringBuffer buffer = new StringBuffer();
for(int j = 0; j < length; j++)
{
buffer.append(s.readChar());
}
errorMessages[i] = buffer.toString();
}
// read errors
length = s.readInt();
errors = new int[length];
for(int i = 0; i < errors.length; i++)
{
errors[i] = s.readInt();
}
s.close();
}
catch(Exception e)
{
throw new RuntimeException("The file \"parser.dat\" is either missing or corrupted.");
}
}
}
// 18.04.2001 ml start
// method to feed the parser via a new Lexer
public void setLexer(Lexer lexer) {
this.lexer = lexer;
}
// 18.04.2001 ml end
private int goTo(int index)
{
int state = state();
int low = 1;
int high = gotoTable[index].length - 1;
int value = gotoTable[index][0][1];
while(low <= high)
{
int middle = (low + high) / 2;
if(state < gotoTable[index][middle][0])
{
high = middle - 1;
}
else if(state > gotoTable[index][middle][0])
{
low = middle + 1;
}
else
{
value = gotoTable[index][middle][1];
break;
}
}
return value;
}
private void push(int state, Node node, boolean filter) throws ParserException, LexerException, IOException
{
this.node = node;
if(filter)
{
filter();
}
if(!stack.hasNext())
{
stack.add(new State(state, this.node));
return;
}
State s = (State) stack.next();
s.state = state;
s.node = this.node;
}
private int state()
{
State s = (State) stack.previous();
stack.next();
return s.state;
}
private Node pop()
{
return (Node) ((State) stack.previous()).node;
}
private int index(Switchable token)
{
converter.index = -1;
token.apply(converter);
return converter.index;
}
public Start parse() throws ParserException, LexerException, IOException
{
// 18.04.2001 ml start
// get a new Iterator to reuse the stack
stack = parserStack.listIterator();
// 18.04.2001 ml end
push(0, null, false);
List ign = null;
while(true)
{
while(index(lexer.peek()) == -1)
{
if(ign == null)
{
ign = new TypedLinkedList(NodeCast.instance);
}
ign.add(lexer.next());
}
if(ign != null)
{
ignoredTokens.setIn(lexer.peek(), ign);
ign = null;
}
last_pos = lexer.peek().getPos();
last_line = lexer.peek().getLine();
last_token = lexer.peek();
int index = index(lexer.peek());
action[0] = actionTable[state()][0][1];
action[1] = actionTable[state()][0][2];
int low = 1;
int high = actionTable[state()].length - 1;
while(low <= high)
{
int middle = (low + high) / 2;
if(index < actionTable[state()][middle][0])
{
high = middle - 1;
}
else if(index > actionTable[state()][middle][0])
{
low = middle + 1;
}
else
{
action[0] = actionTable[state()][middle][1];
action[1] = actionTable[state()][middle][2];
break;
}
}
switch(action[0])
{
case SHIFT:
push(action[1], lexer.next(), true);
last_shift = action[1];
break;
case REDUCE:
switch(action[1])
{
$
Macro:ParserReduceFilter
case $0$: { Node node = new$0$(); push(goTo($1$), node, true); } break;
$
Macro:ParserReduceNoFilter
case $0$: { Node node = new$0$(); push(goTo($1$), node, false); } break;
$
Macro:ParserParseTail
}
break;
case ACCEPT:
{
EOF node2 = (EOF) lexer.next();
$0$ node1 = ($0$) pop();
Start node = new Start(node1, node2);
return node;
}
case ERROR:
throw new ParserException(last_token,
"[" + last_line + "," + last_pos + "] " +
errorMessages[errors[action[1]]]);
}
}
}
$
Macro:ParserNewHeader
Node new$0$()
{
$
Macro:ParserNewBodyDecl
$0$ node$1$ = ($0$) pop();
$
Macro:ParserNewBodyDeclNull
$0$ node$1$ = null;
$
Macro:ParserNewBodyNew
$0$ node = new $0$(
$
Macro:ParserNewBodyParams
$0$node$1$
$
Macro:ParserNewTail
);
return node;
}
$
Macro:ParserActionHeader
private static int[][][] actionTable;
/* {
$
Macro:ParserActionTail
};*/
$
Macro:ParserGotoHeader
private static int[][][] gotoTable;
/* {
$
Macro:ParserGotoTail
};*/
$
Macro:ParserErrorsHeader
private static String[] errorMessages;
/* {
$
Macro:ParserErrorsTail
};*/
$
Macro:ParserErrorIndexHeader
private static int[] errors;
/* {
$
Macro:ParserErrorIndexTail
};*/
$
Macro:ParserTail
}
$
Macro:TokenIndexHeader
/* This file was generated by SableCC (http://www.sablecc.org/). */
package $0$;
import $1$.*;
import $2$.*;
class TokenIndex extends AnalysisAdapter
{
int index;
$
Macro:TokenIndexBody
public void case$0$($0$ node)
{
index = $1$;
}
$
Macro:TokenIndexTail
public void caseEOF(EOF node)
{
index = $0$;
}
}
$
Macro:ParserException
/* This file was generated by SableCC (http://www.sablecc.org/). */
package $0$;
import $1$.*;
public class ParserException extends Exception
{
Token token;
public ParserException(Token token, String message)
{
super(message);
this.token = token;
}
public Token getToken()
{
return token;
}
}
$
Macro:State
/* This file was generated by SableCC (http://www.sablecc.org/). */
package $0$;
final class State
{
int state;
Object node;
State(int state, Object node)
{
this.state = state;
this.node = node;
}
}
$