wurstscript / WurstScript / 265

Committed 29 Sep 2025 09:00AM UTC coverage: 62.244% (+0.02%) from 62.222%

Build # 265

Build Type

Pull #1096

circleci

Committed by

Frotty

Commit Message

restore determinism, fix tests

Pull Request Pull Request #1096: Perf improvements

Run Details

17480 of 28083 relevant lines covered (62.24%)

0.62 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.45

de.peeeq.wurstscript/src/main/java/de/peeeq/wurstscript/jurst/ExtendedJurstLexer.java

package de.peeeq.wurstscript.jurst;

import de.peeeq.wurstscript.WLogger;
import de.peeeq.wurstscript.jurst.antlr.JurstLexer;
import de.peeeq.wurstscript.jurst.antlr.JurstParser;
import de.peeeq.wurstscript.utils.LineOffsets;
import org.antlr.v4.runtime.*;
import org.antlr.v4.runtime.misc.Pair;
import org.eclipse.jdt.annotation.Nullable;

import java.util.ArrayDeque;
import java.util.LinkedList;
import java.util.Queue;

public class ExtendedJurstLexer implements TokenSource {

    private final JurstLexer orig;
    private final Queue<Token> nextTokens = new ArrayDeque<>();
    private State state = State.INIT;
    private @Nullable Token eof = null;
    private @Nullable Token firstNewline;
    private final LineOffsets lineOffsets = new LineOffsets();
    public StringBuilder debugSb = new StringBuilder();
    private final boolean debug = false;
    private final Pair<TokenSource, CharStream> sourcePair;
    private boolean isWurst = false;


    enum State {
        INIT, WRAP_CHAR, NEWLINES, BEGIN_LINE
    }


    public ExtendedJurstLexer(CharStream input) {
        orig = new JurstLexer(input);
        sourcePair = new Pair<>(orig, input);
    }


    @Override
    public int getCharPositionInLine() {
        return orig.getCharPositionInLine();
    }

    @Override
    public CharStream getInputStream() {
        return orig.getInputStream();
    }

    @Override
    public int getLine() {
        return orig.getLine();
    }

    @Override
    public String getSourceName() {
        return orig.getSourceName();
    }

    @Override
    public TokenFactory<?> getTokenFactory() {
        return orig.getTokenFactory();
    }

    @Override
    public Token nextToken() {
        Token t = nextTokenIntern();

        debugSb.append(t.getText()).append(" ");
        if (debug) WLogger.info("                new token: " + t);
        return t;
    }


    private Token nextTokenIntern() {
        if (!nextTokens.isEmpty()) {
            return nextTokens.poll();
        }

        Token eof_local = eof;
        if (eof_local != null) {
            return makeToken(JurstParser.EOF, "$EOF", eof_local.getStartIndex(), eof_local.getStopIndex());
        }


        for (; ; ) {
            Token token = orig.nextToken();

            if (debug) WLogger.info("orig token = " + token);

            if (token == null) {
                continue;
            }

            if (isWurst) {
                if (token.getType() == JurstParser.ENDPACKAGE
                        || token.getType() == JurstParser.ENDLIBRARY
                        || token.getType() == JurstParser.ENDSCOPE) {
                    isWurst = false;
                }
            } else {
                if (token.getType() == JurstParser.PACKAGE
                        || token.getType() == JurstParser.LIBRARY
                        || token.getType() == JurstParser.SCOPE) {
                    isWurst = true;
                } else if (isWurstOnlyKeyword(token)) {
                    token = makeToken(JurstParser.ID, token.getText(), token.getStartIndex(), token.getStopIndex());
                    assert token != null;
                } else if (token.getType() == JurstParser.HOTDOC_COMMENT) {
                    continue;
                }
            }


            if (token.getType() == JurstParser.NL) {
                lineOffsets.set(token.getLine(), token.getStopIndex());
            }

            if (token.getType() == JurstParser.EOF) {
                // at EOF close all blocks and return an extra newline
                eof = token;
                if (isWurst) {
                    // if inside wurst, add a closing 'endpackage' and a newline
                    nextTokens.add(makeToken(JurstParser.ENDPACKAGE, "endpackage", token.getStartIndex(), token.getStopIndex()));
                    nextTokens.add(makeToken(JurstParser.NL, "$NL", token.getStartIndex(), token.getStopIndex()));
                }
                // add a single newline
                return makeToken(JurstParser.NL, "$NL", token.getStartIndex(), token.getStopIndex());
            }

            if (token.getType() == JurstParser.ID
                    && token.getText().equals("debug")
                    && (state == State.NEWLINES)) {
                // ignore 'debug' at beginning of line
                continue;
            }


            switch (state) {
                case INIT:
                    if (isWrapCharEndLine(token.getType())) {
                        state(State.WRAP_CHAR);
                        return token;
                    } else if (token.getType() == JurstParser.NL) {
                        firstNewline = token;
                        state(State.NEWLINES);
                        continue;
                    }
                    return token;
                case NEWLINES:
                    if (isWrapCharBeginLine(token.getType())) {
                        // ignore all the newlines when a wrap char comes after newlines
                        state(State.WRAP_CHAR);
                        return token;
                    } else if (token.getType() == JurstParser.NL) {
                        continue;
                    } else {
                        nextTokens.add(token);
                        state(State.INIT);
                        return firstNewline;
                    }
                case WRAP_CHAR:
                    if (isWrapCharEndLine(token.getType())) {
                        return token;
                    } else if (token.getType() == JurstParser.NL) {
                        // ignore newlines after wrap char
                        continue;
                    } else {
                        state(State.INIT);
                        return token;
                    }
                case BEGIN_LINE:
                    if (token.getType() == JurstParser.NL) {
                        state(State.NEWLINES);
                    } else if (isWrapCharBeginLine(token.getType())) {
                        // ignore all the newlines when a wrap char comes after newlines
                        state(State.WRAP_CHAR);
                        return token;
                    } else {
                        state(State.INIT);
                        nextTokens.add(token);
                        return firstNewline;
                    }
            }
        }
    }


    private boolean isWurstOnlyKeyword(Token token) {
        switch (token.getType()) {
            case JurstParser.VAR:
            case JurstParser.LET:
                // TODO other tokens
                return true;
            default:
                return false;
        }
    }


    private void state(State s) {
        if (debug) WLogger.info("state " + state + " -> " + s);
        state = s;
    }


    private boolean isWrapChar(int type) {
        switch (type) {
//                case JurstParser.PAREN_LEFT:
//                case JurstParser.BRACKET_LEFT:
            case JurstParser.COMMA:
            case JurstParser.PLUS:
            case JurstParser.MULT:
            case JurstParser.MINUS:
            case JurstParser.DIV:
            case JurstParser.DIV_REAL:
            case JurstParser.MOD:
            case JurstParser.MOD_REAL:
            case JurstParser.AND:
            case JurstParser.OR:

                return true;
        }
        return false;
    }


    private boolean isWrapCharEndLine(int type) {
        switch (type) {
            case JurstParser.PAREN_LEFT:
            case JurstParser.BRACKET_LEFT:
                return true;
            default:
                return isWrapChar(type);
        }
    }


    private boolean isWrapCharBeginLine(int type) {
        switch (type) {
            case JurstParser.PAREN_RIGHT:
            case JurstParser.BRACKET_RIGHT:
            case JurstParser.DOT:
            case JurstParser.DOTDOT:
                return true;
            default:
                return isWrapChar(type);
        }
    }


    private Token makeToken(int type, String text, int start, int stop) {
        Pair<TokenSource, CharStream> source = sourcePair;
        int channel = 0;
        CommonToken t = new CommonToken(source, type, channel, start, stop);
        return t;
    }

    @Override
    public void setTokenFactory(@SuppressWarnings("null") TokenFactory<?> factory) {
        orig.setTokenFactory(factory);
    }


    public LineOffsets getLineOffsets() {
        return lineOffsets;
    }


    public void addErrorListener(ANTLRErrorListener listener) {
        orig.addErrorListener(listener);
    }
}

1	package de.peeeq.wurstscript.jurst;
2
3	import de.peeeq.wurstscript.WLogger;
4	import de.peeeq.wurstscript.jurst.antlr.JurstLexer;
5	import de.peeeq.wurstscript.jurst.antlr.JurstParser;
6	import de.peeeq.wurstscript.utils.LineOffsets;
7	import org.antlr.v4.runtime.*;
8	import org.antlr.v4.runtime.misc.Pair;
9	import org.eclipse.jdt.annotation.Nullable;
10
11	import java.util.ArrayDeque;
12	import java.util.LinkedList;
13	import java.util.Queue;
14
15	public class ExtendedJurstLexer implements TokenSource {	1✔
16
17	private final JurstLexer orig;
18	private final Queue<Token> nextTokens = new ArrayDeque<>();	1✔
19	private State state = State.INIT;	1✔
20	private @Nullable Token eof = null;	1✔
21	private @Nullable Token firstNewline;
22	private final LineOffsets lineOffsets = new LineOffsets();	1✔
23	public StringBuilder debugSb = new StringBuilder();	1✔
24	private final boolean debug = false;	1✔
25	private final Pair<TokenSource, CharStream> sourcePair;
26	private boolean isWurst = false;	1✔
27
28
29	enum State {	1✔
30	INIT, WRAP_CHAR, NEWLINES, BEGIN_LINE	1✔
31	}
32
33
34	public ExtendedJurstLexer(CharStream input) {	1✔
35	orig = new JurstLexer(input);	1✔
36	sourcePair = new Pair<>(orig, input);	1✔
37	}	1✔
38
39
40	@Override
41	public int getCharPositionInLine() {
42	return orig.getCharPositionInLine();	×
43	}
44
45	@Override
46	public CharStream getInputStream() {
47	return orig.getInputStream();	×
48	}
49
50	@Override
51	public int getLine() {
52	return orig.getLine();	×
53	}
54
55	@Override
56	public String getSourceName() {
57	return orig.getSourceName();	×
58	}
59
60	@Override
61	public TokenFactory<?> getTokenFactory() {
62	return orig.getTokenFactory();	×
63	}
64
65	@Override
66	public Token nextToken() {
67	Token t = nextTokenIntern();	1✔
68
69	debugSb.append(t.getText()).append(" ");	1✔
70	if (debug) WLogger.info(" new token: " + t);
71	return t;	1✔
72	}
73
74
75	private Token nextTokenIntern() {
76	if (!nextTokens.isEmpty()) {	1✔
77	return nextTokens.poll();	1✔
78	}
79
80	Token eof_local = eof;	1✔
81	if (eof_local != null) {	1✔
82	return makeToken(JurstParser.EOF, "$EOF", eof_local.getStartIndex(), eof_local.getStopIndex());	1✔
83	}
84
85
86	for (; ; ) {
87	Token token = orig.nextToken();	1✔
88
89	if (debug) WLogger.info("orig token = " + token);
90
91	if (token == null) {	1✔
92	continue;	×
93	}
94
95	if (isWurst) {	1✔
96	if (token.getType() == JurstParser.ENDPACKAGE	1✔
97	\|\| token.getType() == JurstParser.ENDLIBRARY	1✔
98	\|\| token.getType() == JurstParser.ENDSCOPE) {	1✔
99	isWurst = false;	1✔
100	}
101	} else {
102	if (token.getType() == JurstParser.PACKAGE	1✔
103	\|\| token.getType() == JurstParser.LIBRARY	1✔
104	\|\| token.getType() == JurstParser.SCOPE) {	1✔
105	isWurst = true;	1✔
106	} else if (isWurstOnlyKeyword(token)) {	1✔
107	token = makeToken(JurstParser.ID, token.getText(), token.getStartIndex(), token.getStopIndex());	×
108	assert token != null;	×
109	} else if (token.getType() == JurstParser.HOTDOC_COMMENT) {	1✔
110	continue;	×
111	}
112	}
113
114
115	if (token.getType() == JurstParser.NL) {	1✔
116	lineOffsets.set(token.getLine(), token.getStopIndex());	1✔
117	}
118
119	if (token.getType() == JurstParser.EOF) {	1✔
120	// at EOF close all blocks and return an extra newline
121	eof = token;	1✔
122	if (isWurst) {	1✔
123	// if inside wurst, add a closing 'endpackage' and a newline
124	nextTokens.add(makeToken(JurstParser.ENDPACKAGE, "endpackage", token.getStartIndex(), token.getStopIndex()));	1✔
125	nextTokens.add(makeToken(JurstParser.NL, "$NL", token.getStartIndex(), token.getStopIndex()));	1✔
126	}
127	// add a single newline
128	return makeToken(JurstParser.NL, "$NL", token.getStartIndex(), token.getStopIndex());	1✔
129	}
130
131	if (token.getType() == JurstParser.ID	1✔
132	&& token.getText().equals("debug")	1✔
133	&& (state == State.NEWLINES)) {
134	// ignore 'debug' at beginning of line
135	continue;	×
136	}
137
138
139	switch (state) {	1✔
140	case INIT:
141	if (isWrapCharEndLine(token.getType())) {	1✔
142	state(State.WRAP_CHAR);	1✔
143	return token;	1✔
144	} else if (token.getType() == JurstParser.NL) {	1✔
145	firstNewline = token;	1✔
146	state(State.NEWLINES);	1✔
147	continue;	1✔
148	}
149	return token;	1✔
150	case NEWLINES:
151	if (isWrapCharBeginLine(token.getType())) {	1✔
152	// ignore all the newlines when a wrap char comes after newlines
153	state(State.WRAP_CHAR);	×
154	return token;	×
155	} else if (token.getType() == JurstParser.NL) {	1✔
156	continue;	×
157	} else {
158	nextTokens.add(token);	1✔
159	state(State.INIT);	1✔
160	return firstNewline;	1✔
161	}
162	case WRAP_CHAR:
163	if (isWrapCharEndLine(token.getType())) {	1✔
164	return token;	×
165	} else if (token.getType() == JurstParser.NL) {	1✔
166	// ignore newlines after wrap char
167	continue;	1✔
168	} else {
169	state(State.INIT);	1✔
170	return token;	1✔
171	}
172	case BEGIN_LINE:
173	if (token.getType() == JurstParser.NL) {	×
174	state(State.NEWLINES);	×
175	} else if (isWrapCharBeginLine(token.getType())) {	×
176	// ignore all the newlines when a wrap char comes after newlines
177	state(State.WRAP_CHAR);	×
178	return token;	×
179	} else {
180	state(State.INIT);	×
181	nextTokens.add(token);	×
182	return firstNewline;	×
183	}
184	}
185	}	×
186	}
187
188
189	private boolean isWurstOnlyKeyword(Token token) {
190	switch (token.getType()) {	1✔
191	case JurstParser.VAR:
192	case JurstParser.LET:
193	// TODO other tokens
194	return true;	×
195	default:
196	return false;	1✔
197	}
198	}
199
200
201	private void state(State s) {
202	if (debug) WLogger.info("state " + state + " -> " + s);
203	state = s;	1✔
204	}	1✔
205
206
207	private boolean isWrapChar(int type) {
208	switch (type) {	1✔
209	// case JurstParser.PAREN_LEFT:
210	// case JurstParser.BRACKET_LEFT:
211	case JurstParser.COMMA:
212	case JurstParser.PLUS:
213	case JurstParser.MULT:
214	case JurstParser.MINUS:
215	case JurstParser.DIV:
216	case JurstParser.DIV_REAL:
217	case JurstParser.MOD:
218	case JurstParser.MOD_REAL:
219	case JurstParser.AND:
220	case JurstParser.OR:
221
222	return true;	1✔
223	}
224	return false;	1✔
225	}
226
227
228	private boolean isWrapCharEndLine(int type) {
229	switch (type) {	1✔
230	case JurstParser.PAREN_LEFT:
231	case JurstParser.BRACKET_LEFT:
232	return true;	1✔
233	default:
234	return isWrapChar(type);	1✔
235	}
236	}
237
238
239	private boolean isWrapCharBeginLine(int type) {
240	switch (type) {	1✔
241	case JurstParser.PAREN_RIGHT:
242	case JurstParser.BRACKET_RIGHT:
243	case JurstParser.DOT:
244	case JurstParser.DOTDOT:
245	return true;	×
246	default:
247	return isWrapChar(type);	1✔
248	}
249	}
250
251
252	private Token makeToken(int type, String text, int start, int stop) {
253	Pair<TokenSource, CharStream> source = sourcePair;	1✔
254	int channel = 0;	1✔
255	CommonToken t = new CommonToken(source, type, channel, start, stop);	1✔
256	return t;	1✔
257	}
258
259	@Override
260	public void setTokenFactory(@SuppressWarnings("null") TokenFactory<?> factory) {
261	orig.setTokenFactory(factory);	×
262	}	×
263
264
265	public LineOffsets getLineOffsets() {
266	return lineOffsets;	1✔
267	}
268
269
270	public void addErrorListener(ANTLRErrorListener listener) {
271	orig.addErrorListener(listener);	1✔
272	}	1✔
273	}

wurstscript / WurstScript / 265

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous