001: /*
002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
003: *
004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
005: *
006: * The contents of this file are subject to the terms of either the GNU
007: * General Public License Version 2 only ("GPL") or the Common
008: * Development and Distribution License("CDDL") (collectively, the
009: * "License"). You may not use this file except in compliance with the
010: * License. You can obtain a copy of the License at
011: * http://www.netbeans.org/cddl-gplv2.html
012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
013: * specific language governing permissions and limitations under the
014: * License. When distributing the software, include this License Header
015: * Notice in each file and include the License file at
016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
017: * particular file as subject to the "Classpath" exception as provided
018: * by Sun in the GPL Version 2 section of the License file that
019: * accompanied this code. If applicable, add the following below the
020: * License Header, with the fields enclosed by brackets [] replaced by
021: * your own identifying information:
022: * "Portions Copyrighted [year] [name of copyright owner]"
023: *
024: * Contributor(s):
025: *
026: * The Original Software is NetBeans. The Initial Developer of the Original
027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
028: * Microsystems, Inc. All Rights Reserved.
029: *
030: * If you wish your version of this file to be governed by only the CDDL
031: * or only the GPL Version 2, indicate your decision by adding
032: * "[Contributor] elects to include this software in this distribution
033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
034: * single choice of license, a recipient has the option to distribute
035: * your version of this file under either the CDDL, the GPL Version 2 or
036: * to extend the choice of license to its licensees as provided above.
037: * However, if you add GPL Version 2 code and therefore, elected the GPL
038: * Version 2 license, then the option applies only if the new code is
039: * made subject to such option by the copyright holder.
040: */
041:
042: package org.netbeans.modules.lexer.demo;
043:
044: import java.util.List;
045: import java.util.ArrayList;
046: import java.util.Arrays;
047: import javax.swing.text.BadLocationException;
048: import javax.swing.text.Document;
049: import javax.swing.text.PlainDocument;
050: import javax.swing.event.DocumentListener;
051: import javax.swing.event.DocumentEvent;
052: import org.netbeans.api.lexer.Language;
053: import org.netbeans.api.lexer.LexerInput;
054: import org.netbeans.api.lexer.TokenUpdater;
055: import org.netbeans.api.lexer.Token;
056: import org.netbeans.api.lexer.Lexer;
057: import org.netbeans.spi.lexer.util.LexerTestDescription;
058: import org.netbeans.spi.lexer.util.LexerUtilities;
059:
060: /**
061: * Random test that helps to test lexer correctness.
062: * Document is created and updated by subsequent modifications.
063: * After each modification the token updater updates token elements
064: * of the document and compares them to another token list
065: * created by batch lexing of the entire document.
066: * <BR>If the two token lists do not match it means that the lexer
067: * must be fixed otherwise it would not function well
068: * in the incremental setting.
069: * <BR><CODE>createLexer()</CODE> can be overriden if necessary.
070: *
071: * @author Miloslav Metelka
072: * @version 1.00
073: */
074:
075: public class LexerRandomTest extends DemoTokenUpdater {
076:
077: private LexerTestDescription td;
078:
079: private int debugLevel;
080:
081: public LexerRandomTest(LexerTestDescription td,
082: boolean maintainLookbacks) {
083: super (new PlainDocument(), td.getLanguage(), maintainLookbacks);
084:
085: this .td = td;
086: debugLevel = td.getDebugLevel();
087: }
088:
089: public void test() {
090: LexerTestDescription.TestRound[] rounds = td.getTestRounds();
091:
092: // Fill in insertItems list
093: List insertItems = new ArrayList();
094: LexerTestDescription.TestChar[] testChars = td.getTestChars();
095: if (testChars != null) {
096: insertItems.addAll(Arrays.asList(testChars));
097: }
098: LexerTestDescription.TestCharInterval[] testCharIntervals = td
099: .getTestCharIntervals();
100: if (testCharIntervals != null) {
101: insertItems.addAll(Arrays.asList(testCharIntervals));
102: }
103: LexerTestDescription.TestString[] testStrings = td
104: .getTestStrings();
105: if (testStrings != null) {
106: insertItems.addAll(Arrays.asList(testStrings));
107: }
108:
109: // Compute total insertItemsRatioSum
110: double insertItemsRatioSum = 0;
111: int insertItemsLength = insertItems.size();
112: for (int i = 0; i < insertItemsLength; i++) {
113: insertItemsRatioSum += getInsertRatio(insertItems.get(i));
114: }
115:
116: int maxDocumentLength = td.getMaxDocumentLength();
117:
118: Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
119: System.err.println("Test started ...");
120: long tm = System.currentTimeMillis();
121:
122: for (int i = 0; i < rounds.length; i++) {
123: LexerTestDescription.TestRound r = rounds[i];
124:
125: System.out.println("Test round: " + r);
126:
127: int operationCount = r.getOperationCount();
128: double insertRatio = r.getInsertRatio();
129: int maxInsertLength = r.getMaxInsertLength();
130: double removeRatio = r.getRemoveRatio();
131: int maxRemoveLength = r.getMaxRemoveLength();
132:
133: double operationRatioSum = r.getInsertRatio()
134: + r.getRemoveRatio();
135:
136: // Test string for not yet completed insert or null if no incomplete string
137: String incompleteString = null;
138: int incompleteStringRemainLength = 0;
139: int incompleteStringInsertOffset = 0;
140:
141: // force remove after extra insert due to previous forced insert of incomplete string
142: boolean forceRemove = false;
143:
144: int moduloOperationCount = operationCount / 10;
145: while (--operationCount >= 0) {
146: double operationRatio = Math.random()
147: * operationRatioSum;
148: operationRatio -= insertRatio;
149: if (forceRemove
150: || getDocument().getLength() > maxDocumentLength) {
151: operationRatio = 0;
152: }
153:
154: if (operationRatio < 0 || incompleteString != null) { // do insert
155: StringBuffer insertBuffer = new StringBuffer();
156: int insertLength = (int) (maxInsertLength * Math
157: .random()) + 1;
158: int insertOffset = (int) ((getDocument()
159: .getLength() + 1) * Math.random());
160: if (incompleteString != null && operationRatio >= 0) { // would be remove normally
161: insertLength = Math.min(insertLength,
162: incompleteStringRemainLength);
163: }
164:
165: if (incompleteString != null) {
166: insertOffset = incompleteStringInsertOffset;
167:
168: int isLen = incompleteString.length();
169:
170: if (incompleteStringRemainLength <= insertLength) {
171: insertLength -= incompleteStringRemainLength;
172: insertBuffer
173: .append(incompleteString
174: .substring(isLen
175: - incompleteStringRemainLength));
176:
177: insertLength -= incompleteStringRemainLength;
178: incompleteString = null;
179: incompleteStringRemainLength = 0;
180:
181: } else { // incomplete string is longer than insert length
182: insertBuffer
183: .append(incompleteString
184: .substring(
185: isLen
186: - incompleteStringRemainLength,
187: isLen
188: - incompleteStringRemainLength
189: + insertLength));
190:
191: incompleteStringRemainLength -= insertLength;
192: insertLength = 0;
193: }
194: }
195:
196: while (insertLength > 0) {
197: double insertItemsRatio = Math.random()
198: * insertItemsRatioSum;
199: for (int j = 0; j < insertItemsLength; j++) {
200: Object item = insertItems.get(j);
201: insertItemsRatio -= getInsertRatio(item);
202: if (insertItemsRatio < 0) {
203: // Perform insert
204: if (item instanceof LexerTestDescription.TestChar) {
205: LexerTestDescription.TestChar tc = (LexerTestDescription.TestChar) item;
206: insertBuffer.append(tc.getChar());
207: insertLength--;
208:
209: } else if (item instanceof LexerTestDescription.TestCharInterval) {
210: LexerTestDescription.TestCharInterval tci = (LexerTestDescription.TestCharInterval) item;
211: insertBuffer.append((char) (tci
212: .getChar() + ((tci
213: .getLastChar()
214: - tci.getChar() + 1) * Math
215: .random())));
216: insertLength--;
217:
218: } else if (item instanceof LexerTestDescription.TestString) {
219: LexerTestDescription.TestString ts = (LexerTestDescription.TestString) item;
220: String s = ts.getString();
221: int sLen = s.length();
222: if (sLen <= insertLength) {
223: insertBuffer.append(s);
224: insertLength -= insertLength;
225:
226: } else { // sLen > insertLength
227: insertBuffer.append(s
228: .substring(0,
229: insertLength));
230: incompleteString = s;
231: incompleteStringRemainLength = sLen
232: - insertLength;
233: insertLength = 0;
234: }
235:
236: } else { // unsupported
237: throw new IllegalStateException();
238: }
239:
240: break;
241: }
242: }
243: }
244:
245: String text = insertBuffer.toString();
246: try {
247: if (debugLevel > 0) {
248: System.err.print("+Insert");
249:
250: if (debugLevel >= 2) { // debug text
251: System.err.print(" \""
252: + LexerUtilities.toSource(text)
253: + '"');
254: }
255:
256: System.err.print(" at offset="
257: + insertOffset + "("
258: + getDocument().getLength()
259: + "), length=" + text.length());
260: }
261:
262: getDocument().insertString(insertOffset, text,
263: null);
264:
265: if (debugLevel >= 3) { // debug doc text
266: System.err
267: .print(", docText=\""
268: + LexerUtilities
269: .toSource(getDocText(getDocument()))
270: + "\"");
271: }
272:
273: incompleteStringInsertOffset = insertOffset
274: + text.length();
275: } catch (BadLocationException e) {
276: throw new IllegalStateException(e.toString());
277: }
278:
279: } else { // not insert
280: operationRatio -= removeRatio;
281: if (operationRatio < 0 || forceRemove) { // do remove
282: forceRemove = false;
283:
284: int removeLength = (int) (maxRemoveLength * Math
285: .random()) + 1;
286: removeLength = Math.min(removeLength,
287: getDocument().getLength());
288:
289: int removeOffset = (int) ((getDocument()
290: .getLength()
291: - removeLength + 1) * Math.random());
292: try {
293: if (debugLevel > 0) {
294: System.err.print("-Remove");
295:
296: if (debugLevel >= 2) {
297: String text = getDocument()
298: .getText(removeOffset,
299: removeLength);
300: System.err.print(" \""
301: + LexerUtilities
302: .toSource(text)
303: + '"');
304: }
305:
306: System.err.print(" at offset="
307: + removeOffset + "("
308: + getDocument().getLength()
309: + "), length=" + removeLength);
310:
311: }
312:
313: getDocument().remove(removeOffset,
314: removeLength);
315:
316: if (debugLevel >= 3) {
317: System.err
318: .print(", docText=\""
319: + LexerUtilities
320: .toSource(getDocText(getDocument()))
321: + "\"");
322: }
323: } catch (BadLocationException e) {
324: throw new IllegalStateException(e
325: .toString());
326: }
327: }
328: }
329:
330: // Test the correctness of tokens produced by algorithm
331: // by true batch lexing with a fresh instance of lexer
332: int tokenIndex = 0; // need to reference if exception thrown
333: try {
334: ArrayList lbList = new ArrayList();
335: String docText;
336: try {
337: docText = getDocument().getText(0,
338: getDocument().getLength());
339: } catch (BadLocationException e) {
340: throw new IllegalStateException(e.toString());
341: }
342: LexerInput input = new StringLexerInput(docText);
343: Lexer lexer = getLanguage().createLexer();
344: lexer.restart(input, null);
345:
346: int shift = relocate(0);
347: if (shift != 0) {
348: throw new IllegalStateException(
349: "Invalid relocate shift=" + shift);
350: }
351:
352: int tokenTotalLength = 0;
353: int lbOffset = 0;
354: while (true) {
355: Token token = lexer.nextToken();
356: if (token != null) {
357: Token itToken = next();
358: checkTokensEqual(itToken, token);
359: int tokenLength = token.getText().length();
360: tokenTotalLength += tokenLength;
361:
362: int la = getLookahead();
363: if (input.getReadLookahead() != la) {
364: throw new IllegalStateException(
365: "incremental environment lookahead="
366: + la
367: + ", batch lexer lookahead="
368: + input
369: .getReadLookahead());
370: }
371:
372: Object state = getState();
373: Object lexerState = lexer.getState();
374: if (!((state == null && lexerState == null) || (state != null && state
375: .equals(lexerState)))) {
376: throw new IllegalStateException(
377: "States do not match incremental environment lexer-state="
378: + state
379: + ", batch lexer state="
380: + lexerState);
381: }
382:
383: lbList.add(new Integer(tokenLength));
384: lbList.add(new Integer(la));
385:
386: while (lbList.size() > 0) {
387: int tlen = ((Integer) lbList.get(0))
388: .intValue();
389: int tla = ((Integer) lbList.get(1))
390: .intValue();
391: if (lbOffset + tlen + tla <= tokenTotalLength) {
392: lbOffset += tlen;
393: lbList.remove(0); // remove len
394: lbList.remove(0); // remove la
395: } else {
396: break;
397: }
398: }
399:
400: int lb = getLookback();
401: if (lb >= 0) {
402: if (lb != lbList.size() / 2) {
403: throw new IllegalStateException(
404: "iterator-lb="
405: + lb
406: + ", lexer-lb="
407: + (lbList.size() / 2));
408: }
409: }
410:
411: tokenIndex++;
412:
413: } else { // no more tokens
414: if (hasNext()) {
415: throw new IllegalStateException();
416: }
417: if (tokenTotalLength != docText.length()) {
418: throw new IllegalStateException();
419: }
420: break;
421: }
422: }
423:
424: if (debugLevel > 0) {
425: System.err.println(", " + tokenIndex
426: + " tokens");
427: }
428:
429: if (operationCount > 0
430: && (operationCount % moduloOperationCount) == 0) {
431: System.err.println(operationCount
432: + " operations remain. docLength="
433: + getDocument().getLength()
434: + ", tokenCount=" + tokenIndex);
435: }
436:
437: } catch (RuntimeException e) {
438: try {
439: System.err
440: .println("\n\nException thrown - document text=\""
441: + LexerUtilities
442: .toSource(getDocument()
443: .getText(
444: 0,
445: getDocument()
446: .getLength()))
447: + "\"\ntokens:\n"
448: + allTokensToString()
449: + "tokenIndex=" + tokenIndex);
450:
451: } catch (BadLocationException ex) {
452: throw new IllegalStateException(ex.toString());
453: }
454: throw e; // rethrow
455: }
456:
457: } // while (--operationCount >= 0)
458: }
459:
460: System.err.println("Test finished in "
461: + (System.currentTimeMillis() - tm) / 1000
462: + " seconds.");
463: }
464:
465: private static double getInsertRatio(Object o) {
466: if (o instanceof LexerTestDescription.TestChar) {
467: return ((LexerTestDescription.TestChar) o).getInsertRatio();
468: } else if (o instanceof LexerTestDescription.TestCharInterval) {
469: return ((LexerTestDescription.TestCharInterval) o)
470: .getInsertRatio();
471: } else if (o instanceof LexerTestDescription.TestString) {
472: return ((LexerTestDescription.TestString) o)
473: .getInsertRatio();
474: } else {
475: throw new IllegalArgumentException();
476: }
477: }
478:
479: private void dump() {
480: System.err.println("Dump of token iterator\n"
481: + allTokensToString());
482: }
483:
484: private static void checkTokensEqual(Token t1, Token t2) {
485: if (t1.getId() != t2.getId()) {
486: throw new IllegalStateException("t1.id=" + t1.getId()
487: + ", t2.id=" + t2.getId());
488: }
489:
490: CharSequence t1Text = t1.getText();
491: CharSequence t2Text = t2.getText();
492: if (t1Text.length() != t2Text.length()) {
493: throw new IllegalStateException("t1=\""
494: + LexerUtilities.toSource(t1Text.toString())
495: + "\", t2=\""
496: + LexerUtilities.toSource(t2Text.toString()) + '"');
497: }
498: for (int i = t1Text.length() - 1; i >= 0; i--) {
499: if (t1Text.charAt(i) != t2Text.charAt(i)) {
500: throw new IllegalStateException();
501: }
502: }
503: }
504:
505: private static String getDocText(Document doc) {
506: try {
507: return doc.getText(0, doc.getLength());
508: } catch (BadLocationException e) {
509: throw new IllegalStateException();
510: }
511: }
512:
513: public static void main(String[] args) {
514: try {
515: if (args.length == 0) {
516: System.err.println("Usage: java "
517: + LexerRandomTest.class.getName()
518: + " <test-description-class-name>");
519: System.exit(1);
520: }
521:
522: Class langCls = Class.forName(args[0]);
523: LexerTestDescription td = (LexerTestDescription) langCls
524: .newInstance();
525:
526: new LexerRandomTest(td, false).test();
527:
528: } catch (Exception ex) {
529: ex.printStackTrace();
530: }
531: }
532:
533: }
|