0001: /*
0002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
0003: *
0004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
0005: *
0006: * The contents of this file are subject to the terms of either the GNU
0007: * General Public License Version 2 only ("GPL") or the Common
0008: * Development and Distribution License("CDDL") (collectively, the
0009: * "License"). You may not use this file except in compliance with the
0010: * License. You can obtain a copy of the License at
0011: * http://www.netbeans.org/cddl-gplv2.html
0012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
0013: * specific language governing permissions and limitations under the
0014: * License. When distributing the software, include this License Header
0015: * Notice in each file and include the License file at
0016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
0017: * particular file as subject to the "Classpath" exception as provided
0018: * by Sun in the GPL Version 2 section of the License file that
0019: * accompanied this code. If applicable, add the following below the
0020: * License Header, with the fields enclosed by brackets [] replaced by
0021: * your own identifying information:
0022: * "Portions Copyrighted [year] [name of copyright owner]"
0023: *
0024: * Contributor(s):
0025: *
0026: * The Original Software is NetBeans. The Initial Developer of the Original
0027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
0028: * Microsystems, Inc. All Rights Reserved.
0029: *
0030: * If you wish your version of this file to be governed by only the CDDL
0031: * or only the GPL Version 2, indicate your decision by adding
0032: * "[Contributor] elects to include this software in this distribution
0033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
0034: * single choice of license, a recipient has the option to distribute
0035: * your version of this file under either the CDDL, the GPL Version 2 or
0036: * to extend the choice of license to its licensees as provided above.
0037: * However, if you add GPL Version 2 code and therefore, elected the GPL
0038: * Version 2 license, then the option applies only if the new code is
0039: * made subject to such option by the copyright holder.
0040: */
0041:
0042: package org.netbeans.modules.cnd.modelimpl.trace;
0043:
0044: import java.text.NumberFormat;
0045: import org.netbeans.modules.cnd.apt.support.StartEntry;
0046: import org.netbeans.modules.cnd.editor.parser.CppFoldRecord;
0047: import org.netbeans.modules.cnd.modelimpl.debug.Diagnostic;
0048: import org.netbeans.modules.cnd.modelimpl.debug.TraceFlags;
0049: import org.netbeans.modules.cnd.modelimpl.parser.CPPParserEx;
0050: import org.netbeans.modules.cnd.apt.support.APTBuilder;
0051: import org.netbeans.modules.cnd.apt.support.APTTokenStreamBuilder;
0052: import org.netbeans.modules.cnd.apt.support.APTSystemStorage;
0053: import org.netbeans.modules.cnd.apt.support.APTDriver;
0054: import org.netbeans.modules.cnd.apt.utils.APTCommentsFilter;
0055: import org.netbeans.modules.cnd.apt.utils.APTTraceUtils;
0056: import org.netbeans.modules.cnd.modelimpl.cache.CacheManager;
0057: import org.netbeans.modules.cnd.modelimpl.csm.core.LibProjectImpl;
0058: import java.io.*;
0059: import java.util.*;
0060: import java.util.List;
0061:
0062: import antlr.*;
0063: import antlr.collections.*;
0064:
0065: import org.netbeans.modules.cnd.api.model.*;
0066: import org.netbeans.modules.cnd.api.model.util.*;
0067: import org.netbeans.modules.cnd.api.project.NativeFileItem;
0068: import org.netbeans.modules.cnd.api.project.NativeProject;
0069: import org.netbeans.modules.cnd.modelimpl.csm.core.*;
0070: import org.netbeans.modules.cnd.apt.support.APTMacroExpandedStream;
0071: import org.netbeans.modules.cnd.apt.structure.APT;
0072: import org.netbeans.modules.cnd.apt.structure.APTFile;
0073: import org.netbeans.modules.cnd.apt.support.APTIncludeHandler;
0074: import org.netbeans.modules.cnd.apt.support.APTLanguageSupport;
0075: import org.netbeans.modules.cnd.apt.support.APTMacroMap;
0076: import org.netbeans.modules.cnd.apt.support.APTPreprocHandler;
0077: import org.netbeans.modules.cnd.apt.support.APTHandlersSupport;
0078: import org.netbeans.modules.cnd.apt.utils.APTUtils;
0079: import org.netbeans.modules.cnd.editor.parser.FoldingParser;
0080: import org.netbeans.modules.cnd.modelimpl.debug.DiagnosticExceptoins;
0081: import org.netbeans.modules.cnd.modelimpl.parser.CsmAST;
0082: import org.netbeans.modules.cnd.modelimpl.repository.RepositoryUtils;
0083: import org.netbeans.modules.cnd.repository.api.RepositoryAccessor;
0084: import org.netbeans.modules.cnd.utils.cache.CharSequenceKey;
0085: import org.openide.util.Lookup;
0086:
0087: /**
0088: * Tracer for model
0089: * @author Vladimir Kvasihn
0090: */
0091: public class TraceModel extends TraceModelBase {
0092:
0093: private static class TestResult {
0094:
0095: private long time;
0096: private long lineCount;
0097:
0098: public TestResult() {
0099: this (0);
0100: }
0101:
0102: public TestResult(long time) {
0103: this .setTime(time);
0104: }
0105:
0106: public TestResult(long time, long lineCount) {
0107: this .setTime(time);
0108: this .setLineCount(lineCount);
0109: }
0110:
0111: public String getLPS() {
0112: if (time == 0 || lineCount <= 0) {
0113: return "N/A"; // NOI18N
0114: } else {
0115: return "" + (lineCount * 1000 / time);
0116: }
0117: }
0118:
0119: public long getTime() {
0120: return time;
0121: }
0122:
0123: public void setTime(long time) {
0124: this .time = time;
0125: }
0126:
0127: public long getLineCount() {
0128: return (lineCount < 0) ? 0 : lineCount;
0129: }
0130:
0131: public boolean isLineCountValid() {
0132: return lineCount >= 0;
0133: }
0134:
0135: public void setLineCount(long lineCount) {
0136: this .lineCount = lineCount;
0137: }
0138:
0139: public void accumulate(TestResult toAdd) {
0140: time += toAdd.time;
0141: if (isLineCountValid()) {
0142: if (toAdd.isLineCountValid()) {
0143: lineCount += toAdd.getLineCount();
0144: } else {
0145: // lineCount = -1;
0146: }
0147: }
0148: }
0149: }
0150:
0151: private static final int APT_REPEAT_TEST = Integer.getInteger(
0152: "apt.repeat.test", 3).intValue(); // NOI18N
0153:
0154: public static void main(String[] args) {
0155: new TraceModel().test(args);
0156: if (TraceFlags.USE_AST_CACHE) {
0157: CacheManager.getInstance().close();
0158: } else {
0159: APTDriver.getInstance().close();
0160: }
0161: //System.out.println("" + org.netbeans.modules.cnd.apt.utils.APTIncludeUtils.getHitRate());
0162: }
0163:
0164: private Cache cache;
0165:
0166: private static CsmTracer tracer = new CsmTracer(false);
0167:
0168: private boolean showAstWindow = false;
0169: private boolean dumpAst = false;
0170: private boolean dumpModel = false;
0171: private boolean dumpLib = false;
0172: private boolean dumpFileOnly = false;
0173: private boolean showTime = false;
0174: //private boolean showErrorCount = false;
0175: private boolean writeAst = false;
0176: private boolean readAst = false;
0177: private boolean useZip = false;
0178: private boolean testLibProject = false;
0179: private boolean deep = true;
0180: private boolean showMemoryUsage = false;
0181: private boolean testUniqueName = false;
0182: private boolean testAPT = false;
0183: private boolean testAPTLexer = false;
0184: private boolean testAPTDriver = false;
0185: private boolean testAPTWalkerVisit = false;
0186: private boolean testAPTWalkerGetStream = false;
0187: private boolean testAPTWalkerGetExpandedStream = false;
0188: private boolean testAPTWalkerGetFilteredStream = false;
0189: private boolean testAPTParser = false;
0190: private boolean breakAfterAPT = false;
0191:
0192: private boolean stopBeforeAll = false;
0193: private boolean stopAfterAll = false;
0194: private boolean printTokens = false;
0195: private boolean dumpModelAfterCleaningCache = false; // --clean4dump
0196:
0197: private int repeatCount = 1; // --repeat
0198:
0199: private boolean dumpStatistics = false;
0200: private static final int DEFAULT_TRACEMODEL_STATISTICS_LEVEL = 1;
0201: private String dumpFile = null;
0202: private String dumpDir = null;
0203: private static final String statPostfix = ".stat"; // NOI18N
0204:
0205: // Cache options
0206: private boolean enableCache = false;
0207: private boolean testCache = false;
0208:
0209: // Callback options
0210: private boolean dumpPPState = false;
0211:
0212: public void setDumpModel(boolean dumpModel) {
0213: this .dumpModel = dumpModel;
0214: }
0215:
0216: public void setDumpPPState(boolean dumpPPState) {
0217: this .dumpPPState = dumpPPState;
0218: }
0219:
0220: private boolean listFilesAtEnd = false;
0221: private boolean testRawPerformance = false;
0222: private boolean printUserFileList = false;
0223: private boolean quiet = false;
0224: private boolean memBySize = false;
0225: private boolean doCleanRepository = Boolean
0226: .getBoolean("cnd.clean.repository");
0227:
0228: private boolean testFolding = false;
0229:
0230: private Map<String, Long> cacheTimes = new HashMap<String, Long>();
0231:
0232: private int lap = 0;
0233:
0234: private final Map<CsmFile, APTPreprocHandler> states = new HashMap<CsmFile, APTPreprocHandler>();
0235:
0236: FileImpl.Hook hook = new FileImpl.Hook() {
0237: public void parsingFinished(CsmFile file,
0238: APTPreprocHandler preprocHandler) {
0239: states.put(file, preprocHandler);
0240: }
0241:
0242: };
0243:
0244: public TraceModel() {
0245: FileImpl.setHook(hook);
0246: }
0247:
0248: @Override
0249: protected ProcessFlagResult processFlag(char flag, String argRest) {
0250: ProcessFlagResult result = super .processFlag(flag, argRest);
0251: if (result != ProcessFlagResult.NONE_PROCESSED) {
0252: return result;
0253: }
0254: // it's easier to set the most "popular" return value here and NONE_... in default case
0255: result = ProcessFlagResult.CHAR_PROCESSED;
0256: switch (flag) {
0257: case 'n':
0258: deep = false;
0259: break;
0260: case 'e':
0261: System.setErr(System.out);
0262: break;
0263: case 'w':
0264: showAstWindow = true;
0265: break;
0266: case 'a':
0267: dumpAst = true;
0268: break;
0269: case 'm':
0270: dumpModel = true;
0271: dumpFileOnly = false;
0272: break; // -m overrides -f
0273: case 'M':
0274: showMemoryUsage = true;
0275: break;
0276: case 'u':
0277: testUniqueName = true;
0278: break;
0279: case 'f':
0280: if (!dumpModel) { // do not ovverride -m
0281: dumpModel = true;
0282: dumpFileOnly = true;
0283: }
0284: break;
0285: case 't':
0286: showTime = true;
0287: break;
0288: // case 'L': testLexer = true; break;
0289: //case 'c': showErrorCount = true; break;
0290: case 'W':
0291: writeAst = true;
0292: break;
0293: case 'R':
0294: readAst = true;
0295: break;
0296: case 'Z':
0297: useZip = true;
0298: break;
0299: case 'C':
0300: enableCache = true;
0301: break;
0302: case 'l':
0303: testLibProject = true;
0304: break;
0305: case 'c':
0306: testCache = true;
0307: break;
0308: case 'p':
0309: dumpPPState = true;
0310: break;
0311: // "-SDir" defines dump directory for per file statistics
0312: case 'S':
0313: dumpStatistics = true;
0314: if (argRest.length() > 0) {
0315: // dump directory for per file statistics
0316: File perFileDumpDir = new File(argRest);
0317: perFileDumpDir.mkdirs();
0318: if (!perFileDumpDir.isDirectory()) {
0319: print("Parameter -S" + argRest
0320: + " does not specify valid directory"); // NOI18N
0321: } else {
0322: this .dumpDir = perFileDumpDir.getAbsolutePath();
0323: }
0324: result = ProcessFlagResult.ALL_PROCESSED;
0325: }
0326: break;
0327: // "-sFileName" defines global statistics dump file
0328: case 's':
0329: dumpStatistics = true;
0330: if (argRest.length() > 0) {
0331: File globalDumpFile = new File(argRest);
0332: if (globalDumpFile.exists()) {
0333: globalDumpFile.delete();
0334: }
0335: try {
0336: if (globalDumpFile.getParentFile() != null) {
0337: globalDumpFile.getParentFile().mkdirs();
0338: globalDumpFile.createNewFile();
0339: this .dumpFile = globalDumpFile
0340: .getAbsolutePath();
0341: result = ProcessFlagResult.ALL_PROCESSED;
0342: }
0343: } catch (IOException ex) {
0344: ex.printStackTrace();
0345: }
0346: }
0347: break;
0348: case 'A':
0349: testAPT = true;
0350: testAPTWalkerVisit = true;
0351: testAPTWalkerGetStream = true;
0352: testAPTWalkerGetExpandedStream = true;
0353: testAPTWalkerGetFilteredStream = true;
0354: testAPTLexer = true;
0355: breakAfterAPT = true;
0356: testAPTDriver = true;
0357: break;
0358: // case 'b': testAPTPlainLexer = true; testAPT = true; breakAfterAPT = true; break;
0359: case 'B':
0360: testAPTLexer = true;
0361: testAPT = true;
0362: breakAfterAPT = true;
0363: break;
0364: case 'o':
0365: printTokens = true;
0366: break;
0367: case 'v':
0368: testAPTWalkerVisit = true;
0369: testAPT = true;
0370: breakAfterAPT = true;
0371: break;
0372: case 'g':
0373: testAPTWalkerGetStream = true;
0374: testAPT = true;
0375: breakAfterAPT = true;
0376: break;
0377: case 'G':
0378: testAPTWalkerGetExpandedStream = true;
0379: testAPT = true;
0380: breakAfterAPT = true;
0381: break;
0382: case 'F':
0383: testAPTWalkerGetFilteredStream = true;
0384: testAPT = true;
0385: breakAfterAPT = true;
0386: break;
0387: case 'd':
0388: testAPTDriver = true;
0389: testAPT = true;
0390: breakAfterAPT = true;
0391: break;
0392: case 'h':
0393: testAPT = true;
0394: breakAfterAPT = true;
0395: break;
0396: case 'H':
0397: testAPTParser = true;
0398: testAPT = true;
0399: breakAfterAPT = true;
0400: break;
0401: case 'O':
0402: stopBeforeAll = true;
0403: stopAfterAll = true;
0404: break;
0405: case 'q':
0406: quiet = true;
0407: break;
0408: default:
0409: result = ProcessFlagResult.NONE_PROCESSED;
0410: }
0411: return result;
0412: }
0413:
0414: @Override
0415: protected boolean processFlag(String flag) {
0416: if (super .processFlag(flag)) {
0417: return true;
0418: } else if ("dumplib".equals(flag)) { // NOI18N
0419: // NOI18N
0420: dumpLib = true;
0421: } else if ("listfiles".equals(flag)) { // NOI18N
0422: // NOI18N
0423: listFilesAtEnd = true;
0424: } else if ("raw".equals(flag)) { // NOI18N
0425: // NOI18N
0426: testRawPerformance = true;
0427: //TraceFlags.DO_NOT_RENDER = true;
0428: } else if ("listfiles".equals(flag)) { // NOI18N
0429: // NOI18N
0430: printUserFileList = true;
0431: } else if ("mbs".equals(flag)) { // NOI18N
0432: // NOI18N
0433: memBySize = true;
0434: } else if ("cleanrepository".equals(flag)) { // NOI18N
0435: // NOI18N
0436: doCleanRepository = true;
0437: } else if ("folding".equals(flag)) { // NOI18N
0438: // NOI18N
0439: testFolding = true;
0440: } else if ("clean4dump".equals(flag)) { // NOI18N
0441: // NOI18N
0442: dumpModelAfterCleaningCache = true;
0443: } else if ("repeat".equals(flag) || flag.startsWith("repeat:")) { // NOI18N
0444: int len = "repeat".length(); // NOI18N
0445: if (flag.length() == len) {
0446: repeatCount = 2;
0447: } else {
0448: repeatCount = Integer.parseInt(flag.substring(len + 1));
0449: }
0450: } else {
0451: return false;
0452: }
0453: return true;
0454: }
0455:
0456: private void test(String[] args) {
0457: try {
0458: processArguments(args);
0459: doTest();
0460: } catch (Error thr) {
0461: System.err.printf("\n");
0462: DiagnosticExceptoins.register(thr);
0463: return;
0464: } finally {
0465: getModel().shutdown();
0466: }
0467: }
0468:
0469: /*package*/
0470: void doTest() {
0471: if (repeatCount > 1) {
0472: for (int i = 0; i < repeatCount; i++) {
0473: print("\n\n==================== Pass " + i
0474: + "====================\n"); // NOI18N
0475: doTest2();
0476: resetProject();
0477: //sleep(2000, "Waiting for repository to shutdown");
0478: }
0479: } else {
0480: doTest2();
0481: }
0482: }
0483:
0484: /*package*/
0485: void doTest2() {
0486: if (stopBeforeAll) {
0487: waitAnyKey();
0488: }
0489: if (writeAst || readAst) {
0490: try {
0491: cache = new Cache(useZip);
0492: } catch (Exception e) {
0493: DiagnosticExceptoins.register(e);
0494: return;
0495: }
0496: }
0497:
0498: if (writeAst && readAst) {
0499: print("Impossible options combination: both writing and reading AST\n"); // NOI18N
0500: return;
0501: }
0502: if (useZip && !(writeAst || readAst)) {
0503: print("Impossible options combination: using ZIP format, but neither writing nor reading AST. Ignoring ZIP format.\n"); // NOI18N
0504: useZip = false;
0505: }
0506:
0507: if (testCache) {
0508: //print("Test cache mode ON." + '\n');
0509: enableCache = true;
0510: } else if (dumpStatistics) {
0511: if (dumpFile == null && dumpDir == null) {
0512: print("Turning OFF statistics as neither global file nor directory is specified"); // NOI18N
0513: dumpStatistics = false;
0514: } else {
0515: print("Dumping Statistics is ON"); // NOI18N
0516: if (Diagnostic.getStatisticsLevel() == 0) {
0517: // need to set the default level
0518: Diagnostic
0519: .setStatisticsLevel(DEFAULT_TRACEMODEL_STATISTICS_LEVEL);
0520: }
0521: if (dumpFile != null) {
0522: print("Global Dump file is " + dumpFile); // NOI18N
0523: }
0524: if (dumpDir != null) {
0525: print("Dump directory for per file statistics is "
0526: + dumpDir); // NOI18N
0527: }
0528: }
0529: }
0530:
0531: if (testLibProject) {
0532: testLibProject();
0533: }
0534:
0535: if (printUserFileList) {
0536: print("Processing files:\n"); // NOI18N
0537: for (NativeFileItem file : getFileItems()) {
0538: print(file.getFile().getAbsolutePath() + ' ', false);
0539: }
0540: print("");
0541: }
0542:
0543: long memUsed = 0;
0544: if (showMemoryUsage) {
0545: memUsed = usedMemory();
0546: }
0547:
0548: long t = System.currentTimeMillis();
0549: TestResult total = test();
0550: total.time = System.currentTimeMillis() - t;
0551:
0552: if (testRawPerformance) {
0553: print("Take one finished."); // NOI18N
0554: print("Total parsing time " + total.time + " ms"); // NOI18N
0555: calculateAverageLPS(total, true);
0556: print("Lines count " + total.lineCount); // NOI18N
0557: print("Average LPS " + total.getLPS()); // NOI18N
0558: if (showMemoryUsage) {
0559: showMemoryUsage(memUsed);
0560: }
0561:
0562: // for (int i = 0; i < 100; i++) {
0563: // initProject();
0564: // test();
0565: // showMemoryUsage(memUsed);
0566: // }
0567: print("\nTesting raw performance: parsing project, take two\n"); // NOI18N
0568: resetProject();
0569: if (stopBeforeAll) {
0570: waitAnyKey();
0571: }
0572: t = System.currentTimeMillis();
0573: total = test();
0574: total.time = System.currentTimeMillis() - t;
0575: }
0576:
0577: /* this unnecessary since we call waitProjectParsed() for each file
0578: if( showTime ) {
0579: print("Waiting for the rest of the parser queue to be parsed");
0580: }
0581: waitProjectParsed();
0582: */
0583:
0584: if (dumpLib) {
0585: for (Iterator it = getProject().getLibraries().iterator(); it
0586: .hasNext();) {
0587: CsmProject lib = (CsmProject) it.next();
0588: tracer.dumpModel(lib);
0589: }
0590: }
0591:
0592: if (isShowTime()) {
0593:
0594: int maxLen = 0;
0595: for (int i = 0; i < CPPParserEx.MAX_GUESS_IDX; i++) {
0596: if (CPPParserEx.guessingNames[i] != null) {
0597: int len = CPPParserEx.guessingNames[i].length();
0598: if (len > maxLen) {
0599: maxLen = len;
0600: }
0601: }
0602: }
0603:
0604: boolean printGuessStat = false;
0605: // check if we had the statistics
0606: for (int i = 0; i < CPPParserEx.MAX_GUESS_IDX; i++) {
0607: if (CPPParserEx.guessingCount[i] != 0) {
0608: printGuessStat = true;
0609: break;
0610: }
0611: }
0612: if (listFilesAtEnd) {
0613: print("\n========== User project files =========="); // NOI18N
0614: List<CharSequence> l = new ArrayList<CharSequence>(
0615: getProject().getAllFiles().size());
0616: for (Iterator it = getProject().getAllFiles()
0617: .iterator(); it.hasNext();) {
0618: CsmFile file = (CsmFile) it.next();
0619: l.add(file.getAbsolutePath());
0620: }
0621: Collections.sort(l, CharSequenceKey.Comparator);
0622: for (Iterator it = l.iterator(); it.hasNext();) {
0623: print((String) it.next());
0624: }
0625: print("\n========== Library files =========="); // NOI18N
0626: l = new ArrayList<CharSequence>();
0627: for (Iterator it1 = getProject().getLibraries()
0628: .iterator(); it1.hasNext();) {
0629: ProjectBase lib = (ProjectBase) it1.next();
0630: for (Iterator it2 = lib.getAllFiles().iterator(); it2
0631: .hasNext();) {
0632: CsmFile file = (CsmFile) it2.next();
0633: l.add(file.getAbsolutePath());
0634: }
0635: }
0636: Collections.sort(l, CharSequenceKey.Comparator);
0637: for (Iterator it = l.iterator(); it.hasNext();) {
0638: print((String) it.next());
0639: }
0640: }
0641: if (printGuessStat) {
0642: print("\nGuessing statistics:"); // NOI18N
0643: print("Id" // NOI18N
0644: + "\t" + padR("Rule:Line", maxLen) // NOI18N
0645: + "\tTime" // NOI18N
0646: + "\tCount" // NOI18N
0647: + "\tFail" // NOI18N
0648: //+ "\tTime in failures"
0649: + "\tSuccess, %"); // NOI18N
0650: long guessingTime = 0;
0651: for (int i = 0; i < CPPParserEx.MAX_GUESS_IDX; i++) {
0652: guessingTime += CPPParserEx.guessingTimes[i];
0653: //double sps = (CPPParserEx.guessingTimes[i] !=0) ? ((double)CPPParserEx.guessingCount[i])/CPPParserEx.guessingTimes[i] : 0;
0654: double usa = 0;
0655: if (CPPParserEx.guessingCount[i] != 0) {
0656: usa = (1 - ((double) CPPParserEx.guessingFailures[i])
0657: / CPPParserEx.guessingCount[i]) * 100;
0658: }
0659: print(""
0660: + i
0661: + "\t"
0662: + padR(CPPParserEx.guessingNames[i], maxLen) // NOI18N
0663: + "\t" + CPPParserEx.guessingTimes[i] // NOI18N
0664: + "\t" + CPPParserEx.guessingCount[i] // NOI18N
0665: + "\t" + CPPParserEx.guessingFailures[i] // NOI18N
0666: //+ "\t" + (int)sps
0667: + "\t" + (int) usa); // NOI18N
0668: }
0669:
0670: print("\nTotal guessing time: "
0671: + guessingTime
0672: + "ms "
0673: + "("
0674: + ((total.getTime() != 0) ? guessingTime * 100
0675: / total.getTime() : -1)
0676: + "% of total parse time)"); // NOI18N
0677: }
0678: }
0679: if (isShowTime() || testRawPerformance) {
0680: print("Total parsing time: " + total.getTime() + "ms"); // NOI18N
0681: //print("Average LPS: " + total.getLPS());
0682: calculateAverageLPS(total, !testRawPerformance);
0683: print("Lines count " + total.lineCount); // NOI18N
0684: String text = testRawPerformance ? "Raw performance (average LPS): "
0685: : "Average LPS: "; // NOI18N
0686: print(text + total.getLPS());
0687: int userFiles = countUserFiles();
0688: int systemHeaders = countSystemHeaders();
0689: print("" + userFiles + " user files"); // NOI18N
0690: print("" + systemHeaders + " system headers"); // NOI18N
0691: }
0692: if (showMemoryUsage) {
0693: showMemoryUsage(memUsed);
0694: }
0695: if (isShowTime() || showMemoryUsage || dumpModel
0696: || dumpFileOnly || dumpPPState) {
0697: print("\n"); // NOI18N
0698: }
0699: if (dumpStatistics) {
0700: if (this .dumpFile != null) {
0701: try {
0702: Diagnostic.dumpUnresolvedStatistics(this .dumpFile,
0703: true);
0704: } catch (FileNotFoundException e) {
0705: DiagnosticExceptoins.register(e);
0706: }
0707: }
0708: }
0709:
0710: if (TraceFlags.CLEAN_MACROS_AFTER_PARSE) {
0711: List restoredFiles = ProjectBase.testGetRestoredFiles();
0712: if (restoredFiles != null) {
0713: System.err.println("the number of restored files "
0714: + restoredFiles.size());
0715: for (int i = 0; i < restoredFiles.size(); i++) {
0716: System.err.println("#" + i + ":"
0717: + restoredFiles.get(i));
0718: }
0719: }
0720: }
0721:
0722: if (dumpModelAfterCleaningCache) {
0723: anyKey("Press any key to clean repository:"); // NOI18N
0724: RepositoryAccessor.getRepository().debugClear();
0725: System.gc();
0726: System.gc();
0727: System.gc();
0728: System.gc();
0729: System.gc();
0730: anyKey("Press any key to dump model:"); // NOI18N
0731: if (!dumpFileOnly) {
0732: tracer.dumpModel(getProject());
0733: }
0734: }
0735: if (stopAfterAll) {
0736: System.gc();
0737: anyKey("Press any key to finish:"); // NOI18N
0738: }
0739: }
0740:
0741: private void anyKey(String message) {
0742: System.err.println(message);
0743: try {
0744: System.in.read();
0745: } catch (IOException ex) {
0746: DiagnosticExceptoins.register(ex);
0747: }
0748: }
0749:
0750: private void showMemoryUsage(long memUsed) {
0751: long newMemUsed = usedMemory();
0752: long memDelta = newMemUsed - memUsed;
0753: NumberFormat nf = NumberFormat.getIntegerInstance();
0754: nf.setGroupingUsed(true);
0755: nf.setMinimumIntegerDigits(6);
0756: print("Amount of memory used" + getLap() + ": "
0757: + nf.format((memDelta) / 1024) + " Kb"); // NOI18N
0758: if (memBySize) {
0759: TestResult rInc = new TestResult();
0760: TestResult rExc = new TestResult();
0761: calculateAverageLPS(rInc, true);
0762: calculateAverageLPS(rExc, false);
0763: print("User code lines: " + rExc.lineCount); // NOI18N
0764: print("Total lines (including all headers): "
0765: + rInc.lineCount); // NOI18N
0766: print("Memory usage per (user) line " + getLap() + '\t'
0767: + nf.format(memDelta / rExc.lineCount)
0768: + " bytes per line"); // NOI18N
0769: print("Memory usage per (total) line" + getLap() + '\t'
0770: + nf.format(memDelta / rInc.lineCount)
0771: + " bytes per line"); // NOI18N
0772: }
0773: }
0774:
0775: private void waitAnyKey() {
0776: System.out.println("Press any key to continue:"); // NOI18N
0777: try {
0778: System.in.read();
0779: } catch (IOException ex) {
0780: DiagnosticExceptoins.register(ex);
0781: }
0782: }
0783:
0784: private TestResult test() {
0785: lap++;
0786: TestResult total = new TestResult();
0787: //for (int i = 0; i < fileList.size(); i++) {
0788: for (NativeFileItem item : getFileItems()) {
0789: try {
0790: if (!testFolding) {
0791: TestResult res = test(item);
0792: total.accumulate(res);
0793: } else {
0794: testFolding(item.getFile());
0795: }
0796: } catch (Exception e) {
0797: DiagnosticExceptoins.register(e);
0798: }
0799: }
0800:
0801: return total;
0802: }
0803:
0804: private String getLap() {
0805: return " (lap " + lap + ") "; // NOI18N
0806: }
0807:
0808: private String padR(String s, int len) {
0809: if (s == null) {
0810: s = "";
0811: }
0812: if (s.length() >= len) {
0813: return s;
0814: } else {
0815: StringBuilder sb = new StringBuilder(s);
0816: sb.setLength(len);
0817: for (int i = s.length(); i < len; i++) {
0818: sb.setCharAt(i, ' ');
0819: }
0820: return sb.toString();
0821: }
0822: }
0823:
0824: private void waitProjectParsed(boolean trace) {
0825: boolean wasWait = true;
0826: while (wasWait) {
0827: wasWait = waitParsed(getProject(), trace);
0828: if (getProject().getLibraries().size() > 0) {
0829: if (trace) {
0830: System.err.println("checking libraries");
0831: }
0832: for (Iterator it = getProject().getLibraries()
0833: .iterator(); it.hasNext();) {
0834: CsmProject lib = (CsmProject) it.next();
0835: if (trace) {
0836: System.err.println("checking library "
0837: + lib.getName());
0838: }
0839: wasWait |= waitParsed((ProjectBase) lib, trace);
0840: }
0841: }
0842: }
0843: }
0844:
0845: private boolean waitParsed(ProjectBase project, boolean trace) {
0846: boolean wasWait = false;
0847: project.waitParse();
0848: return wasWait;
0849: }
0850:
0851: private void sleep(int timeout, String message) {
0852: System.err.printf("Sleeping: %s\n", message);
0853: sleep(timeout);
0854: System.err.printf("Awoke (%s)\n", message);
0855: }
0856:
0857: private void sleep(int timeout) {
0858: try {
0859: Thread.sleep(timeout);
0860: } catch (InterruptedException ex) {
0861: //ex.printStackTrace();
0862: }
0863: }
0864:
0865: private APTSystemStorage sysAPTData = APTSystemStorage.getDefault();
0866:
0867: private APTIncludeHandler getIncludeHandler(File file) {
0868: List<String> sysIncludes = sysAPTData.getIncludes(
0869: "TraceModelSysIncludes", getSystemIncludes()); // NOI18N
0870: List<String> qInc = getQuoteIncludePaths();
0871: if (isPathsRelCurFile()) {
0872: qInc = new ArrayList<String>(getQuoteIncludePaths().size());
0873: for (Iterator it = getQuoteIncludePaths().iterator(); it
0874: .hasNext();) {
0875: String path = (String) it.next();
0876: if (!(new File(path).isAbsolute())) {
0877: File dirFile = file.getParentFile();
0878: File pathFile = new File(dirFile, path);
0879: path = pathFile.getAbsolutePath();
0880: }
0881: qInc.add(path);
0882: }
0883: }
0884: StartEntry startEntry = new StartEntry(file.getAbsolutePath(),
0885: RepositoryUtils.UIDtoKey(getProject().getUID()));
0886: return APTHandlersSupport.createIncludeHandler(startEntry,
0887: sysIncludes, qInc);
0888: }
0889:
0890: private APTMacroMap getMacroMap(File file) {
0891: //print("SystemIncludePaths: " + systemIncludePaths.toString() + "\n");
0892: //print("QuoteIncludePaths: " + quoteIncludePaths.toString() + "\n");
0893: APTMacroMap map = APTHandlersSupport.createMacroMap(
0894: getSysMap(file), getMacros());
0895: return map;
0896: }
0897:
0898: private APTPreprocHandler getPreprocHandler(File file) {
0899: APTPreprocHandler preprocHandler = APTHandlersSupport
0900: .createPreprocHandler(getMacroMap(file),
0901: getIncludeHandler(file), !file.getPath()
0902: .endsWith(".h")); // NOI18N
0903: return preprocHandler;
0904: }
0905:
0906: private APTMacroMap getSysMap(File file) {
0907: APTMacroMap map = sysAPTData.getMacroMap("TraceModelSysMacros",
0908: getSysMacros()); // NOI18N
0909: return map;
0910: }
0911:
0912: //
0913: // @Override
0914: // protected List<String> getSystemIncludes() {
0915: // List<String> result = super.getSystemIncludes();
0916: // if( result.isEmpty() && ! dumpPPState ) {
0917: // // NB: want any fake value but not for suite.sh which is run with dumpPPState
0918: // result.add("/usr/non-exists"); // NOI18N
0919: // }
0920: // return result;
0921: // }
0922: //
0923: // @Override
0924: // protected List<String> getSysMacros() {
0925: // List<String> result = super.getSysMacros();
0926: // if( result.isEmpty() && ! dumpPPState ) {
0927: // // NB: want any fake value but not for suite.sh which is run with dumpPPState
0928: // result.add("NO_DEFAULT_DEFINED_SYSTEM_MACROS"); // NOI18N
0929: // }
0930: // return result;
0931: // }
0932:
0933: private long testAPTLexer(File file, boolean printTokens)
0934: throws FileNotFoundException, RecognitionException,
0935: TokenStreamException, IOException, ClassNotFoundException {
0936: print("Testing APT lexer:"); // NOI18N
0937: long time = System.currentTimeMillis();
0938: InputStream stream = null;
0939: try {
0940: stream = new BufferedInputStream(new FileInputStream(file),
0941: TraceFlags.BUF_SIZE);
0942: TokenStream ts = APTTokenStreamBuilder.buildTokenStream(
0943: file.getAbsolutePath(), stream);
0944: for (Token t = ts.nextToken(); !APTUtils.isEOF(t); t = ts
0945: .nextToken()) {
0946: if (printTokens) {
0947: print("" + t);
0948: }
0949: }
0950: time = System.currentTimeMillis() - time;
0951: if (isShowTime()) {
0952: print("APT Lexing " + file.getName() + " took " + time
0953: + " ms"); // NOI18N
0954: }
0955: return time;
0956: } finally {
0957: if (stream != null) {
0958: stream.close();
0959: }
0960: }
0961: }
0962:
0963: private long testAPTWalkerVisit(APTFile apt, FileBuffer buffer)
0964: throws TokenStreamException, IOException {
0965: File file = buffer.getFile();
0966: boolean cleanAPT = apt == null;
0967: long time = System.currentTimeMillis();
0968: if (cleanAPT) {
0969: invalidateAPT(buffer);
0970: time = System.currentTimeMillis();
0971: apt = APTDriver.getInstance().findAPTLight(buffer);
0972: }
0973: APTPreprocHandler ppHandler = APTHandlersSupport
0974: .createPreprocHandler(getMacroMap(file),
0975: getIncludeHandler(file), true);
0976: APTWalkerTest walker = new APTWalkerTest(apt, ppHandler);
0977: walker.visit();
0978: time = System.currentTimeMillis() - time;
0979:
0980: if (isShowTime()) {
0981: print("Visiting APT "
0982: + (cleanAPT ? "with cleaning APT in driver" : "")
0983: + " took " + time + " ms"); // NOI18N
0984: print(" resolving include paths took "
0985: + walker.getIncludeResolvingTime() + " ms"); // NOI18N
0986: }
0987:
0988: // time = System.currentTimeMillis();
0989: // if (cleanAPT) {
0990: // invalidateAPT(file);
0991: // time = System.currentTimeMillis();
0992: // apt = APTDriver.getInstance().findAPT(file);
0993: // }
0994: // walker = new APTWalkerTest(apt, getMacroMap(file), getIncludeHandler(file));
0995: // walker.nonRecurseVisit();
0996: // time = System.currentTimeMillis() - time;
0997: //
0998: // if( showTime ) {
0999: // print("Non recursive visiting APT "+ (cleanAPT ? "with cleaning APT in driver":"") + " took " + time + " ms");
1000: // }
1001: return time;
1002: }
1003:
1004: private long testAPTWalkerGetStream(APTFile apt, FileBuffer buffer,
1005: boolean expand, boolean filter, boolean printTokens)
1006: throws TokenStreamException, IOException {
1007: File file = buffer.getFile();
1008: boolean cleanAPT = apt == null;
1009: long time = System.currentTimeMillis();
1010: if (cleanAPT) {
1011: invalidateAPT(buffer);
1012: time = System.currentTimeMillis();
1013: apt = APTDriver.getInstance().findAPT(buffer);
1014: }
1015: APTMacroMap macroMap = getMacroMap(file);
1016: APTPreprocHandler ppHandler = APTHandlersSupport
1017: .createPreprocHandler(macroMap,
1018: getIncludeHandler(file), true);
1019: APTWalkerTest walker = new APTWalkerTest(apt, ppHandler);
1020: TokenStream ts = walker.getTokenStream();
1021: if (expand) {
1022: ts = new APTMacroExpandedStream(ts, macroMap);
1023: }
1024: if (filter) {
1025: ts = APTLanguageSupport.getInstance().getFilter(
1026: APTLanguageSupport.GNU_CPP).getFilteredStream(
1027: new APTCommentsFilter(ts));
1028: }
1029: int lastLine = -1;
1030: for (Token t = ts.nextToken(); !APTUtils.isEOF(t); t = ts
1031: .nextToken()) {
1032: if (printTokens) {
1033: print(" " + t.getText(), t.getLine() != lastLine); // NOI18N
1034: }
1035: lastLine = t.getLine();
1036: }
1037: if (printTokens && lastLine > 0) {
1038: print("", true);
1039: }
1040: time = System.currentTimeMillis() - time;
1041:
1042: if (isShowTime()) {
1043: print("Getting" + (expand ? " expanded" : "")
1044: + (filter ? " filtered" : "")
1045: + " APT token stream "
1046: + (cleanAPT ? "with cleaning APT in driver" : "")
1047: + " took " + time + " ms"); // NOI18N
1048: print(" resolving include paths took "
1049: + walker.getIncludeResolvingTime() + " ms"); // NOI18N
1050: }
1051: return time;
1052: }
1053:
1054: private long testAPTParser(NativeFileItem item, boolean cleanAPT)
1055: throws IOException, RecognitionException,
1056: TokenStreamException {
1057: FileBuffer buffer = new FileBufferFile(item.getFile());
1058: print("Testing APT Parser"); // NOI18N
1059: int flags = CPPParserEx.CPP_CPLUSPLUS;
1060: File file = buffer.getFile();
1061: long time = System.currentTimeMillis();
1062: if (cleanAPT) {
1063: invalidateAPT(buffer);
1064: time = System.currentTimeMillis();
1065: }
1066: FileImpl fileImpl = null;
1067: fileImpl = (FileImpl) getProject().testAPTParseFile(item);
1068: getProject().waitParse();
1069: time = System.currentTimeMillis() - time;
1070:
1071: if (isShowTime()) {
1072: print("Parsing"
1073: + (cleanAPT ? " with cleaning APT in driver" : "")
1074: + " took " + time + " ms"); // NOI18N
1075: }
1076: return time;
1077: }
1078:
1079: private void testAPT(NativeFileItem item)
1080: throws FileNotFoundException, RecognitionException,
1081: TokenStreamException, IOException, ClassNotFoundException {
1082: File file = item.getFile();
1083: FileBuffer buffer = new FileBufferFile(file);
1084: print("Testing APT:" + file); // NOI18N
1085: long minLexer = Long.MAX_VALUE;
1086: long maxLexer = Long.MIN_VALUE;
1087: long minAPTLexer = Long.MAX_VALUE;
1088: long maxAPTLexer = Long.MIN_VALUE;
1089: if (testAPTLexer) {
1090: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1091: long val = testAPTLexer(file, i == -1 ? printTokens
1092: : false);
1093: minAPTLexer = Math.min(minAPTLexer, val);
1094: maxAPTLexer = Math.max(maxAPTLexer, val);
1095: }
1096: }
1097: APTFile apt = null;
1098: minDriver = Long.MAX_VALUE;
1099: maxDriver = Long.MIN_VALUE;
1100: if (testAPTDriver) {
1101: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1102: invalidateAPT(buffer);
1103: apt = testAPTDriver(buffer, i == -1 ? true : false);
1104: }
1105: }
1106: boolean cleanAPT = minDriver == Long.MAX_VALUE;
1107:
1108: long minVisit = Long.MAX_VALUE;
1109: long maxVisit = Long.MIN_VALUE;
1110: if (testAPTWalkerVisit) {
1111: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1112: long val = testAPTWalkerVisit(apt, buffer);
1113: minVisit = Math.min(minVisit, val);
1114: maxVisit = Math.max(maxVisit, val);
1115: }
1116: }
1117: long minGetTS = Long.MAX_VALUE;
1118: long maxGetTS = Long.MIN_VALUE;
1119: if (testAPTWalkerGetStream) {
1120: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1121: long val = testAPTWalkerGetStream(apt, buffer, false,
1122: false, i == -1 ? printTokens : false);
1123: minGetTS = Math.min(minGetTS, val);
1124: maxGetTS = Math.max(maxGetTS, val);
1125: }
1126: }
1127: long minGetExpandedTS = Long.MAX_VALUE;
1128: long maxGetExpandedTS = Long.MIN_VALUE;
1129: if (testAPTWalkerGetExpandedStream) {
1130: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1131: long val = testAPTWalkerGetStream(apt, buffer, true,
1132: false, i == -1 ? printTokens : false);
1133: minGetExpandedTS = Math.min(minGetExpandedTS, val);
1134: maxGetExpandedTS = Math.max(maxGetExpandedTS, val);
1135: }
1136: }
1137: long minGetFilteredTS = Long.MAX_VALUE;
1138: long maxGetFilteredTS = Long.MIN_VALUE;
1139: if (testAPTWalkerGetFilteredStream) {
1140: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1141: long val = testAPTWalkerGetStream(apt, buffer, true,
1142: true, i == -1 ? printTokens : false);
1143: minGetFilteredTS = Math.min(minGetFilteredTS, val);
1144: maxGetFilteredTS = Math.max(maxGetFilteredTS, val);
1145: }
1146: }
1147: long minParsing = Long.MAX_VALUE;
1148: long maxParsing = Long.MIN_VALUE;
1149: long minAPTParsing = Long.MAX_VALUE;
1150: long maxAPTParsing = Long.MIN_VALUE;
1151: if (testAPTParser) {
1152: for (int i = -1; i < APT_REPEAT_TEST; i++) {
1153: long val = testAPTParser(item, cleanAPT);
1154: minAPTParsing = Math.min(minAPTParsing, val);
1155: maxAPTParsing = Math.max(maxAPTParsing, val);
1156: }
1157: }
1158: if (isShowTime()) {
1159: print("APT BEST/WORST results for "
1160: + file.getAbsolutePath()); // NOI18N
1161: if (minLexer != Long.MAX_VALUE) {
1162: print(minLexer + " ms BEST Plain lexer"); // NOI18N
1163: print(maxLexer + " ms WORST Plain lexer"); // NOI18N
1164: }
1165: if (minAPTLexer != Long.MAX_VALUE) {
1166: print(minAPTLexer + " ms BEST APT lexer"); // NOI18N
1167: print(maxAPTLexer + " ms WORST APT lexer"); // NOI18N
1168: }
1169: if (minDriver != Long.MAX_VALUE) {
1170: print(minDriver + " ms BEST Building APT:"); // NOI18N
1171: print(maxDriver + " ms WORST Building APT:"); // NOI18N
1172: }
1173:
1174: if (minVisit != Long.MAX_VALUE) {
1175: print(minVisit
1176: + " ms BEST Visiting APT"
1177: + (cleanAPT ? " with cleaning APT in driver:"
1178: : ":")); // NOI18N
1179: print(maxVisit
1180: + " ms WORST Visiting APT"
1181: + (cleanAPT ? " with cleaning APT in driver:"
1182: : ":")); // NOI18N
1183: }
1184: if (minGetTS != Long.MAX_VALUE) {
1185: print(minGetTS
1186: + " ms BEST Getting APT token stream"
1187: + (cleanAPT ? " with cleaning APT in driver:"
1188: : ":")); // NOI18N
1189: print(maxGetTS
1190: + " ms WORST Getting APT token stream"
1191: + (cleanAPT ? " with cleaning APT in driver:"
1192: : ":")); // NOI18N
1193: }
1194: if (minGetExpandedTS != Long.MAX_VALUE) {
1195: print(minGetExpandedTS
1196: + " ms BEST Getting Expanded APT token stream"
1197: + (cleanAPT ? " with cleaning APT in driver:"
1198: : ":")); // NOI18N
1199: print(maxGetExpandedTS
1200: + " ms WORST Getting Expanded APT token stream"
1201: + (cleanAPT ? " with cleaning APT in driver:"
1202: : ":")); // NOI18N
1203: }
1204: if (minGetFilteredTS != Long.MAX_VALUE) {
1205: print(minGetFilteredTS
1206: + " ms BEST Getting Expanded Filtered APT token stream"
1207: + (cleanAPT ? " with cleaning APT in driver:"
1208: : ":")); // NOI18N
1209: print(maxGetFilteredTS
1210: + " ms WORST Getting Expanded Filtered APT token stream"
1211: + (cleanAPT ? " with cleaning APT in driver:"
1212: : ":")); // NOI18N
1213: }
1214: if (minParsing != Long.MAX_VALUE) {
1215: print(minParsing + " ms BEST Plaing Parsing"); // NOI18N
1216: print(maxParsing + " ms WORST Plaing Parsing"); // NOI18N
1217: }
1218: if (minAPTParsing != Long.MAX_VALUE) {
1219: print(minAPTParsing
1220: + " ms BEST APT parsing"
1221: + (cleanAPT ? " with cleaning APT in driver:"
1222: : ":")); // NOI18N
1223: print(maxAPTParsing
1224: + " ms WORST APT parsing"
1225: + (cleanAPT ? " with cleaning APT in driver:"
1226: : ":")); // NOI18N
1227: }
1228: }
1229: }
1230:
1231: private static String firstFile = null;
1232:
1233: private void invalidateAPT(final FileBuffer buffer) {
1234: File file = buffer.getFile();
1235: if (firstFile == null
1236: || firstFile.equalsIgnoreCase(file.getAbsolutePath())) {
1237: firstFile = file.getAbsolutePath();
1238: APTDriver.getInstance().invalidateAll();
1239: getProject().invalidateFiles();
1240: } else {
1241: APTDriver.getInstance().invalidateAPT(buffer);
1242: }
1243: }
1244:
1245: long minDriver = Long.MAX_VALUE;
1246: long maxDriver = Long.MIN_VALUE;
1247:
1248: private APTFile testAPTDriver(final FileBuffer buffer,
1249: boolean buildXML) throws IOException, FileNotFoundException {
1250: File file = buffer.getFile();
1251: long oldMem = usedMemory();
1252: long time = System.currentTimeMillis();
1253: APTFile apt = APTDriver.getInstance().findAPT(buffer);
1254: time = System.currentTimeMillis() - time;
1255: long newMem = usedMemory();
1256: if (isShowTime()) {
1257: minDriver = Math.min(minDriver, time);
1258: maxDriver = Math.max(maxDriver, time);
1259: print("Building APT for " + file.getName()
1260: + "\n SIZE OF FILE:" + file.length() / 1024
1261: + "Kb\n TIME: took " + time
1262: + " ms\n MEMORY: changed from " + (oldMem) / (1024)
1263: + " to " + newMem / (1024) + "["
1264: + (newMem - oldMem) / 1024 + "]Kb"); // NOI18N
1265: }
1266:
1267: // System.out.println("apt tree: \n" + APTTraceUtils.toStringList(apt));
1268: if (buildXML) {
1269: File outDir = new File("/tmp/aptout/"); // NOI18N
1270: outDir.mkdirs();
1271: File outFile = new File(outDir, file.getName() + ".xml"); // NOI18N
1272: if (outFile.exists()) {
1273: outFile.delete();
1274: }
1275: outFile.createNewFile();
1276: Writer out = new BufferedWriter(new FileWriter(outFile));
1277: APTTraceUtils.xmlSerialize(apt, out);
1278: out.flush();
1279: APT light = APTBuilder.buildAPTLight(apt);
1280: File outFileLW = new File(outDir, file.getName()
1281: + "_lw.xml"); // NOI18N
1282: if (outFileLW.exists()) {
1283: outFileLW.delete();
1284: }
1285: outFileLW.createNewFile();
1286: Writer outLW = new BufferedWriter(new FileWriter(outFileLW));
1287: APTTraceUtils.xmlSerialize(light, outLW);
1288: outLW.flush();
1289: }
1290: return apt;
1291: }
1292:
1293: private long usedMemory() {
1294: System.gc();
1295: return Runtime.getRuntime().totalMemory()
1296: - Runtime.getRuntime().freeMemory();
1297: }
1298:
1299: /*package*/
1300: void test(String[] args, PrintStream out, PrintStream err)
1301: throws Exception {
1302: tracer.setPrintStream(out);
1303: processArguments(args);
1304: doTest();
1305: initDataObjects();
1306: }
1307:
1308: private TestResult test(NativeFileItem item)
1309: throws FileNotFoundException, RecognitionException,
1310: TokenStreamException, IOException, ClassNotFoundException {
1311:
1312: TestResult result = new TestResult();
1313:
1314: if (testAPT) {
1315: testAPT(item);
1316: if (breakAfterAPT) {
1317: return new TestResult();
1318: }
1319: }
1320:
1321: AST ast = null;
1322:
1323: if (dumpStatistics) {
1324: Diagnostic.initFileStatistics(item.getFile()
1325: .getAbsolutePath());
1326: }
1327:
1328: long time = System.currentTimeMillis();
1329:
1330: if (readAst) {
1331: long t2 = System.currentTimeMillis();
1332: ast = cache.readAst(item.getFile());
1333: t2 = System.currentTimeMillis() - t2;
1334: print("AST read; time: " + t2 + " ms"); // NOI18N
1335: }
1336:
1337: AST tree = null;
1338: int errCount = 0;
1339:
1340: FileImpl fileImpl = (FileImpl) getProject().testAPTParseFile(
1341: item);
1342: waitProjectParsed(false);
1343: if (dumpAst || writeAst || showAstWindow) {
1344: tree = fileImpl.parse(null);
1345: }
1346: errCount = fileImpl.getErrorCount();
1347: if (dumpPPState) {
1348: sleep(100); // so that we don't run ahead of fileParsingFinished event
1349: APTPreprocHandler preprocHandler = states.get(fileImpl);
1350: assert preprocHandler != null;
1351: dumpMacroMap(preprocHandler.getMacroMap());
1352: }
1353: time = System.currentTimeMillis() - time;
1354: if (isShowTime()) {
1355: result.setTime(time);
1356: result.setLineCount(countLines(fileImpl));
1357: if (!quiet) {
1358: print("Processing " + item.getFile().getName()
1359: + " took " + time + " ms; LPS="
1360: + result.getLPS() + "; error count: "
1361: + errCount); // NOI18N
1362: }
1363: }
1364:
1365: if (dumpStatistics) {
1366: if (this .dumpDir != null) {
1367: String postfix = statPostfix;
1368: if (Diagnostic.getStatisticsLevel() > 1) {
1369: postfix += "." + Diagnostic.getStatisticsLevel(); // NOI18N
1370: }
1371: String name = item.getFile().getName() + postfix;
1372: String theDumpFile = new File(this .dumpDir, name)
1373: .getAbsolutePath();
1374: Diagnostic.dumpFileStatistics(theDumpFile);
1375: }
1376: if (this .dumpFile != null) {
1377: Diagnostic.dumpFileStatistics(this .dumpFile, true);
1378: }
1379: }
1380:
1381: if (testCache) {
1382: cacheTimes.put(item.getFile().getName(), new Long(time));
1383: }
1384: if (dumpAst) {
1385: System.out.println("AST DUMP for file "
1386: + item.getFile().getName()); // NOI18N
1387: dumpAst(tree);
1388: }
1389:
1390: if (writeAst && tree != null) {
1391: long t2 = System.currentTimeMillis();
1392: t2 = System.currentTimeMillis() - t2;
1393: cache.writeAst(tree, item.getFile());
1394: print("AST stored; time: " + t2 + " ms"); // NOI18N
1395: }
1396:
1397: if (doCleanRepository) {
1398: CsmProject prj = fileImpl.getProject();
1399: String absPath = fileImpl.getAbsolutePath();
1400: fileImpl = null;
1401: ParserThreadManager.instance().waitEmptyProjectQueue(
1402: (ProjectBase) prj);
1403: sleep(100);
1404: RepositoryAccessor.getRepository().debugClear();
1405: fileImpl = (FileImpl) prj.findFile(absPath);
1406: }
1407:
1408: if (dumpModel) {
1409: if (fileImpl != null) {
1410: tracer.setDeep(deep);
1411: tracer.setTestUniqueName(testUniqueName);
1412: tracer.dumpModel(fileImpl);
1413: if (!dumpFileOnly) {
1414: tracer.dumpModel(getProject());
1415: }
1416: } else {
1417: print("FileImpl is null - not possible to dump File Model"); // NOI18N
1418: }
1419: }
1420:
1421: if (showAstWindow) {
1422: test(tree, item.getFile().getName());
1423: }
1424:
1425: return result;
1426: }
1427:
1428: private boolean hasNonEmptyIncludes(CsmFile fileImpl) {
1429: for (Iterator<CsmInclude> it = fileImpl.getIncludes()
1430: .iterator(); it.hasNext();) {
1431: CsmInclude inc = it.next();
1432: if (inc.getIncludeFile() != null) {
1433: return true;
1434: }
1435: }
1436: return false;
1437: }
1438:
1439: private long countLines(CsmFile fileImpl) {
1440: return countLines(fileImpl, false);
1441: }
1442:
1443: private long countLines(CsmFile fileImpl,
1444: boolean allowResolvedIncludes) {
1445: if (fileImpl == null) {
1446: return -1;
1447: }
1448: if (!allowResolvedIncludes && hasNonEmptyIncludes(fileImpl)) {
1449: //! fileImpl.getIncludes().isEmpty() ) {
1450: return -1;
1451: }
1452: CharSequence text = fileImpl.getText();
1453: long cnt = 0;
1454: for (int pos = 0; pos < text.length(); pos++) {
1455: if (text.charAt(pos) == '\n') {
1456: cnt++;
1457: }
1458: }
1459: return cnt;
1460: }
1461:
1462: private void test(AST tree, String label) {
1463:
1464: // System.out.println("LIST:");
1465: // System.out.println(tree.toStringList());
1466: //
1467: // System.out.println("DUMP:");
1468: // DumpASTVisitor visitor = new DumpASTVisitor();
1469: // visitor.visit(tree);
1470: ASTFrameEx frame = new ASTFrameEx(label, tree);
1471: frame.setVisible(true);
1472: }
1473:
1474: private boolean isDummyUnresolved(CsmDeclaration decl) {
1475: return decl == null
1476: || decl instanceof Unresolved.UnresolvedClass;
1477: }
1478:
1479: public static void dumpAst(AST ast) {
1480: ASTVisitor visitor = new ASTVisitor() {
1481:
1482: public void visit(AST node) {
1483: for (AST node2 = node; node2 != null; node2 = node2
1484: .getNextSibling()) {
1485: String ofStr = (node2 instanceof CsmAST) ? (" offset="
1486: + ((CsmAST) node2).getOffset() + " file = " + ((CsmAST) node2)
1487: .getFilename())
1488: : ""; // NOI18N
1489: print("" + node2.getText() + " [" + node2.getType()
1490: + "] " + node2.getLine() + ':'
1491: + node2.getColumn() + ofStr); // NOI18N
1492: if (node2.getFirstChild() != null) {
1493: indent();
1494: visit(node2.getFirstChild());
1495: unindent();
1496: }
1497: }
1498: }
1499: };
1500: visitor.visit(ast);
1501: }
1502:
1503: private void dumpMacroMap(APTMacroMap macroMap) {
1504: tracer.print("State of macro map:"); // NOI18N
1505: tracer.print(macroMap == null ? "empty macro map" : macroMap
1506: .toString()); // NOI18N
1507: }
1508:
1509: private void testLibProject() {
1510: LibProjectImpl libProject = LibProjectImpl.createInstance(
1511: getModel(), "/usr/include"); // NOI18N
1512: getModel().addProject(libProject);
1513: tracer.dumpModel(libProject);
1514: }
1515:
1516: private static void print(String s) {
1517: tracer.print(s);
1518: }
1519:
1520: private void print(String s, boolean newLine) {
1521: tracer.print(s, newLine);
1522: }
1523:
1524: private static void indent() {
1525: tracer.indent();
1526: }
1527:
1528: private static void unindent() {
1529: tracer.unindent();
1530: }
1531:
1532: private int countUserFiles() {
1533: return getProject().getAllFiles().size();
1534: }
1535:
1536: private int countSystemHeaders() {
1537: int cnt = 0;
1538: Set processedProjects = new HashSet();
1539: for (Iterator it = getProject().getLibraries().iterator(); it
1540: .hasNext();) {
1541: cnt += countFiles((ProjectBase) it.next(),
1542: processedProjects);
1543: }
1544: return cnt;
1545: }
1546:
1547: private int countFiles(ProjectBase prj, Collection processedProjects) {
1548: if (processedProjects.contains(prj)) {
1549: return 0; // already counted
1550: }
1551: int cnt = prj.getAllFiles().size();
1552: for (Iterator it = prj.getLibraries().iterator(); it.hasNext();) {
1553: cnt += countFiles((ProjectBase) it.next(),
1554: processedProjects);
1555: }
1556: return cnt;
1557: }
1558:
1559: private void calculateAverageLPS(TestResult total,
1560: boolean includeLibs) {
1561: total.lineCount = 0;
1562: for (Iterator it = getProject().getAllFiles().iterator(); it
1563: .hasNext();) {
1564: CsmFile file = (CsmFile) it.next();
1565: total.lineCount += countLines(file, true);
1566: }
1567: if (includeLibs) {
1568: for (Iterator it1 = getProject().getLibraries().iterator(); it1
1569: .hasNext();) {
1570: ProjectBase lib = (ProjectBase) it1.next();
1571: for (Iterator it2 = lib.getAllFiles().iterator(); it2
1572: .hasNext();) {
1573: CsmFile file = (CsmFile) it2.next();
1574: total.lineCount += countLines(file, true);
1575: }
1576: }
1577: }
1578: }
1579:
1580: private void testFolding(File file) {
1581: InputStream is;
1582: try {
1583: is = new FileInputStream(file);
1584: } catch (FileNotFoundException ex) {
1585: ex.printStackTrace();
1586: return;
1587: }
1588: if (is == null) {
1589: return;
1590: }
1591: Reader reader = new InputStreamReader(is);
1592: reader = new BufferedReader(reader);
1593: FoldingParser p = Lookup.getDefault().lookup(
1594: FoldingParser.class);
1595: if (p != null) {
1596: List<CppFoldRecord> folds = p.parse(file.getAbsolutePath(),
1597: reader);
1598: try {
1599: reader.close();
1600: } catch (IOException ex) {
1601: ex.printStackTrace();
1602: }
1603: printFolds(file.getAbsolutePath(), folds);
1604: } else {
1605: System.out.println("No Folding Provider"); // NOI18N
1606: }
1607: }
1608:
1609: private void printFolds(String file, List<CppFoldRecord> folds) {
1610: Collections.sort(folds, FOLD_COMPARATOR);
1611: System.out.println("Foldings of the file " + file); // NOI18N
1612: for (Iterator it = folds.iterator(); it.hasNext();) {
1613: CppFoldRecord fold = (CppFoldRecord) it.next();
1614: System.out.println(fold);
1615: }
1616: }
1617:
1618: private static Comparator<CppFoldRecord> FOLD_COMPARATOR = new Comparator<CppFoldRecord>() {
1619:
1620: public int compare(CppFoldRecord o1, CppFoldRecord o2) {
1621: int start1 = o1.getStartLine();
1622: int start2 = o2.getStartLine();
1623: if (start1 == start2) {
1624: return o1.getStartOffset() - o2.getStartOffset();
1625: } else {
1626: return start1 - start2;
1627: }
1628: }
1629: };
1630:
1631: boolean isShowTime() {
1632: return showTime;
1633: }
1634:
1635: private List<NativeFileItem> getFileItems() {
1636: List<NativeFileItem> result = new ArrayList<NativeFileItem>();
1637:
1638: Object platformProject = getProject().getPlatformProject();
1639: if (platformProject instanceof NativeProject) {
1640: NativeProject nativeProject = (NativeProject) platformProject;
1641: if (nativeProject != null) {
1642: result.addAll(nativeProject.getAllFiles());
1643: // these are all files specified in command line; it does not make sense to filter them
1644: // for(NativeFileItem item : nativeProject.getAllFiles()){
1645: // if (!item.isExcluded()) {
1646: // switch(item.getLanguage()){
1647: // case C:
1648: // case CPP:
1649: // case C_HEADER:
1650: // result.add(item);
1651: // break;
1652: // default:
1653: // break;
1654: // }
1655: // }
1656: // }
1657: }
1658: }
1659: return result;
1660: }
1661: }
|