001: /*
002: *
003: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
004: *
005: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
006: *
007: * The contents of this file are subject to the terms of either the GNU
008: * General Public License Version 2 only ("GPL") or the Common
009: * Development and Distribution License("CDDL") (collectively, the
010: * "License"). You may not use this file except in compliance with the
011: * License. You can obtain a copy of the License at
012: * http://www.netbeans.org/cddl-gplv2.html
013: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
014: * specific language governing permissions and limitations under the
015: * License. When distributing the software, include this License Header
016: * Notice in each file and include the License file at
017: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
018: * particular file as subject to the "Classpath" exception as provided
019: * by Sun in the GPL Version 2 section of the License file that
020: * accompanied this code. If applicable, add the following below the
021: * License Header, with the fields enclosed by brackets [] replaced by
022: * your own identifying information:
023: * "Portions Copyrighted [year] [name of copyright owner]"
024: *
025: * Contributor(s):
026: *
027: * The Original Software is NetBeans. The Initial Developer of the Original
028: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
029: * Microsystems, Inc. All Rights Reserved.
030: *
031: * If you wish your version of this file to be governed by only the CDDL
032: * or only the GPL Version 2, indicate your decision by adding
033: * "[Contributor] elects to include this software in this distribution
034: * under the [CDDL or GPL Version 2] license." If you do not indicate a
035: * single choice of license, a recipient has the option to distribute
036: * your version of this file under either the CDDL, the GPL Version 2 or
037: * to extend the choice of license to its licensees as provided above.
038: * However, if you add GPL Version 2 code and therefore, elected the GPL
039: * Version 2 license, then the option applies only if the new code is
040: * made subject to such option by the copyright holder.
041: */
042: package org.netbeans.xtest.testrunner;
043:
044: import java.io.File;
045: import java.io.FileOutputStream;
046: import java.io.IOException;
047: import java.io.OutputStream;
048: import java.io.PrintWriter;
049: import java.io.StringWriter;
050: import java.util.ArrayList;
051: import java.util.Enumeration;
052: import java.util.Iterator;
053: import junit.extensions.TestDecorator;
054: import junit.framework.Test;
055: import junit.framework.TestCase;
056: import junit.framework.TestResult;
057: import junit.framework.TestSuite;
058: import org.netbeans.junit.AssertionKnownBugError;
059: import org.netbeans.junit.Manager;
060: import org.netbeans.junit.NbPerformanceTest;
061: import org.netbeans.junit.NbTest;
062: import org.netbeans.junit.NbTestCase;
063: import org.netbeans.xtest.pe.xmlbeans.Data;
064: import org.netbeans.xtest.pe.xmlbeans.PerformanceData;
065: import org.netbeans.xtest.pe.xmlbeans.UnitTestCase;
066: import org.netbeans.xtest.pe.xmlbeans.UnitTestSuite;
067: import org.netbeans.xtest.pe.xmlbeans.XMLBean;
068: import org.netbeans.xtest.util.SerializeDOM;
069:
070: /**
071: * Creates XML results. Listens for failures and errors and handles error
072: * messages, times and success rates.
073: *
074: * @author mb115822
075: */
076: public class XMLReporter implements JUnitTestListener {
077:
078: private UnitTestSuite currentTestSuite;
079: private UnitTestCase currentTestCase;
080: private ArrayList runTestCases;
081: private ArrayList performanceData;
082:
083: private long caseTime;
084: private long suiteTime;
085:
086: private long testsTotal = 0;
087: private long testsPassed = 0;
088: private long testsFailed = 0;
089: private long testsErrors = 0;
090: private long testsUnexpectedPassed = 0;
091: private long testsExpectedFailed = 0;
092:
093: private File resultsDirectory;
094: private OutputStream outStream;
095: private File outFile;
096:
097: private static final String DID_NOT_FINISH_TEST = "Did not finish.";
098: private static final String DID_NOT_START_TEST = "Did not start.";
099: private static final String UNKNOWN_TEST = "unknown";
100: private String currentTestName = UNKNOWN_TEST;
101: private String currentClassName = UNKNOWN_TEST;
102: private String currentSuiteName = UNKNOWN_TEST;
103:
104: /** Creates new XMLResultProcessor */
105: public XMLReporter(File resultsDirectory) {
106: //
107: this .resultsDirectory = resultsDirectory;
108: //
109: }
110:
111: public static String stackTraceToString(Throwable t) {
112: StringWriter swr = new StringWriter();
113: t.printStackTrace(new PrintWriter(swr, true));
114: return swr.toString();
115: }
116:
117: public void addError(junit.framework.Test test,
118: java.lang.Throwable throwable) {
119: if (currentTestCase == null) {
120: StringWriter s = new StringWriter();
121: throwable.printStackTrace(new PrintWriter(s));
122: currentTestSuite.xmlat_unexpectedFailure = s.toString();
123:
124: saveCurrentSuite();
125: return;
126: }
127: // check whether result was already set (it can be when XTestErrorManager is registered)
128: if (currentTestCase.xmlat_result == UnitTestCase.TEST_FAIL) {
129: // error takes precedens before fail
130: testsFailed--;
131: }
132: currentTestCase.xml_cdata = stackTraceToString(throwable);
133: currentTestCase.xmlat_message = throwable.getMessage();
134: currentTestCase.xmlat_result = UnitTestCase.TEST_ERROR;
135: testsErrors++;
136: }
137:
138: public void addFailure(junit.framework.Test test,
139: junit.framework.AssertionFailedError assertionFailedError) {
140: // check whether result was already set (it can be when XTestErrorManager is registered)
141: if (currentTestCase.xmlat_result == UnitTestCase.TEST_ERROR) {
142: // error takes precedens before fail
143: return;
144: }
145: // check whether result was already set to pass (it can happen when addFailure is called after endTest)
146: if (currentTestCase.xmlat_result == UnitTestCase.TEST_PASS) {
147: testsPassed--;
148: }
149: currentTestCase.xml_cdata = stackTraceToString(assertionFailedError);
150: currentTestCase.xmlat_message = assertionFailedError
151: .getMessage();
152: currentTestCase.xmlat_result = UnitTestCase.TEST_FAIL;
153: testsFailed++;
154:
155: if (test instanceof NbTest) {
156: String exp_mesg = ((NbTest) test).getExpectedFail();
157: if (assertionFailedError instanceof AssertionKnownBugError) {
158: AssertionKnownBugError assertionKBE = (AssertionKnownBugError) assertionFailedError;
159: currentTestCase.xmlat_result = UnitTestCase.TEST_EXPECTED_FAIL;
160: currentTestCase.xmlat_failReason = "KNOWN BUG "
161: + assertionKBE.getBugID();
162: testsExpectedFailed++;
163: } else if (exp_mesg != null) {
164: currentTestCase.xmlat_result = UnitTestCase.TEST_EXPECTED_FAIL;
165: currentTestCase.xmlat_failReason = exp_mesg;
166: testsExpectedFailed++;
167: }
168: }
169: }
170:
171: public void endTest(junit.framework.Test test) {
172: // test didn't hangs so it isn't error.
173: testsErrors--;
174:
175: // did the test used workdir ?
176: if (test instanceof NbTestCase) {
177: NbTestCase nbtest = (NbTestCase) test;
178: String workdir = nbtest.getWorkDirPath();
179: File workdirFile = new File(workdir);
180: if (workdirFile.exists()) {
181: String rootWorkdir = Manager.getWorkDirPath()
182: + File.separator;
183: String relativePath = XMLBean.cutPrefix(workdir,
184: rootWorkdir);
185: currentTestCase.xmlat_workdir = relativePath;
186: }
187: }
188:
189: currentTestCase.xmlat_time = System.currentTimeMillis()
190: - caseTime;
191: if (currentTestCase.xmlat_result
192: .equals(UnitTestCase.TEST_UNKNOWN)) {
193: // test didn't fail or finished with error -> it passed :-)
194: currentTestCase.xmlat_result = UnitTestCase.TEST_PASS;
195: // reset message
196: currentTestCase.xmlat_message = "";
197: testsPassed++;
198: if (test instanceof NbTest) {
199: String exp_mesg = ((NbTest) test).getExpectedFail();
200: if (exp_mesg != null) {
201: currentTestCase.xmlat_result = UnitTestCase.TEST_UNEXPECTED_PASS;
202: currentTestCase.xmlat_failReason = exp_mesg;
203: testsUnexpectedPassed++;
204: }
205: }
206: }
207: //add data to the current suite
208: if (test instanceof NbPerformanceTest) {
209: NbPerformanceTest.PerformanceData pdata[] = ((NbPerformanceTest) test)
210: .getPerformanceData();
211: for (int i = 0; pdata != null && i < pdata.length; i++) {
212: PerformanceData bean = new PerformanceData();
213: bean.xmlat_name = pdata[i].name;
214: bean.xmlat_value = pdata[i].value;
215: bean.xmlat_unit = pdata[i].unit;
216: bean.xmlat_runOrder = pdata[i].runOrder;
217: bean.xmlat_threshold = pdata[i].threshold;
218: performanceData.add(bean);
219: }
220: currentTestSuite.xmlel_Data[0].xmlel_PerformanceData = (PerformanceData[]) (performanceData
221: .toArray(new PerformanceData[0]));
222: }
223:
224: // save the result
225: saveCurrentSuite();
226: }
227:
228: public void startTest(junit.framework.Test test) {
229: //System.out.println("reporter:startTest() test="+test);
230: if (test instanceof TestCase) {
231: currentTestName = ((TestCase) test).getName();
232: } else {
233: currentTestName = UNKNOWN_TEST;
234: }
235: currentClassName = test.getClass().getName();
236: currentTestCase = null;
237: // finds test case in list of test cases to be executed
238: for (Iterator it = runTestCases.iterator(); it.hasNext();) {
239: UnitTestCase testCaseBean = (UnitTestCase) it.next();
240: if (testCaseBean.xmlat_class.equals(currentClassName)
241: && testCaseBean.xmlat_name.equals(currentTestName)
242: && testCaseBean.getMessage().equals(
243: DID_NOT_START_TEST)) {
244: currentTestCase = testCaseBean;
245: break;
246: }
247: }
248: if (currentTestCase == null) {
249: // Test case not found. It can happen when the same test run for the second time
250: // or a new test case is added in runtime.
251: // we need to create a new test case bean
252: currentTestCase = new UnitTestCase();
253: currentTestCase.xmlat_name = currentTestName;
254: currentTestCase.xmlat_class = currentClassName;
255: currentTestCase.xmlat_result = UnitTestCase.TEST_UNKNOWN;
256: // add the testcase to the current suite
257: runTestCases.add(currentTestCase);
258: currentTestSuite.xmlel_UnitTestCase = (UnitTestCase[]) (runTestCases
259: .toArray(new UnitTestCase[0]));
260: testsTotal++;
261: testsErrors++;
262: }
263: // Change message from "Did not start" to "Did not finish". Message is
264: // then updated in endTest, addFailure or addError.
265: currentTestCase.xmlat_message = DID_NOT_FINISH_TEST;
266:
267: // here comes the statistics for up to date results
268: currentTestSuite.xmlat_testsTotal = testsTotal;
269: currentTestSuite.xmlat_testsPass = testsPassed;
270: currentTestSuite.xmlat_testsFail = testsFailed;
271: currentTestSuite.xmlat_testsError = testsErrors;
272: currentTestSuite.xmlat_testsUnexpectedPass = testsUnexpectedPassed;
273: currentTestSuite.xmlat_testsExpectedFail = testsExpectedFailed;
274:
275: // now get the suite out -- suite is still not completed
276: saveCurrentSuite();
277:
278: caseTime = System.currentTimeMillis();
279: }
280:
281: private void recreateOutput() throws IOException {
282: // cannot recreate output if outStream is not a file
283: if (outFile == null) {
284: throw new IOException(
285: "outFile is not specified - cannot create test results files");
286: }
287: // close the file first
288: if (outStream != null) {
289: outStream.close();
290: }
291: // if file exists, delete (is this required ???)
292: if (outFile.exists()) {
293: if (!outFile.delete()) {
294: throw new IOException("cannot delete file " + outFile);
295: }
296: //outFile.createNewFile();
297: }
298: // create the file and output stream
299: this .outStream = new FileOutputStream(outFile);
300: }
301:
302: /**
303: * The whole testsuite ended.
304: */
305: public void endTestSuite(TestSuite suite, TestResult suiteResult) {
306: // add remaining data and write it to a file
307: //System.out.println("reporter:endTestSuite()");
308: //System.err.println("xmlel_UnitTestCase:"+currentTestSuite.xmlel_UnitTestCase);
309: currentTestSuite.xmlat_time = System.currentTimeMillis()
310: - suiteTime;
311: // here comes the statistics
312: currentTestSuite.xmlat_testsTotal = testsTotal;
313: currentTestSuite.xmlat_testsPass = testsPassed;
314: currentTestSuite.xmlat_testsFail = testsFailed;
315: currentTestSuite.xmlat_testsError = testsErrors;
316: currentTestSuite.xmlat_testsUnexpectedPass = testsUnexpectedPassed;
317: currentTestSuite.xmlat_testsExpectedFail = testsExpectedFailed;
318: // suite is finished
319: currentTestSuite.xmlat_unexpectedFailure = null;
320:
321: // now get it out !!!
322: saveCurrentSuite();
323: }
324:
325: /** The whole testsuite started. */
326: public void startTestSuite(TestSuite suite) {
327: //System.out.println("reporter:startTestSuite()-"+suite.getName());
328: // reset the arrays ...
329: runTestCases = new ArrayList();
330: performanceData = new ArrayList();
331: // create new results file
332: outFile = new File(resultsDirectory, "TEST-" + suite.getName()
333: + ".xml");
334:
335: currentTestSuite = new UnitTestSuite();
336: suiteTime = System.currentTimeMillis();
337: currentSuiteName = suite.getName();
338: currentTestSuite.xmlat_name = currentSuiteName;
339: currentTestSuite.xmlat_timeStamp = new java.sql.Timestamp(
340: suiteTime);
341: currentTestSuite.xmlat_unexpectedFailure = "Suite is not finished";
342: currentTestSuite.xmlel_Data = new Data[] { new Data() };
343:
344: addTestCaseBeans(suite);
345: currentTestSuite.xmlel_UnitTestCase = (UnitTestCase[]) (runTestCases
346: .toArray(new UnitTestCase[0]));
347:
348: testsTotal = runTestCases.size();
349: testsPassed = 0;
350: testsFailed = 0;
351: testsErrors = testsTotal;
352: testsUnexpectedPassed = 0;
353: testsExpectedFailed = 0;
354: saveCurrentSuite();
355: }
356:
357: /** Recursively add test cases from suite to the list of test case beans. */
358: private void addTestCaseBeans(TestSuite suite) {
359: for (Enumeration e = suite.tests(); e.hasMoreElements();) {
360: Test test = (Test) e.nextElement();
361: // get a real test if it is only TestDecorator
362: while (test instanceof TestDecorator) {
363: test = ((TestDecorator) test).getTest();
364: }
365: if (test instanceof TestSuite) {
366: TestSuite subSuite = (TestSuite) test;
367: // add test cases from suite recursively
368: addTestCaseBeans(subSuite);
369: continue;
370: }
371: if (test instanceof NbTestCase) {
372: NbTestCase nbTestCase = (NbTestCase) test;
373: if (!nbTestCase.canRun()) {
374: // test case excluded in cfg file => do not add it to suite
375: continue;
376: }
377: }
378: // now add test case bean to the list
379: UnitTestCase testCaseBean = new UnitTestCase();
380: if (test instanceof TestCase) {
381: testCaseBean.xmlat_name = ((TestCase) test).getName();
382: } else {
383: testCaseBean.xmlat_name = UNKNOWN_TEST;
384: }
385: testCaseBean.xmlat_class = test.getClass().getName();
386: testCaseBean.xmlat_result = UnitTestCase.TEST_UNKNOWN;
387: testCaseBean.xmlat_message = DID_NOT_START_TEST;
388: // add the testcase to the current suite
389: runTestCases.add(testCaseBean);
390: }
391: }
392:
393: private boolean saveCurrentSuite() {
394: try {
395: //System.out.println("reporter:saveCurrentSuite()");
396: org.w3c.dom.Document doc = null;
397: if (currentTestSuite == null) {
398: // there is nothing to save
399: System.err
400: .println("Trying to save a suite.xml, but no current test suite is defined");
401: return false;
402: }
403: doc = currentTestSuite.toDocument();
404: //System.err.println("Try to serialize doc to :"+outStream);
405: // better to recreate output
406: recreateOutput();
407: // serialize !!!
408: SerializeDOM.serializeToStream(doc, outStream);
409: // make sure all data are flushed
410: outStream.flush();
411: // better close it as well
412: outStream.close();
413: outStream = null;
414:
415: } catch (IOException ioe) {
416: System.err
417: .println("XMLResultProcessor.endTestSuite(): Unable to write out test suite in XML: IOException:");
418: ioe.printStackTrace(System.err);
419: cleanOutStreamAndFile();
420: return false;
421: } catch (Exception e) {
422: System.err
423: .println("XMLResultProcessor.endTestSuite(): Unable to write out test suite in XML: XMLBean exception:");
424: e.printStackTrace();
425: cleanOutStreamAndFile();
426: return false;
427: }
428: //System.out.println("reporter:suiteSavedOk()");
429: return true;
430: }
431:
432: private boolean cleanOutStreamAndFile() {
433: boolean result = true;
434: if (outStream != null) {
435: try {
436: outStream.close();
437: outStream = null;
438: } catch (IOException ioe) {
439: System.err
440: .println("XMLResultProcessor.cleanOutStreamAndFile - cannot close ative stream from file"
441: + outFile);
442: result = false;
443: }
444: }
445: if (outFile.exists()) {
446: if (outFile.length() == 0) {
447: if (!outFile.delete()) {
448: System.err
449: .println("XMLResultProcessor.cleanOutStreamAndFile - cannot delete empty file:"
450: + outFile);
451: result = false;
452: }
453:
454: }
455: }
456: return result;
457: }
458: }
|