001: /*
002: **********************************************************************
003: * Copyright (c) 2002-2006, International Business Machines
004: * Corporation and others. All Rights Reserved.
005: **********************************************************************
006: */
007: package com.ibm.icu.dev.test.perf;
008:
009: import java.io.BufferedReader;
010: import java.io.FileInputStream;
011: import java.io.IOException;
012: import java.io.InputStream;
013: import java.io.InputStreamReader;
014: import java.lang.reflect.InvocationTargetException;
015: import java.lang.reflect.Method;
016: import java.util.ArrayList;
017: import java.util.HashMap;
018: import java.util.HashSet;
019: import java.util.Iterator;
020: import java.util.List;
021: import java.util.Locale;
022: import java.util.Map;
023: import java.util.Set;
024:
025: import com.ibm.icu.dev.tool.UOption;
026: import com.ibm.icu.impl.LocaleUtility;
027:
028: /**
029: * Base class for performance testing framework. To use, the subclass can simply
030: * define one or more instance methods with names beginning with "test" (case
031: * ignored). The prototype of the method is
032: *
033: * PerfTest.Function testTheName()
034: *
035: * The actual performance test will execute on the returned Commond object
036: * (refer to Command Pattern). To call a test from command line, the 'test'
037: * prefix of the test method name can be ignored/removed.
038: *
039: * In addition, the subclass should define a main() method that calls
040: * PerfTest.run() as defined here.
041: *
042: * If the subclasses uses any command line arguments (beyond those handled
043: * automatically by this calss) then it should override PerfTest.setup() to
044: * handle its arguments. If the subclasse needs more sophisticated management
045: * for controlling finding/calling test method, it can replace the default
046: * implementation for PerfTest.testProvider before calling PerfTest.run().
047: *
048: * Example invocation: java -cp classes -verbose:gc
049: * com.ibm.icu.dev.test.perf.UnicodeSetPerf --gc --passes 4 --iterations 100
050: * UnicodeSetAdd [[:l:][:c:]]
051: *
052: * Example output: [GC 511K->192K(1984K), 0.0086170 secs] [GC 704K->353K(1984K),
053: * 0.0059619 secs] [Full GC 618K->371K(1984K), 0.0242779 secs] [Full GC
054: * 371K->371K(1984K), 0.0228649 secs] = testUnicodeSetAdd begin 100 =
055: * testUnicodeSetAdd end 11977 1109044 = testUnicodeSetAdd begin 100 =
056: * testUnicodeSetAdd end 12047 1109044 = testUnicodeSetAdd begin 100 =
057: * testUnicodeSetAdd end 11987 1109044 = testUnicodeSetAdd begin 100 =
058: * testUnicodeSetAdd end 11978 1109044
059: *
060: * The [] lines are emitted by the JVM as a result of the -verbose:gc switch.
061: *
062: * Lines beginning with '=' are emitted by PerfTest: = testUnicodeSetAdd begin
063: * 100 A 'begin' statement contains the name of the setup method, which
064: * determines what test function is measures, and the number of iterations that
065: * will be times. = testUnicodeSetAdd end 12047 1109044 An 'end' statement gives
066: * the name of the setup method again, and then two integers. The first is the
067: * total elapsed time in milliseconds, and the second is the number of events
068: * per iteration. In this example, the time per event is 12047 / (100 * 1109044)
069: * or 108.6 ns/event.
070: *
071: * Raw times are given as integer ms, because this is what the system measures.
072: *
073: * @author Alan Liu
074: * @since ICU 2.4
075: */
076: public abstract class PerfTest {
077: // Command-line options set these:
078: protected boolean verbose;
079: protected String sourceDir;
080: protected String fileName;
081: //protected String resolvedFileName;
082: protected String encoding;
083: protected boolean uselen;
084: protected int iterations;
085: protected int passes;
086: protected int time;
087: protected boolean line_mode;
088: protected boolean bulk_mode;
089: protected Locale locale;
090: protected boolean doPriorGC;
091: protected TestCmdProvider testProvider = new TestPrefixProvider(
092: this );
093:
094: static interface TestCmdProvider {
095: /**
096: * @return The names for all available test.
097: */
098: public Set getAllTestCmdNames();
099:
100: /**
101: * @param name
102: * @return Whether the given name is a test name. The implementation may
103: * have more sophisticated naming control here.
104: * TestCmdProvider.isTestCmd() != Set.contains()
105: */
106: public boolean isTestCmd(String name);
107:
108: /**
109: * @param name
110: * @return the test Command or null
111: */
112: public PerfTest.Function getTestCmd(String name);
113: }
114:
115: /**
116: * Treat all method beginning with 'test' prefix (ignoring case)
117: * for given object as the test methods.
118: */
119: static class TestPrefixProvider implements TestCmdProvider {
120: private Map theTests = null; // Map<string(no case), string(with case)>
121: private Set orgNames = null; // shadow reference, ==theTests, for better output
122: private Object refer;
123:
124: TestPrefixProvider(Object theProvider) {
125: refer = theProvider;
126: }
127:
128: public Set getAllTestCmdNames() {
129: if (theTests == null) {
130: theTests = new HashMap();
131: orgNames = new HashSet();
132: Method[] methods = refer.getClass()
133: .getDeclaredMethods();
134: for (int i = 0; i < methods.length; i++) {
135: String org = methods[i].getName();
136: String name = org.toLowerCase(); // ignoring case
137: // beginning with 'test'
138: // Note: methods named 'test()' are ignored
139: if (name.length() > 4 && name.startsWith("test")) {
140: if (theTests.containsKey(name)) {
141: throw new Error(
142: "Duplicate method name ignoring case: "
143: + name);
144: }
145: theTests.put(name, org);
146: orgNames.add(org);
147: }
148: }
149: }
150: return orgNames; // beggining with 'test', keeping case
151: }
152:
153: /**
154: * The given name will map to a method of the same name, or a method
155: * named "test" + name. Case is ignored.
156: */
157: private String isTestCmd_impl(String name) {
158: getAllTestCmdNames();
159: String tn1 = name.toLowerCase();
160: String tn2 = "test" + tn1;
161: if (theTests.containsKey(tn1)) {
162: return tn1;
163: } else if (theTests.containsKey(tn2)) {
164: return tn2;
165: }
166: return null;
167: }
168:
169: public boolean isTestCmd(String name) {
170: return isTestCmd_impl(name) != null;
171: }
172:
173: public Function getTestCmd(String aname) {
174: String name = (String) theTests.get(isTestCmd_impl(aname));
175: if (name == null) {
176: return null;
177: }
178:
179: try {
180: Method m = refer.getClass().getDeclaredMethod(name,
181: null);
182: return (Function) m.invoke(refer, new Object[] {});
183: } catch (Exception e) {
184: throw new Error(
185: "TestPrefixProvider implementation error. Finding: "
186: + name, e);
187: }
188: }
189: };
190:
191: /**
192: * Subclasses of PerfTest will need to create subclasses of
193: * Function that define a call() method which contains the code to
194: * be timed. They then call setTestFunction() in their "Test..."
195: * method to establish this as the current test functor.
196: */
197: public abstract static class Function {
198:
199: /**
200: * Subclasses must implement this method to do the action to be
201: * measured.
202: */
203: public abstract void call();
204:
205: /**
206: * Subclasses may implement this method to return positive
207: * integer indicating the number of operations in a single
208: * call to this object's call() method. If subclasses do not
209: * override this method, the default implementation returns 1.
210: */
211: public long getOperationsPerIteration() {
212: return 1;
213: }
214:
215: /**
216: * Subclasses may implement this method to return either positive
217: * or negative integer indicating the number of events in a single
218: * call to this object's call() method. If subclasses do not
219: * override this method, the default implementation returns -1,
220: * indicating that events are not applicable to this test.
221: * e.g: Number of breaks / iterations for break iterator
222: */
223: public long getEventsPerIteration() {
224: return -1;
225: }
226:
227: /**
228: * Call call() n times in a tight loop and return the elapsed
229: * milliseconds. If n is small and call() is fast the return
230: * result may be zero. Small return values have limited
231: * meaningfulness, depending on the underlying VM and OS.
232: */
233: public final long time(long n) {
234: long start, stop;
235: start = System.currentTimeMillis();
236: while (n-- > 0) {
237: call();
238: }
239: stop = System.currentTimeMillis();
240: return stop - start; // ms
241: }
242: }
243:
244: /**
245: * Exception indicating a usage error.
246: */
247: public static class UsageException extends Exception {
248: public UsageException(String message) {
249: super (message);
250: }
251:
252: public UsageException() {
253: super ();
254: }
255: }
256:
257: /**
258: * Constructor.
259: */
260: protected PerfTest() {
261: }
262:
263: /**
264: * Framework method. Default implementation does not parse any
265: * extra arguments. Subclasses may override this to parse extra
266: * arguments. Subclass implementations should NOT call the base
267: * class implementation.
268: */
269: protected void setup(String[] args) {
270: if (args.length > 0) {
271: throw new RuntimeException("Extra arguments received");
272: }
273: }
274:
275: /**
276: * These must be kept in sync with getOptions().
277: */
278: static final int HELP1 = 0;
279: static final int HELP2 = 1;
280: static final int VERBOSE = 2;
281: static final int SOURCEDIR = 3;
282: static final int ENCODING = 4;
283: static final int USELEN = 5;
284: static final int FILE_NAME = 6;
285: static final int PASSES = 7;
286: static final int ITERATIONS = 8;
287: static final int TIME = 9;
288: static final int LINE_MODE = 10;
289: static final int BULK_MODE = 11;
290: static final int LOCALE = 12;
291: // Options above here are identical to those in C; keep in sync with C
292: // Options below here are unique to Java; shift down as necessary
293: static final int GARBAGE_COLLECT = 13;
294: static final int LIST = 14;
295:
296: UOption[] getOptions() {
297: return new UOption[] { UOption.HELP_H(),
298: UOption.HELP_QUESTION_MARK(), UOption.VERBOSE(),
299: UOption.SOURCEDIR(),
300: UOption.ENCODING(),
301: UOption.DEF("uselen", 'u', UOption.NO_ARG),
302: UOption.DEF("file-name", 'f', UOption.REQUIRES_ARG),
303: UOption.DEF("passes", 'p', UOption.REQUIRES_ARG),
304: UOption.DEF("iterations", 'i', UOption.REQUIRES_ARG),
305: UOption.DEF("time", 't', UOption.REQUIRES_ARG),
306: UOption.DEF("line-mode", 'l', UOption.NO_ARG),
307: UOption.DEF("bulk-mode", 'b', UOption.NO_ARG),
308: UOption.DEF("locale", 'L', UOption.REQUIRES_ARG),
309:
310: // Options above here are identical to those in C; keep in sync
311: // Options below here are unique to Java
312:
313: UOption.DEF("gc", 'g', UOption.NO_ARG),
314: UOption.DEF("list", (char) -1, UOption.NO_ARG), };
315: }
316:
317: /**
318: * Subclasses should call this method in their main(). run() will
319: * in turn call setup() with any arguments it does not parse.
320: * This method parses the command line and runs the tests given on
321: * the command line, with the given parameters. See the class
322: * description for details.
323: */
324: protected final void run(String[] args) throws Exception {
325: Set testList = parseOptions(args);
326:
327: // Run the tests
328: for (Iterator iter = testList.iterator(); iter.hasNext();) {
329: String meth = (String) iter.next();
330:
331: // Call meth to set up the test
332: // long eventsPerCall = -1;
333: Function testFunction = testProvider.getTestCmd(meth);
334: if (testFunction == null) {
335: throw new RuntimeException(meth
336: + " failed to return a test function");
337: }
338: if (testFunction.getOperationsPerIteration() < 1) {
339: throw new RuntimeException(meth
340: + " returned an illegal operations/iteration()");
341: }
342:
343: long t;
344: //long b = System.currentTimeMillis();
345: long loops = getIteration(meth, testFunction);
346: //System.out.println("The guess cost: " + (System.currentTimeMillis() - b)/1000. + " s.");
347:
348: for (int j = 0; j < passes; ++j) {
349: long events = -1;
350: if (verbose) {
351: if (iterations > 0) {
352: System.out.println("= " + meth + " begin "
353: + iterations);
354: } else {
355: System.out.println("= " + meth + " begin "
356: + time + " seconds");
357: }
358: } else {
359: System.out.println("= " + meth + " begin ");
360: }
361:
362: t = testFunction.time(loops); //ms
363: events = testFunction.getEventsPerIteration();
364:
365: if (verbose) {
366: if (events == -1) {
367: System.out.println("= "
368: + meth
369: + " end "
370: + (t / 1000.0)
371: + " loops: "
372: + loops
373: + " operations: "
374: + testFunction
375: .getOperationsPerIteration());
376: } else {
377: System.out.println("= "
378: + meth
379: + " end "
380: + (t / 1000.0)
381: + " loops: "
382: + loops
383: + " operations: "
384: + testFunction
385: .getOperationsPerIteration()
386: + " events: " + events);
387: }
388: } else {
389: if (events == -1) {
390: System.out.println("= "
391: + meth
392: + " end "
393: + (t / 1000.0)
394: + " "
395: + loops
396: + " "
397: + testFunction
398: .getOperationsPerIteration());
399: } else {
400: System.out.println("= "
401: + meth
402: + " end "
403: + (t / 1000.0)
404: + " "
405: + loops
406: + " "
407: + testFunction
408: .getOperationsPerIteration()
409: + " " + events);
410: }
411: }
412:
413: }
414: }
415: }
416:
417: /**
418: * @param args
419: * @return the method list to call
420: * @throws UsageException
421: */
422: private Set parseOptions(String[] args) throws UsageException {
423:
424: doPriorGC = false;
425: encoding = "";
426: uselen = false;
427: fileName = null;
428: sourceDir = null;
429: //lines = null;
430: line_mode = false;
431: //buffer = null;
432: //bufferLen = 0;
433: verbose = false;
434: bulk_mode = false;
435: passes = iterations = time = -1;
436: locale = null;
437:
438: UOption[] options = getOptions();
439: int remainingArgc = UOption.parseArgs(args, options);
440:
441: if (args.length == 0 || options[HELP1].doesOccur
442: || options[HELP2].doesOccur) {
443: throw new UsageException();
444: }
445:
446: if (options[VERBOSE].doesOccur) {
447: verbose = true;
448: }
449:
450: if (options[SOURCEDIR].doesOccur) {
451: sourceDir = options[SOURCEDIR].value;
452: }
453:
454: if (options[ENCODING].doesOccur) {
455: encoding = options[ENCODING].value;
456: }
457:
458: if (options[USELEN].doesOccur) {
459: uselen = true;
460: }
461:
462: if (options[FILE_NAME].doesOccur) {
463: fileName = options[FILE_NAME].value;
464: }
465:
466: if (options[TIME].doesOccur && options[ITERATIONS].doesOccur) {
467: throw new UsageException(
468: "Cannot specify both '-t time' and '-i iterations'");
469: }
470:
471: if (!options[TIME].doesOccur && !options[ITERATIONS].doesOccur) {
472: throw new UsageException(
473: "Either '-t time' or '-i iteration' must be specified");
474: }
475:
476: if (options[PASSES].doesOccur) {
477: passes = Integer.parseInt(options[PASSES].value);
478: } else {
479: throw new UsageException("'-p pass' must be specified");
480: }
481:
482: if (options[ITERATIONS].doesOccur) {
483: iterations = Integer.parseInt(options[ITERATIONS].value);
484: }
485:
486: if (options[TIME].doesOccur) {
487: time = Integer.parseInt(options[TIME].value);
488: }
489:
490: if (options[LINE_MODE].doesOccur
491: && options[BULK_MODE].doesOccur) {
492: throw new UsageException(
493: "Cannot specify both line mode and bulk mode");
494: }
495:
496: if (options[LINE_MODE].doesOccur) {
497: line_mode = true;
498: bulk_mode = false;
499: }
500:
501: if (options[BULK_MODE].doesOccur) {
502: bulk_mode = true;
503: line_mode = false;
504: }
505:
506: if (options[LOCALE].doesOccur) {
507: locale = LocaleUtility
508: .getLocaleFromName(options[LOCALE].value);
509: }
510:
511: if (options[GARBAGE_COLLECT].doesOccur) {
512: doPriorGC = true;
513: }
514:
515: if (options[LIST].doesOccur) {
516: System.err.println("Available tests:");
517: Set testNames = testProvider.getAllTestCmdNames();
518: for (Iterator iter = testNames.iterator(); iter.hasNext();) {
519: String name = (String) iter.next();
520: System.err.println(" " + name);
521: }
522: System.exit(0);
523: }
524:
525: Set testList = new HashSet();
526: int i, j;
527: for (i = 0; i < remainingArgc; ++i) {
528:
529: // is args[i] a method name?
530: if (testProvider.isTestCmd(args[i])) {
531: testList.add(args[i]);
532: } else {
533: // args[i] is neither a method name nor a number. Pass
534: // everything from here on through to the subclass via
535: // setup().
536: break;
537: }
538: }
539:
540: if (testList.size() < 1) { // run all tests
541: Set testNames = testProvider.getAllTestCmdNames();
542: for (Iterator iter = testNames.iterator(); iter.hasNext();) {
543: String name = (String) iter.next();
544: testList.add(name);
545: }
546: }
547:
548: // Pass remaining arguments, if any, through to the subclass
549: // via setup() method.
550: String[] subclassArgs = new String[remainingArgc - i];
551: for (j = 0; i < remainingArgc; ++j) {
552: subclassArgs[j] = args[i++];
553: }
554: setup(subclassArgs);
555:
556: if (doPriorGC) {
557: // Put the heap in a consistent state
558: gc();
559: }
560: return testList;
561: }
562:
563: /**
564: * Translate '-t time' to iterations (or just return '-i iteration')
565: *
566: * @param meth
567: * @param fn
568: * @return
569: */
570: private long getIteration(String methName, Function fn) {
571: long iter = 0;
572: if (time < 0) { // && iterations > 0
573: iter = iterations;
574: } else { // && iterations < 0
575: // Translate time to iteration
576: // Assuming there is a linear relation between time and iterations
577:
578: if (verbose) {
579: System.out.println("= " + methName + " calibrating "
580: + time + " seconds");
581: }
582:
583: long base = time * 1000;
584: // System.out.println("base :" + base);
585: long seed = 1;
586: long t = 0;
587: while (t < base * 0.9 || base * 1.1 < t) { // + - 10%
588: if (iter == 0 || t == 0) {
589: iter = seed; // start up from 1
590: seed *= 100; // if the method is too fast (t == 0), multiply 100 times
591: // 100 is rational because 'base' is always larger than 1000
592: } else {
593: // If 't' is large enough, use linear function to calculate new iteration
594: //
595: // new iter(base) old iter
596: // -------------- = -------- = k
597: // new time old time
598: //
599: // System.out.println("before guess t: " + t);
600: // System.out.println("before guess iter: " + iter);
601: iter = (long) ((double) iter / t * base); // avoid long cut, eg. 1/10 == 0
602: if (iter == 0) {
603: throw new RuntimeException(
604: "Unable to converge on desired duration");
605: }
606: }
607: t = fn.time(iter);
608: }
609: // System.out.println("final t : " + t);
610: // System.out.println("final i : " + iter);
611: }
612: return iter;
613: }
614:
615: /**
616: * Invoke the runtime's garbage collection procedure repeatedly
617: * until the amount of free memory stabilizes to within 10%.
618: */
619: protected void gc() {
620: if (false) {
621: long last;
622: long free = 1;
623: Runtime runtime = Runtime.getRuntime();
624: do {
625: runtime.gc();
626: last = free;
627: free = runtime.freeMemory();
628: } while (((double) Math.abs(free - last)) / free > 0.1);
629: // Wait for the change in free memory to drop under 10%
630: // between successive calls.
631: }
632:
633: // From "Java Platform Performance". This is the procedure
634: // recommended by Javasoft.
635: try {
636: System.gc();
637: Thread.sleep(100);
638: System.runFinalization();
639: Thread.sleep(100);
640:
641: System.gc();
642: Thread.sleep(100);
643: System.runFinalization();
644: Thread.sleep(100);
645: } catch (InterruptedException e) {
646: }
647: }
648:
649: /**
650: * Private utility to convert a List of Integer objects to int[].
651: */
652: private static int[] toIntArray(List list) {
653: int[] result = new int[list.size()];
654: for (int i = 0; i < list.size(); ++i) {
655: result[i] = ((Integer) list.get(i)).intValue();
656: }
657: return result;
658: }
659:
660: public static char[] readToEOS(InputStreamReader stream) {
661: ArrayList vec = new ArrayList();
662: int count = 0;
663: int pos = 0;
664: final int MAXLENGTH = 0x8000; // max buffer size - 32K
665: int length = 0x80; // start with small buffers and work up
666: do {
667: pos = 0;
668: length = length >= MAXLENGTH ? MAXLENGTH : length * 2;
669: char[] buffer = new char[length];
670: try {
671: do {
672: int n = stream.read(buffer, pos, length - pos);
673: if (n == -1) {
674: break;
675: }
676: pos += n;
677: } while (pos < length);
678: } catch (IOException e) {
679: }
680: vec.add(buffer);
681: count += pos;
682: } while (pos == length);
683:
684: char[] data = new char[count];
685: pos = 0;
686: for (int i = 0; i < vec.size(); ++i) {
687: char[] buf = (char[]) vec.get(i);
688: int len = Math.min(buf.length, count - pos);
689: System.arraycopy(buf, 0, data, pos, len);
690: pos += len;
691: }
692: return data;
693: }
694:
695: public static byte[] readToEOS(InputStream stream) {
696:
697: ArrayList vec = new ArrayList();
698: int count = 0;
699: int pos = 0;
700: final int MAXLENGTH = 0x8000; // max buffer size - 32K
701: int length = 0x80; // start with small buffers and work up
702: do {
703: pos = 0;
704: length = length >= MAXLENGTH ? MAXLENGTH : length * 2;
705: byte[] buffer = new byte[length];
706: try {
707: do {
708: int n = stream.read(buffer, pos, length - pos);
709: if (n == -1) {
710: break;
711: }
712: pos += n;
713: } while (pos < length);
714: } catch (IOException e) {
715: }
716: vec.add(buffer);
717: count += pos;
718: } while (pos == length);
719:
720: byte[] data = new byte[count];
721: pos = 0;
722: for (int i = 0; i < vec.size(); ++i) {
723: byte[] buf = (byte[]) vec.get(i);
724: int len = Math.min(buf.length, count - pos);
725: System.arraycopy(buf, 0, data, pos, len);
726: pos += len;
727: }
728: return data;
729: }
730:
731: public String[] readLines(String fileName, String encoding,
732: boolean bulkMode) {
733: FileInputStream fis = null;
734: InputStreamReader isr = null;
735: BufferedReader br = null;
736: try {
737: fis = new FileInputStream(fileName);
738: isr = new InputStreamReader(fis, encoding);
739: br = new BufferedReader(isr);
740: } catch (Exception e) {
741: System.err.println("Error: File access exception: "
742: + e.getMessage() + "!");
743: System.exit(1);
744: }
745: ArrayList list = new ArrayList();
746: while (true) {
747: String line = null;
748: try {
749: line = readDataLine(br);
750: } catch (Exception e) {
751: System.err.println("Read File Error" + e.getMessage()
752: + "!");
753: System.exit(1);
754: }
755: if (line == null)
756: break;
757: if (line.length() == 0)
758: continue;
759: list.add(line);
760: }
761:
762: int size = list.size();
763: String[] lines = null;
764:
765: if (bulkMode) {
766: lines = new String[1];
767: StringBuffer buffer = new StringBuffer("");
768: for (int i = 0; i < size; ++i) {
769: buffer.append((String) list.get(i));
770: /*if (i < (size - 1)) {
771: buffer.append("\r\n");
772: }*/
773: }
774: lines[0] = buffer.toString();
775: } else {
776: lines = new String[size];
777: for (int i = 0; i < size; ++i) {
778: lines[i] = (String) list.get(i);
779: }
780: }
781:
782: return lines;
783: }
784:
785: public String readDataLine(BufferedReader br) throws Exception {
786: String originalLine = "";
787: String line = "";
788: try {
789: line = originalLine = br.readLine();
790: if (line == null)
791: return null;
792: if (line.length() > 0 && line.charAt(0) == 0xFEFF)
793: line = line.substring(1);
794: int commentPos = line.indexOf('#');
795: if (commentPos >= 0)
796: line = line.substring(0, commentPos);
797: line = line.trim();
798: } catch (Exception e) {
799: throw new Exception("Line \"{0}\", \"{1}\"" + originalLine
800: + " " + line + " " + e.toString());
801: }
802: return line;
803: }
804:
805: }
806:
807: //eof
|