001: /*
002: * This file or a portion of this file is licensed under the terms of
003: * the Globus Toolkit Public License, found in file GTPL, or at
004: * http://www.globus.org/toolkit/download/license.html. This notice must
005: * appear in redistributions of this file, with or without modification.
006: *
007: * Redistributions of this Software, with or without modification, must
008: * reproduce the GTPL in: (1) the Software, or (2) the Documentation or
009: * some other similar material which is provided with the Software (if
010: * any).
011: *
012: * Copyright 1999-2004 University of Chicago and The University of
013: * Southern California. All rights reserved.
014: */
015:
016: package org.griphyn.vdl.directive;
017:
018: import java.io.*;
019: import java.sql.SQLException;
020: import java.util.Iterator;
021: import java.util.List;
022: import java.util.Date;
023: import java.util.ArrayList;
024: import java.util.MissingResourceException;
025:
026: import org.griphyn.common.util.Version;
027: import org.griphyn.common.util.Currently;
028: import org.griphyn.vdl.parser.InvocationParser;
029: import org.griphyn.vdl.invocation.*;
030: import org.griphyn.vdl.dbschema.*;
031: import org.griphyn.vdl.util.Logging;
032: import org.griphyn.vdl.toolkit.FriendlyNudge;
033: import org.griphyn.vdl.util.ChimeraProperties;
034:
035: /**
036: * Main objective of this class is to extract the exit status from
037: * the invocation record returned by kickstart. The expected usage
038: * is another Java class passing a filename, and obtaining the
039: * cooked exit status for the parse. All other details, like removing
040: * non-XML header and tailers, de-concatenation, are handled internally.<p>
041: *
042: * Usage of the class is divided into typically three steps. The first
043: * step is to obtain an instance of the the parser, and configure it
044: * to fit your needs.<p>
045: *
046: * <pre>
047: * ParseKickstart pks = new ParseKickstart();
048: * ... // set flags
049: * pks.setDatabaseSchema( ptcschema );
050: * </pre>
051: *
052: * The next step can be executed multiple times, and parse one or more
053: * kickstart output files.<p>
054: *
055: * <pre>
056: * List result = null;
057: * try {
058: * result = pks.parseFile( file );
059: * } catch ( FriendlyNudge fn ) {
060: * // handle failures
061: * }
062: * </pre>
063: *
064: * Once you are definitely done, it is recommend to dis-associate yourself
065: * from the active database connection.<p>
066: *
067: * <pre>
068: * pks.close();
069: * pks = null;
070: * </pre>
071: *
072: * @author Jens-S. Vöckler
073: * @author Yong Zhao
074: * @version $Revision: 50 $
075: *
076: * @see org.griphyn.vdl.toolkit.ExitCode
077: * @see org.griphyn.vdl.parser.InvocationParser
078: */
079: public class ParseKickstart extends Directive {
080: /**
081: * Determines, if an empty output record constitutes a failure or success.
082: * In old Globus 2.0, empty output frequently occurred. With the NFS
083: * bug alleviation, while not fixed, it occurs a lot less frequently.
084: */
085: private boolean m_emptyFail = true;
086:
087: /**
088: * Determines, if the invocation records go back into the VDC or not.
089: */
090: private boolean m_noDBase = false;
091:
092: /**
093: * Determines, if the invocation records, when incurring a database failure,
094: * will fail the application or not.
095: */
096: private boolean m_ignoreDBFail = false;
097:
098: /**
099: * The database schema driver used to connect to the PTC.
100: */
101: private DatabaseSchema m_dbschema = null;
102:
103: /**
104: * Semi-singleton, dynamically instantiated once for the lifetime.
105: * The properties determine which Xerces parser is being used.
106: */
107: private InvocationParser m_ip = null;
108:
109: /**
110: * Attaches a workflow label (tag) to all workflows passing thru.
111: */
112: private String m_wf_label = null;
113:
114: /**
115: * Attaches a workflow mtime to all workflows passing thru.
116: */
117: private Date m_wf_mtime = null;
118:
119: /**
120: * Default c'tor.
121: */
122: public ParseKickstart() throws IOException,
123: MissingResourceException {
124: super ();
125: }
126:
127: /**
128: * C'tor which permits the setting of a PTC connection.
129: *
130: * @param dbschema is the database schema to use for the PTC.
131: */
132: public ParseKickstart(DatabaseSchema dbschema) throws IOException,
133: MissingResourceException {
134: super ();
135: if ((m_dbschema = dbschema) == null)
136: m_noDBase = true;
137: }
138:
139: /**
140: * C'tor which permits the setting of a PTC connection.
141: *
142: * @param dbschema is the database schema to use for the PTC.
143: * @param emptyFail determines, if empty input files are error or OK.
144: */
145: public ParseKickstart(DatabaseSchema dbschema, boolean emptyFail)
146: throws IOException, MissingResourceException {
147: super ();
148: if ((m_dbschema = dbschema) == null)
149: m_noDBase = true;
150: m_emptyFail = emptyFail;
151: }
152:
153: /**
154: * Sets the database schema.
155: *
156: * @param dbschema is a database schema instance for the PTC.
157: */
158: public void setDatabaseSchema(DatabaseSchema dbschema) {
159: m_dbschema = dbschema;
160: }
161:
162: /**
163: * Closes the associated database backend and invalidates the schema.
164: */
165: public void close() throws SQLException {
166: if (m_dbschema != null)
167: m_dbschema.close();
168: m_dbschema = null;
169: m_ip = null;
170: }
171:
172: /**
173: * Obtains the fail-on-empty-file value.
174: *
175: * @return true, if to fail on empty files.
176: * @see #setEmptyFail( boolean )
177: */
178: public boolean getEmptyFail() {
179: return m_emptyFail;
180: }
181:
182: /**
183: * Sets the fail-on-empty-file value.
184: *
185: * @param emptyFail contains the new value, if to fail on empty files.
186: * @see #getEmptyFail()
187: */
188: public void setEmptyFail(boolean emptyFail) {
189: m_emptyFail = emptyFail;
190: }
191:
192: /**
193: * Gets the variable to permit connections to the PTC, or
194: * use parse-only mode.
195: *
196: * @return true, if the PTC is intended to be used, false for
197: * parse-only mode.
198: * @see #setNoDBase(boolean)
199: */
200: public boolean getNoDBase() {
201: return this .m_noDBase;
202: }
203:
204: /**
205: * Sets the parse-only versus PTC mode.
206: *
207: * @param noDBase is true to use the parse-only mode.
208: * @see #getNoDBase()
209: */
210: public void setNoDBase(boolean noDBase) {
211: this .m_noDBase = noDBase;
212: }
213:
214: /**
215: * Obtains a dont-fail-on-database-errors mode.
216: *
217: * @return true, if database failures are not fatal.
218: * @see #setIgnoreDBFail(boolean)
219: */
220: public boolean getIgnoreDBFail() {
221: return this .m_ignoreDBFail;
222: }
223:
224: /**
225: * Sets the dont-fail-on-dbase-errors mode.
226: *
227: * @param ignore is true to render database error non-fatal.
228: * @see #getIgnoreDBFail()
229: */
230: public void setIgnoreDBFail(boolean ignore) {
231: this .m_ignoreDBFail = ignore;
232: }
233:
234: /**
235: * Obtains the current value of the workflow label to use.
236: *
237: * @return current workflow label to use, may be <code>null</code>.
238: * @see #setWorkflowLabel(String)
239: */
240: public String getWorkflowLabel() {
241: return this .m_wf_label;
242: }
243:
244: /**
245: * Sets the workflow label.
246: *
247: * @param label is the (new) workflow label.
248: * @see #getWorkflowLabel()
249: */
250: public void setWorkflowLabel(String label) {
251: this .m_wf_label = label;
252: }
253:
254: /**
255: * Obtains the current value of the workflow modification time to use.
256: *
257: * @return current workflow mtime, may be <code>null</code>.
258: * @see #setWorkflowTimestamp(Date)
259: */
260: public Date getWorkflowTimestamp() {
261: return this .m_wf_mtime;
262: }
263:
264: /**
265: * Sets the workflow modification time to record.
266: *
267: * @param mtime is the (new) workflow mtime.
268: * @see #getWorkflowTimestamp()
269: */
270: public void setWorkflowTimestamp(Date mtime) {
271: this .m_wf_mtime = mtime;
272: }
273:
274: /**
275: * Determines the exit code of an invocation record. Currently,
276: * we will determine the exit code from all jobs until failure
277: * or no more jobs. However, set-up and clean-up jobs are ignored.
278: *
279: * @param ivr is the invocation record to put into the database
280: * @return the status code as exit code to signal failure etc.
281: * <pre>
282: * 0 regular exit with exit code 0
283: * 1 regular exit with exit code > 0
284: * 2 failure to run program from kickstart
285: * 3 application had died on signal
286: * 4 application was suspended (should not happen)
287: * 5 failure in exit code parsing
288: * 6 impossible case
289: * </pre>
290: */
291: public int determineExitStatus(InvocationRecord ivr) {
292: boolean seen = false;
293: for (Iterator i = ivr.iterateJob(); i.hasNext();) {
294: Job job = (Job) i.next();
295:
296: // set-up/clean-up jobs don't count in failure modes
297: if (job.getTag().equals("cleanup"))
298: continue;
299: if (job.getTag().equals("setup"))
300: continue;
301:
302: // obtains status from job
303: Status status = job.getStatus();
304: if (status == null)
305: return 6;
306:
307: JobStatus js = status.getJobStatus();
308: if (js == null) {
309: // should not happen
310: return 6;
311: } else if (js instanceof JobStatusRegular) {
312: // regular exit code - success or failure?
313: int exitcode = ((JobStatusRegular) js).getExitCode();
314: if (exitcode != 0)
315: return 1;
316: else
317: seen = true;
318: // continue, if exitcode of 0 to implement chaining !!!!
319: } else if (js instanceof JobStatusFailure) {
320: // kickstart failure
321: return 2;
322: } else if (js instanceof JobStatusSignal) {
323: // died on signal
324: return 3;
325: } else if (js instanceof JobStatusSuspend) {
326: // suspended???
327: return 4;
328: } else {
329: // impossible/unknown case
330: return 6;
331: }
332: }
333:
334: // success, or no [matching] jobs
335: return seen ? 0 : 5;
336: }
337:
338: /**
339: * Extracts records from the given input file. Since there may be
340: * more than one record per file, especially in the case of MPI,
341: * multiple results are possible, though traditionally only one
342: * will be used.
343: *
344: * @param input is the name of the file that contains the records
345: * @return a list of strings, each representing one invocation record.
346: * The result should not be empty (exception will be thrown).
347: * @throws FriendlyNudge, if the input format was invalid.
348: * The caller has to assume failure to parse the record provided.
349: */
350: public List extractToMemory(java.io.File input)
351: throws FriendlyNudge {
352: List result = new ArrayList();
353: StringWriter out = null;
354: Logging log = getLogger();
355:
356: // open the files
357: int p1, p2, state = 0;
358: try {
359: BufferedReader in = new BufferedReader(
360: new FileReader(input));
361: out = new StringWriter(4096);
362:
363: String line = null;
364: while ((line = in.readLine()) != null) {
365: if ((state & 1) == 0) {
366: // try to copy the XML line in any case
367: if ((p1 = line.indexOf("<?xml")) > -1)
368: if ((p2 = line.indexOf("?>", p1)) > -1) {
369: out.write(line, p1, p2 + 2);
370: log.log("parser", 2, "state=" + state
371: + ", seen <?xml ...?>");
372: }
373: // start state with the correct root element
374: if ((p1 = line.indexOf("<invocation")) > -1) {
375: if (p1 > 0)
376: line = line.substring(p1);
377: log.log("parser", 2, "state=" + state
378: + ", seen <invocation>");
379: ++state;
380: }
381: }
382: if ((state & 1) == 1) {
383: out.write(line);
384: if ((p1 = line.indexOf("</invocation>")) > -1) {
385: log.log("parser", 2, "state=" + state
386: + ", seen </invocation>");
387: ++state;
388:
389: out.flush();
390: out.close();
391: result.add(out.toString());
392: out = new StringWriter(4096);
393: }
394: }
395: }
396:
397: in.close();
398: out.close();
399: } catch (IOException ioe) {
400: throw new FriendlyNudge("While copying " + input.getPath()
401: + " into temp. file: " + ioe.getMessage(), 5);
402: }
403:
404: // some sanity checks
405: if (state == 0)
406: throw new FriendlyNudge("File " + input.getPath()
407: + " does not contain invocation records,"
408: + " assuming failure", 5);
409: if ((state & 1) == 1)
410: throw new FriendlyNudge("File " + input.getPath()
411: + " contains an incomplete invocation record,"
412: + " assuming failure", 5);
413:
414: // done
415: return result;
416: }
417:
418: /**
419: * Parses the contents of a kickstart output file, and returns a
420: * list of exit codes obtains from the records.
421: *
422: * @param arg0 is the name of the file to read
423: * @return a list with one or more exit code, one for each record.
424: * @throws FriendlyNudge, if parsing of the file goes hay-wire.
425: * @throws IOException if something happens while reading properties
426: * to instantiate the XML parser.
427: * @throws SQLException if accessing the database fails.
428: */
429: public List parseFile(String arg0) throws FriendlyNudge,
430: IOException, SQLException {
431: List result = new ArrayList();
432: Logging me = getLogger();
433: me.log("kickstart", 2, "working with file " + arg0);
434:
435: // get access to the invocation parser
436: if (m_ip == null) {
437: ChimeraProperties props = ChimeraProperties.instance();
438: String psl = props.getPTCSchemaLocation();
439: me.log("kickstart", 2, "using XML schema location " + psl);
440: m_ip = new InvocationParser(psl);
441: }
442:
443: // check input file
444: java.io.File check = new java.io.File(arg0);
445:
446: // test 1: file exists
447: if (!check.exists()) {
448: me.log("kickstart", 2, "file does not exist, fail with 5");
449: throw new FriendlyNudge("file does not exist " + arg0
450: + ", assuming failure", 5);
451: }
452:
453: // test 2: file is readable
454: if (!check.canRead()) {
455: me.log("kickstart", 2, "file not readable, fail with 5");
456: throw new FriendlyNudge("unable to read file " + arg0
457: + ", assuming failure", 5);
458: }
459:
460: // test 3: file has nonzero size
461: // FIXME: Actually need to check the record size
462: me.log("kickstart", 2, "file has size " + check.length());
463: if (check.length() == 0) {
464: // deal with 0-byte file
465: if (getEmptyFail()) {
466: me.log("kickstart", 2, "zero size file, fail with 5");
467: throw new FriendlyNudge("file has zero length " + arg0
468: + ", assuming failure", 5);
469: } else {
470: me
471: .log("kickstart", 2,
472: "zero size file, succeed with 0");
473: me.log("app", 1, "file has zero length " + arg0
474: + ", assuming success");
475: result.add(new Integer(0));
476: return result;
477: }
478: }
479:
480: // test 4: extract XML into tmp file
481: me.log("kickstart", 2, "about to extract content into memory");
482: List extract = extractToMemory(check);
483: me.log("kickstart", 2, extract.size() + " records extracted");
484:
485: // testme: for each record obtained, work on it
486: for (int j = 1; j - 1 < extract.size(); ++j) {
487: String temp = (String) extract.get(j - 1);
488: me.log("kickstart", 2, "content[" + j
489: + "] extracted, length " + temp.length());
490:
491: // test 5: try to parse XML
492: me.log("app", 2, "starting to parse invocation");
493: me.log("kickstart", 2, "about to parse invocation record");
494: InvocationRecord invocation = m_ip.parse(new StringReader(
495: temp));
496: me.log("kickstart", 2, "done parsing invocation");
497:
498: if (invocation == null) {
499: me.log("kickstart", 2, "result record " + j
500: + " is invalid (null), fail with 5");
501: throw new FriendlyNudge(
502: "invalid XML invocation record " + j + " in "
503: + arg0 + ", assuming failure", 5);
504: } else {
505: me.log("kickstart", 2, "result record " + j
506: + " appears valid");
507: me.log("app", 1, "invocation " + j
508: + " was parsed successfully");
509: }
510:
511: // NEW: attached workflow tag and mtime
512: if (m_wf_label != null)
513: invocation.setWorkflowLabel(m_wf_label);
514: if (m_wf_mtime != null)
515: invocation.setWorkflowTimestamp(m_wf_mtime);
516:
517: // insert into database -- iff it is available
518: if (!m_noDBase && m_dbschema != null
519: && m_dbschema instanceof PTC) {
520: PTC ptc = (PTC) m_dbschema;
521:
522: try {
523: // FIXME: (start,host,pid) may not be a sufficient secondary key
524: me.log("kickstart", 2,
525: "about to obtain secondary key triple");
526: if (ptc.getInvocationID(invocation.getStart(),
527: invocation.getHostAddress(), invocation
528: .getPID()) == -1) {
529: me
530: .log("kickstart", 2,
531: "new invocation, adding");
532: me.log("app", 1,
533: "adding invocation to database");
534: // may throw SQLException
535: ptc.saveInvocation(invocation);
536: } else {
537: me.log("kickstart", 2,
538: "existing invocation, skipping");
539: me.log("app", 1,
540: "invocation already exists, skipping!");
541: }
542: } catch (SQLException sql) {
543: if (m_ignoreDBFail) {
544: // if dbase errors are non-fatal, just protocol what is going on.
545: for (int n = 0; sql != null; ++n) {
546: me.log("default", 0,
547: "While inserting PTR [" + j + "]:"
548: + n + ": "
549: + sql.getMessage()
550: + ", ignoring");
551: sql = sql.getNextException();
552: }
553: } else {
554: // rethrow, if dbase errors are fatal (default)
555: throw sql;
556: }
557: } // catch
558: } // if use dbase
559:
560: // determine result code
561: int status = 0;
562: me.log("kickstart", 2, "about to determine exit status");
563: status = determineExitStatus(invocation);
564: me.log("kickstart", 2, "exit status is " + status);
565: result.add(new Integer(status));
566: } // for
567:
568: // done
569: return result;
570: }
571: }
|