001: /******************************************************************
002: * File: WGReasonerTester.java
003: * Created by: Dave Reynolds
004: * Created on: 09-Feb-03
005: *
006: * (c) Copyright 2003, 2004, 2005, 2006, 2007, 2008 Hewlett-Packard Development Company, LP
007: * [See end of file]
008: * $Id: WGReasonerTester.java,v 1.30 2008/01/02 12:08:31 andy_seaborne Exp $
009: *****************************************************************/package com.hp.hpl.jena.reasoner.test;
010:
011: import com.hp.hpl.jena.rdf.model.*;
012: import com.hp.hpl.jena.graph.*;
013: import com.hp.hpl.jena.graph.query.*;
014: import com.hp.hpl.jena.rdf.model.impl.PropertyImpl;
015: import com.hp.hpl.jena.rdf.model.impl.ResourceImpl;
016: import com.hp.hpl.jena.reasoner.*;
017: import com.hp.hpl.jena.vocabulary.RDF;
018: import com.hp.hpl.jena.rdf.arp.test.ARPTests;
019:
020: import com.hp.hpl.jena.shared.*;
021:
022: import junit.framework.TestCase;
023: import org.apache.commons.logging.Log;
024: import org.apache.commons.logging.LogFactory;
025:
026: import java.io.*;
027: import java.util.*;
028: import java.net.*;
029:
030: /**
031: * A utility to support execution of the RDFCode working group entailment
032: * tests as specified in <a href="http://www.w3.org/TR/2003/WD-rdf-testcases-20030123/">
033: * http://www.w3.org/TR/2003/WD-rdf-testcases-20030123/</a>.
034: *
035: * <p>The manifest file defines a set of tests. Only the positive and negative
036: * entailment tests are handled by this utility. Each test defines a set
037: * of data files to load. For normal positive entailment tests we check each
038: * triple in the conclusions file to ensure it is included in the inferred
039: * graph. For postive entailment tests which are supposed to entail the
040: * false document we run an additional validation check. For
041: * negative entailment tests which tests all triples in the non-conclusions file
042: * and check that at least one trile is missing. </p>
043: *
044: * @author <a href="mailto:der@hplb.hpl.hp.com">Dave Reynolds</a>
045: * @version $Revision: 1.30 $ on $Date: 2008/01/02 12:08:31 $
046: */
047: public class WGReasonerTester {
048:
049: /** The namespace for the test specification schema */
050: public static final String NS = "http://www.w3.org/2000/10/rdf-tests/rdfcore/testSchema#";
051:
052: /** The base URI in which the files are purported to reside */
053: public static final String BASE_URI = "http://www.w3.org/2000/10/rdf-tests/rdfcore/";
054:
055: /** Default location for the test data */
056: public static final String DEFAULT_BASE_DIR = "testing/wg/";
057:
058: /** The base directory in which the test data is actually stored */
059: final protected String baseDir;
060:
061: /** The rdf class for positive tests */
062: public static final Resource PositiveEntailmentTest;
063:
064: /** The rdf class for positive tests */
065: public static final Resource NegativeEntailmentTest;
066:
067: /** The constant used to indicate an invalid document */
068: public static final Resource FalseDocument;
069:
070: /** The predicate defining the description of the test */
071: public static final Property descriptionP;
072:
073: /** The predicate defining the status of the test */
074: public static final Property statusP;
075:
076: /** The predicate defining the rule sets used */
077: public static final Property entailmentRulesP;
078:
079: /** The predicate defining a premise for the test */
080: public static final Property premiseDocumentP;
081:
082: /** The predicate defining the conclusion from the test */
083: public static final Property conclusionDocumentP;
084:
085: /** The type of the current test */
086: Resource testType;
087:
088: /** List of tests block because they are only intended for non-dt aware processors */
089: public static final String[] blockedTests = {
090: BASE_URI
091: + "datatypes/Manifest.rdf#language-important-for-non-dt-entailment-1",
092: BASE_URI
093: + "datatypes/Manifest.rdf#language-important-for-non-dt-entailment-2",
094: // Additional blocked tests, because we do not implement them ... jjc
095: BASE_URI + "pfps-10/Manifest.rdf#non-well-formed-literal-1",
096: BASE_URI + "xmlsch-02/Manifest.rdf#whitespace-facet-3",
097: // BASE_URI + "xmlsch-02/Manifest.rdf#whitespace-facet-2",
098: // BASE_URI + "xmlsch-02/Manifest.rdf#whitespace-facet-1",
099: // BASE_URI + "datatypes-intensional/Manifest.rdf#xsd-integer-string-incompatible",
100: };
101:
102: // Static initializer for the predicates
103: static {
104: PositiveEntailmentTest = new ResourceImpl(NS,
105: "PositiveEntailmentTest");
106: NegativeEntailmentTest = new ResourceImpl(NS,
107: "NegativeEntailmentTest");
108: FalseDocument = new ResourceImpl(NS, "False-Document");
109: descriptionP = new PropertyImpl(NS, "description");
110: statusP = new PropertyImpl(NS, "status");
111: entailmentRulesP = new PropertyImpl(NS, "entailmentRules");
112: premiseDocumentP = new PropertyImpl(NS, "premiseDocument");
113: conclusionDocumentP = new PropertyImpl(NS, "conclusionDocument");
114: }
115:
116: /** The rdf defining all the tests to be run */
117: protected Model testManifest;
118:
119: protected static Log logger = LogFactory
120: .getLog(WGReasonerTester.class);
121:
122: /**
123: * Constructor.
124: * @param manifest the name of the manifest file defining these
125: * tests - relative to baseDir
126: * @param baseDir override default base directory for the tests and manifest
127: */
128: public WGReasonerTester(String manifest, String baseDir)
129: throws IOException {
130: this .baseDir = baseDir;
131: testManifest = loadFile(manifest);
132: }
133:
134: /**
135: * Constructor.
136: * @param manifest the name of the manifest file defining these
137: * tests - relative to baseDir
138: */
139: public WGReasonerTester(String manifest) throws IOException {
140: this (manifest, DEFAULT_BASE_DIR);
141: }
142:
143: /**
144: * Utility to load a file in rdf/nt/n3 format as a Model.
145: * Files are assumed to be relative to the BASE_URI.
146: * @param file the file name, relative to baseDir
147: * @return the loaded Model
148: */
149: public Model loadFile(String file) throws IOException {
150: String langType = "RDF/XML";
151: if (file.endsWith(".nt")) {
152: langType = "N-TRIPLE";
153: } else if (file.endsWith("n3")) {
154: langType = "N3";
155: }
156: Model result = ModelFactory.createNonreifyingModel();
157: String fname = file;
158: if (fname.startsWith(BASE_URI)) {
159: fname = fname.substring(BASE_URI.length());
160: }
161:
162: /* Change note - jjc
163: * Now use InputStream instead of Reader (general hygine).
164: * Also treat http:.... as URL not local file.
165: */
166: InputStream in;
167: if (baseDir.startsWith("http:")) {
168: in = new URL(baseDir + fname).openStream();
169: } else {
170: in = new FileInputStream(baseDir + fname);
171: }
172: in = new BufferedInputStream(in);
173:
174: result.read(in, BASE_URI + fname, langType);
175: return result;
176: }
177:
178: /**
179: * Load the datafile given by the property name.
180: * @param test the test being processed
181: * @param predicate the property of the test giving the file name to load
182: * @return a graph containing the file contents or an empty graph if the property
183: * is not present
184: * @throws IOException if the property is present but the file can't be found
185: */
186: private Graph loadTestFile(Resource test, Property predicate)
187: throws IOException {
188: if (test.hasProperty(predicate)) {
189: String fileName = test.getRequiredProperty(predicate)
190: .getObject().toString();
191: return loadFile(fileName).getGraph();
192: } else {
193: return Factory.createGraphMem();
194: }
195: }
196:
197: /**
198: * Run all the tests in the manifest
199: * @param reasonerF the factory for the reasoner to be tested
200: * @param testcase the JUnit test case which is requesting this test
201: * @param configuration optional configuration information
202: * @return true if all the tests pass
203: * @throws IOException if one of the test files can't be found
204: * @throws RDFException if the test can't be found or fails internally
205: */
206: public boolean runTests(ReasonerFactory reasonerF,
207: TestCase testcase, Resource configuration)
208: throws IOException {
209: for (Iterator i = listTests().iterator(); i.hasNext();) {
210: String test = (String) i.next();
211: if (!runTest(test, reasonerF, testcase, configuration))
212: return false;
213: }
214: return true;
215: }
216:
217: /**
218: * Return a list of all test names defined in the manifest for this test harness.
219: */
220: public List listTests() {
221: List testList = new ArrayList();
222: ResIterator tests = testManifest.listResourcesWithProperty(
223: RDF.type, PositiveEntailmentTest);
224: while (tests.hasNext()) {
225: testList.add(tests.next().toString());
226: }
227: tests = testManifest.listResourcesWithProperty(RDF.type,
228: NegativeEntailmentTest);
229: while (tests.hasNext()) {
230: testList.add(tests.next().toString());
231: }
232: return testList;
233: }
234:
235: /**
236: * Return the type of the last test run. Nasty hack to enable calling test harness
237: * to interpret the success/fail boolen differently according to test type.
238: */
239: public Resource getTypeOfLastTest() {
240: return testType;
241: }
242:
243: /**
244: * Run a single designated test.
245: * @param uri the uri of the test, as defined in the manifest file
246: * @param reasonerF the factory for the reasoner to be tested
247: * @param testcase the JUnit test case which is requesting this test
248: * @param configuration optional configuration information
249: * @return true if the test passes
250: * @throws IOException if one of the test files can't be found
251: * @throws RDFException if the test can't be found or fails internally
252: */
253: public boolean runTest(String uri, ReasonerFactory reasonerF,
254: TestCase testcase, Resource configuration)
255: throws IOException {
256: return runTestDetailedResponse(uri, reasonerF, testcase,
257: configuration) != FAIL;
258: }
259:
260: static final public int FAIL = -1;
261: static final public int NOT_APPLICABLE = 0;
262: static final public int INCOMPLETE = 1;
263: static final public int PASS = 2;
264:
265: /**
266: * Run a single designated test.
267: * @param uri the uri of the test, as defined in the manifest file
268: * @param reasonerF the factory for the reasoner to be tested
269: * @param testcase the JUnit test case which is requesting this test
270: * @param configuration optional configuration information
271: * @return true if the test passes
272: * @throws IOException if one of the test files can't be found
273: * @throws RDFException if the test can't be found or fails internally
274: */
275:
276: public int runTestDetailedResponse(String uri,
277: ReasonerFactory reasonerF, TestCase testcase,
278: Resource configuration) throws IOException {
279:
280: // Find the specification for the named test
281: Resource test = testManifest.getResource(uri);
282: testType = (Resource) test.getRequiredProperty(RDF.type)
283: .getObject();
284: if (!(testType.equals(NegativeEntailmentTest) || testType
285: .equals(PositiveEntailmentTest))) {
286: throw new JenaException("Can't find test: " + uri);
287: }
288:
289: Statement descriptionS = test.getProperty(descriptionP);
290: String description = (descriptionS == null) ? "no description"
291: : descriptionS.getObject().toString();
292: String status = test.getRequiredProperty(statusP).getObject()
293: .toString();
294: logger.debug("WG test " + test.getURI() + " - " + status);
295: if (!status.equals("APPROVED")) {
296: return NOT_APPLICABLE;
297: }
298:
299: // Skip the test designed for only non-datatype aware processors
300: for (int i = 0; i < blockedTests.length; i++) {
301: if (test.getURI().equals(blockedTests[i]))
302: return NOT_APPLICABLE;
303: }
304:
305: // Load up the premise documents
306: Model premises = ModelFactory.createNonreifyingModel();
307: for (StmtIterator premisesI = test
308: .listProperties(premiseDocumentP); premisesI.hasNext();) {
309: premises.add(loadFile(premisesI.nextStatement().getObject()
310: .toString()));
311: }
312:
313: // Load up the conclusions document
314: Model conclusions = null;
315: Resource conclusionsRes = (Resource) test.getRequiredProperty(
316: conclusionDocumentP).getObject();
317: Resource conclusionsType = (Resource) conclusionsRes
318: .getRequiredProperty(RDF.type).getObject();
319: if (!conclusionsType.equals(FalseDocument)) {
320: conclusions = loadFile(conclusionsRes.toString());
321: }
322:
323: // Construct the inferred graph
324: Reasoner reasoner = reasonerF.create(configuration);
325: InfGraph graph = reasoner.bind(premises.getGraph());
326: Model result = ModelFactory.createModelForGraph(graph);
327:
328: // Check the results against the official conclusions
329: boolean correct = true;
330: int goodResult = PASS;
331: boolean noisy = !(baseDir.equals(DEFAULT_BASE_DIR) || ARPTests.internet);
332: if (testType.equals(PositiveEntailmentTest)) {
333: if (conclusions == null) {
334: // Check that the result is flagged as semantically invalid
335: correct = !graph.validate().isValid();
336: if (noisy) {
337: System.out
338: .println("PositiveEntailmentTest of FalseDoc "
339: + test.getURI()
340: + (correct ? " - OK" : " - FAIL"));
341: }
342: } else {
343: correct = testConclusions(conclusions.getGraph(),
344: result.getGraph());
345: if (!graph.validate().isValid()) {
346: correct = false;
347: }
348: if (noisy) {
349: System.out.println("PositiveEntailmentTest "
350: + test.getURI()
351: + (correct ? " - OK" : " - FAIL"));
352: }
353: }
354: } else {
355: goodResult = INCOMPLETE;
356: // A negative entailment check
357: if (conclusions == null) {
358: // Check the result is not flagged as invalid
359: correct = graph.validate().isValid();
360: if (noisy) {
361: System.out
362: .println("NegativentailmentTest of FalseDoc "
363: + test.getURI()
364: + (correct ? " - OK" : " - FAIL"));
365: }
366: } else {
367: correct = !testConclusions(conclusions.getGraph(),
368: result.getGraph());
369: if (noisy) {
370: System.out.println("NegativeEntailmentTest "
371: + test.getURI()
372: + (correct ? " - OK" : " - FAIL"));
373: }
374: }
375: }
376:
377: // Debug output on failure
378: if (!correct) {
379: logger.debug("Premises: ");
380: for (StmtIterator i = premises.listStatements(); i
381: .hasNext();) {
382: logger.debug(" - " + i.nextStatement());
383: }
384: logger.debug("Conclusions: ");
385: if (conclusions != null) {
386: for (StmtIterator i = conclusions.listStatements(); i
387: .hasNext();) {
388: logger.debug(" - " + i.nextStatement());
389: }
390: }
391: }
392:
393: // Signal the results
394: if (testcase != null) {
395: TestCase.assertTrue("Test: " + test + "\n" + description,
396: correct);
397: }
398: return correct ? goodResult : FAIL;
399: }
400:
401: /**
402: * Test a conclusions graph against a result graph. This works by
403: * translating the conclusions graph into a find query which contains one
404: * variable for each distinct bNode in the conclusions graph.
405: */
406: private boolean testConclusions(Graph conclusions, Graph result) {
407: QueryHandler qh = result.queryHandler();
408: Query query = graphToQuery(conclusions);
409: Iterator i = qh.prepareBindings(query, new Node[] {})
410: .executeBindings();
411: return i.hasNext();
412: }
413:
414: /**
415: * Translate a conclusions graph into a query pattern
416: */
417: public static Query graphToQuery(Graph graph) {
418: HashMap bnodeToVar = new HashMap();
419: Query query = new Query();
420: for (Iterator i = graph.find(null, null, null); i.hasNext();) {
421: Triple triple = (Triple) i.next();
422: query.addMatch(translate(triple.getSubject(), bnodeToVar),
423: translate(triple.getPredicate(), bnodeToVar),
424: translate(triple.getObject(), bnodeToVar));
425: }
426: return query;
427: }
428:
429: /**
430: * Translate a blank node to a variable node
431: * @param node the bNode to translate
432: * @param bnodeToVar a map of translations already known about
433: * @return a variable node
434: */
435: private static Node translate(Node node, HashMap bnodeToVar) {
436: String varnames = "abcdefghijklmnopqrstuvwxyz";
437: if (node.isBlank()) {
438: Node t = (Node) bnodeToVar.get(node);
439: if (t == null) {
440: int i = bnodeToVar.size();
441: if (i > varnames.length()) {
442: throw new ReasonerException(
443: "Too many bnodes in query");
444: }
445: t = Node.createVariable(varnames.substring(i, i + 1));
446: bnodeToVar.put(node, t);
447: }
448: return t;
449: } else {
450: return node;
451: }
452: }
453:
454: }
455:
456: /*
457: (c) Copyright 2003, 2004, 2005, 2006, 2007, 2008 Hewlett-Packard Development Company, LP
458: All rights reserved.
459:
460: Redistribution and use in source and binary forms, with or without
461: modification, are permitted provided that the following conditions
462: are met:
463:
464: 1. Redistributions of source code must retain the above copyright
465: notice, this list of conditions and the following disclaimer.
466:
467: 2. Redistributions in binary form must reproduce the above copyright
468: notice, this list of conditions and the following disclaimer in the
469: documentation and/or other materials provided with the distribution.
470:
471: 3. The name of the author may not be used to endorse or promote products
472: derived from this software without specific prior written permission.
473:
474: THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
475: IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
476: OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
477: IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
478: INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
479: NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
480: DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
481: THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
482: (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
483: THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
484: */
|