001: /*
002: * Copyright 2001 Sun Microsystems, Inc. All rights reserved.
003: * PROPRIETARY/CONFIDENTIAL. Use of this product is subject to license terms.
004: */
005:
006: package com.sun.portal.search.rdmgr;
007:
008: import com.sun.portal.search.util.*;
009: import com.sun.portal.search.rdm.*;
010: import com.sun.portal.search.soif.*;
011: import com.sun.portal.search.db.*;
012: import java.io.File;
013: import java.io.FileOutputStream;
014: import java.io.IOException;
015:
016: import java.util.*;
017: import java.util.logging.Level;
018: import java.util.logging.LogRecord;
019:
020: // XXX this should be moved to the database service, or db layer
021:
022: /**
023: * RDSubmit protocol - Used to add/delete/update RD's.
024: */
025: public class RDSubmit {
026: static final int PARTIAL_TEXT_SIZE_LIMIT = 4096;
027: static final int DESCRIPTION_SIZE_LIMIT = 200;
028:
029: protected void log_rds(RDSubmitRequest req, int nobjs, String oper) {
030: String op;
031: if (oper.equals(RDM.SUBMIT_INSERT))
032: op = "Updated";
033: else if (oper.equals(RDM.SUBMIT_UPDATE))
034: op = "Updated";
035: else if (oper.equals(RDM.SUBMIT_RETRIEVE))
036: op = "Retrieved";
037: else if (oper.equals(RDM.SUBMIT_DELETE))
038: op = "Deleted";
039: else
040: op = "???";
041: if (req.progress != null)
042: req.progress.reportProgress(op, nobjs);
043: }
044:
045: protected void convertDocument(SOIF s) {
046: if (s.contains("rd-file-binary")) {
047: byte[] b = s.getBytes("rd-file-binary");
048: if (b != null) {
049: s.replace("content-length", Integer.toString(b.length));
050: }
051: Date now = new Date();
052: String tmpDir = SearchConfig.getSearchConfig().getValue(
053: "tmpdir");
054: String serverRoot = SearchConfig.getSearchConfig()
055: .getValue("server-root");
056: if (tmpDir == null) {
057: tmpDir = "/tmp";
058: }
059: DocumentConverter dc = new DocumentConverter(serverRoot,
060: tmpDir);
061: dc.ConvertToSOIF(s, "rd-file-binary", "partial-text", true);
062: String desc = s.getValue("description");
063: String partialText = s.getValue("partial-text");
064: if (partialText != null
065: && partialText.length() > PARTIAL_TEXT_SIZE_LIMIT) {
066: s.replace("partial-text", partialText.substring(0,
067: PARTIAL_TEXT_SIZE_LIMIT - 1));
068: }
069: if (desc == null || desc.length() == 0) {
070: if (partialText != null) {
071: if (partialText.length() < DESCRIPTION_SIZE_LIMIT) {
072: s.replace("description", partialText);
073: } else {
074: s.replace("description", partialText.substring(
075: 0, DESCRIPTION_SIZE_LIMIT - 1));
076: }
077: }
078: }
079: String name = s.getValue("rd-file-name");
080: if (name != null) {
081: s.insert("title", name);
082: s.remove("rd-file-name");
083: }
084: }
085: }
086:
087: public int process_input(SToken st, RDSubmitRequest req,
088: RDMTransaction t) throws RDMgrException {
089:
090: int rds_processed = 0;
091:
092: try {
093:
094: SOIF s = null, submitHeader = null;
095: String avp = null;
096: boolean should_report_lmt = false;
097: boolean should_report_rdlmt = false;
098: boolean firstTime = true;
099: int on_deck = 0;
100:
101: for (;;) {
102:
103: // Get the next SOIF
104: try {
105: s = req.sis.readSOIF();
106: } catch (Exception e) {
107: // log - Skipped input object: {0}
108: SearchLogger.getLogger().log(Level.WARNING,
109: "PSSH_CSPSRDMR0049", e.getMessage());
110: continue;
111: }
112: if (s == null)
113: break; // EOS
114:
115: String key;
116: if (req.is_taxonomy)
117: key = s.getValue("id"); // XXX should change categories to use URL, or create db.getKey(SOIF s)
118: else
119: key = s.getURL();
120:
121: // handle missing request header, eg, from rdmgr (XXX need a pushback SOIF stream)
122: if (submitHeader == null
123: && !s.getSchemaName().equalsIgnoreCase(
124: "Request")) {
125: submitHeader = new SOIF("Request", "-");
126: // log - creating dummy submit request
127: SearchLogger.getLogger().log(Level.FINEST,
128: "PSSH_CSPSRDMR0050");
129: if (req.csid != null) // XXX csid should be default here? (used to be)
130: submitHeader.insert(RDM.SUBMIT_CSID, req.csid);
131: submitHeader.insert(RDM.SUBMIT_OPER, req.oper);
132: if (req.type != null)
133: submitHeader.insert(RDM.SUBMIT_TYPE, req.type);
134: if (req.view != null)
135: submitHeader.insert(RDM.SUBMIT_VIEW, req.view);
136: process_header(req, submitHeader);
137: } else {
138: // Check for new request
139: if (s.getSchemaName().equalsIgnoreCase("Request")) {
140: submitHeader = s;
141: // log - handling submit request
142: SearchLogger.getLogger().log(Level.FINE,
143: "PSSH_CSPSRDMR0051");
144: req.do_response = true;
145: if (req.progress != null)
146: req.progress
147: .reportProgress("handling_submit_request");
148: process_header(req, submitHeader);
149: continue;
150: }
151: }
152:
153: if (!req.oper.equalsIgnoreCase(RDM.SUBMIT_RETRIEVE)) {
154:
155: // log - s.schema_name {0}
156: SearchLogger.getLogger().log(Level.FINE,
157: "PSSH_CSPSRDMR0052", s.getSchemaName());
158:
159: // XXX should enforce allowed/required attrs here
160: // XXX All this should be moved to IndexedSOIFDB
161:
162: RDMSchema schema = RDMSchema.getSchema(s
163: .getSchemaName());
164:
165: // Skip anything not conforming to the schema
166: // XXX use db allowed object list here
167: if (req.schema_check
168: && (schema == null
169: || req.is_taxonomy
170: && !s.getSchemaName()
171: .equalsIgnoreCase(
172: "CLASSIFICATION")
173: || req.is_rootdb
174: && !s.getSchemaName()
175: .equalsIgnoreCase(
176: "DATABASE") || (!req.is_taxonomy && !req.is_rootdb)
177: && !s.getSchemaName()
178: .equalsIgnoreCase(
179: "DOCUMENT"))) {
180: // log - *** skipped non-schema soif: @{0} { {1}
181: SearchLogger
182: .getLogger()
183: .log(
184: Level.WARNING,
185: "PSSH_CSPSRDMR0053",
186: new Object[] {
187: s.getSchemaName(), key });
188: continue;
189: }
190: SearchLogger.getLogger().log(Level.FINEST,
191: "PSSH_CSPSRDMR0054", key);
192:
193: // XXX this should be schema controlled
194: // Map aliases into normalized attribute - RDs only
195: if (!req.skip_alias_conf && req.alias != null
196: && !req.is_taxonomy)
197: req.alias.map_soif(s);
198:
199: }
200:
201: // Process this RD
202: this .convertDocument(s);
203: Exception ex = null;
204: try {
205:
206: int flags = 0;
207: if (req.type
208: .equalsIgnoreCase(RDM.SUBMIT_PERSISTENT))
209: flags = RDMDb.PONLY;
210: else if (req.type
211: .equalsIgnoreCase(RDM.SUBMIT_NONPERSISTENT))
212: flags = RDMDb.NPONLY;
213: if ((req.which_pass & req.RDM_SAGE) == 0)
214: flags |= RDMDb.NOINDEX;
215: if ((req.which_pass & req.RDM_CSDB) == 0)
216: flags |= RDMDb.NOSTORE;
217: if (req.do_reindex)
218: flags |= RDMDb.NOSTORE;
219:
220: if (req.oper.equalsIgnoreCase(RDM.SUBMIT_INSERT))
221: req.db.store(st, s, null, flags, null);
222: else if (req.oper
223: .equalsIgnoreCase(RDM.SUBMIT_UPDATE))
224: req.db
225: .update(st, s, req.view_attr, flags,
226: null);
227: else if (req.oper
228: .equalsIgnoreCase(RDM.SUBMIT_MERGE))
229: // legacy - merge is same as update with view list
230: req.db
231: .update(st, s, req.view_attr, flags,
232: null);
233: else if (req.oper
234: .equalsIgnoreCase(RDM.SUBMIT_RETRIEVE)) {
235: SOIF outsoif;
236: /*if (req.db.getClass().getName().equals("com.sun.portal.search.IndexedSOIFDb")) {
237: outsoif = req.db.fetch(st, key, req.view_attr, flags, null);
238: if (outsoif != null)
239: req.sos.write(outsoif);
240: }
241: else {
242: req.sos.write(s);
243: }*/
244: if (req.query != null) {
245: // we did a query - we already have the result SOIF
246: req.sos.write(s);
247: } else {
248: // we were given a list - need to fetch the RD from the db
249: outsoif = req.db.fetch(st, key,
250: req.view_attr, flags, null);
251: if (outsoif != null)
252: req.sos.write(outsoif);
253: }
254: } else if (req.oper
255: .equalsIgnoreCase(RDM.SUBMIT_DELETE))
256: req.db.delete(st, s, null, flags, null);
257: else {
258: // log - Unsupported operation: {1}
259: SearchLogger.getLogger().log(Level.WARNING,
260: "PSSH_CSPSRDMR0055", req.oper);
261: error(req, "-", "Unsupported operation: "
262: + req.oper);
263: throw new RDMgrException();
264: }
265: } catch (Exception e) {
266: ex = e;
267: }
268:
269: rds_processed++;
270: // log - Processed <{0}> op={1} type={2}
271: SearchLogger.getLogger().log(Level.FINE,
272: "PSSH_CSPSRDMR0056",
273: new Object[] { key, req.oper, req.type });
274: if (ex != null) {
275: if (SearchLogger.getLogger().isLoggable(
276: Level.WARNING)) {
277: // log - Error processing <URL {0}> op={1} type={2}
278: LogRecord logRecord = new LogRecord(
279: Level.WARNING, "PSSH_CSPSRDMR0057");
280: logRecord.setParameters(new Object[] { key,
281: req.oper, req.type });
282: logRecord.setThrown(ex);
283: logRecord.setLoggerName(SearchLogger
284: .getLogger().getName());
285: SearchLogger.getLogger().log(logRecord);
286: }
287: }
288:
289: if ((rds_processed > 0)
290: && ((rds_processed % req.log_interval) == 0))
291: log_rds(req, rds_processed, req.oper);
292:
293: if (!req.oper.equalsIgnoreCase(RDM.SUBMIT_RETRIEVE)
294: && !req.oper
295: .equalsIgnoreCase(RDM.SUBMIT_DELETE))
296: ++on_deck;
297:
298: if (on_deck >= req.max_rd_batch) {
299: if (req.progress != null)
300: req.progress.reportProgress(
301: "Indexing_batch_of", on_deck);
302: try {
303: //((NovaDb)req.db).indexBatch(st); // XXX
304: //((IndexedSOIFDb)req.db).indexBatch(st); // XXX
305: req.db.indexBatch(st);
306: } catch (Exception e) {
307: // log - Error indexing batch
308: SearchLogger.getLogger().log(Level.WARNING,
309: "PSSH_CSPSRDMR0058", e);
310: throw new RDMgrException();
311: }
312: on_deck = 0;
313: }
314:
315: } // for all SOIFs
316:
317: // check last batch
318: // XXX This whole batching thing should be done by the db layer...
319: if (on_deck > 0) {
320: if (req.progress != null) // XXX use benign progress impl instead of null?
321: req.progress.reportProgress("Indexing_batch_of",
322: on_deck);
323: try {
324: //((NovaDb)req.db).indexBatch(st); // XXX
325: //((IndexedSOIFDb)req.db).indexBatch(st); // XXX
326: req.db.indexBatch(st);
327: } catch (Exception e) {
328: // log - Error indexing batch
329: SearchLogger.getLogger().log(Level.WARNING,
330: "PSSH_CSPSRDMR0058", e);
331: throw new RDMgrException();
332: }
333: on_deck = 0;
334: }
335:
336: } finally {
337: // finish up
338: // nothing
339: }
340: return rds_processed;
341: }
342:
343: void gen_view_attr(RDSubmitRequest req) {
344: if (req.view != null) {
345: req.view_attr = new HashSet();
346: StringTokenizer st = new StringTokenizer(req.view, " ,");
347: int i = 0;
348: while (st.hasMoreTokens()) {
349: req.view_attr.add(st.nextToken());
350: ++i;
351: }
352: }
353: }
354:
355: void error(RDSubmitRequest req, String url, String msg) {
356:
357: SOIF error = new SOIF(RDM.SUBMIT_ERROR, url);
358:
359: if (req.csid != null)
360: error.insert(RDM.SUBMIT_CSID, req.csid);
361: if (req.type != null)
362: error.insert(RDM.SUBMIT_TYPE, req.type);
363: if (req.oper != null)
364: error.insert(RDM.SUBMIT_OPER, req.oper);
365: if (req.view != null)
366: error.insert(RDM.SUBMIT_VIEW, req.view);
367: if (msg != null)
368: error.insert(RDM.SUBMIT_MESSAGE, msg);
369: try {
370: req.sos.write(error);
371: } catch (Exception e) {
372: // log - Output error
373: SearchLogger.getLogger().log(Level.WARNING,
374: "PSSH_CSPSRDMR0059", e);
375: }
376:
377: }
378:
379: void response(RDSubmitRequest req, String msg) {
380:
381: if (!req.do_response)
382: return;
383:
384: SOIF response = new SOIF(RDM.SUBMIT_RESPONSE, "-");
385:
386: if (req.csid != null)
387: response.insert(RDM.SUBMIT_CSID, req.csid);
388: if (req.type != null)
389: response.insert(RDM.SUBMIT_TYPE, req.type);
390: if (req.oper != null)
391: response.insert(RDM.SUBMIT_OPER, req.oper);
392: if (req.view != null)
393: response.insert(RDM.SUBMIT_VIEW, req.view);
394: if (msg != null)
395: response.insert(RDM.SUBMIT_MESSAGE, msg);
396: try {
397: req.sos.write(response);
398: } catch (Exception e) {
399: // log - Output error
400: SearchLogger.getLogger().log(Level.WARNING,
401: "PSSH_CSPSRDMR0059", e);
402: }
403:
404: }
405:
406: void process_header(RDSubmitRequest req, SOIF soif)
407: throws RDMgrException {
408:
409: try {
410:
411: String p;
412: if (soif == null
413: || !soif.getSchemaName().equalsIgnoreCase(
414: RDM.SUBMIT_REQUEST)) {
415: throw new Exception("unsupported object type: "
416: + soif.getSchemaName());
417: }
418:
419: if ((p = soif.getValue(RDM.SUBMIT_TYPE)) == null) {
420: throw new Exception("missing " + RDM.SUBMIT_TYPE
421: + " parameter");
422: }
423: req.type = p;
424:
425: if ((p = soif.getValue(RDM.SUBMIT_OPER)) == null) {
426: throw new Exception("missing " + RDM.SUBMIT_OPER
427: + " parameter");
428: }
429: req.oper = p;
430:
431: /* XXX not supporting db per req yet
432: if ((p = soif.getValue(RDM.SUBMIT_DB)) == null) {
433: throw new Exception("missing " + RDM.SUBMIT_DB + " parameter");
434: }
435: req.dbname = p;
436: req.db = ...
437: */
438:
439: if ((p = soif.getValue(RDM.SUBMIT_VIEW)) != null) {
440: req.view = p;
441: gen_view_attr(req);
442: }
443:
444: if (!(
445: // valid types
446: req.type.equalsIgnoreCase(RDM.SUBMIT_PERSISTENT)
447: || req.type
448: .equalsIgnoreCase(RDM.SUBMIT_NONPERSISTENT) || req.type
449: .equalsIgnoreCase(RDM.SUBMIT_MERGED))
450: || !(
451: // valid ops
452: req.oper.equalsIgnoreCase(RDM.SUBMIT_INSERT)
453: || req.oper
454: .equalsIgnoreCase(RDM.SUBMIT_DELETE)
455: || req.oper
456: .equalsIgnoreCase(RDM.SUBMIT_RETRIEVE)
457: || req.oper
458: .equalsIgnoreCase(RDM.SUBMIT_MERGE) || req.oper
459: .equalsIgnoreCase(RDM.SUBMIT_UPDATE)) || (
460: // 'merged' is only supported for retrieve and delete
461: req.type.equalsIgnoreCase(RDM.SUBMIT_MERGED) && !(req.oper
462: .equalsIgnoreCase(RDM.SUBMIT_RETRIEVE) || req.oper
463: .equalsIgnoreCase(RDM.SUBMIT_DELETE))) || (
464: // 'view' is only supported for retrieve and update
465: req.view != null && !(req.oper
466: .equalsIgnoreCase(RDM.SUBMIT_RETRIEVE) || req.oper
467: .equalsIgnoreCase(RDM.SUBMIT_UPDATE)))) {
468: throw new Exception("unsupported operation");
469: }
470:
471: // normalise the request
472: if (req.type == null) {
473: if (req.oper.equals(RDM.SUBMIT_DELETE)
474: || req.oper.equals(RDM.SUBMIT_RETRIEVE))
475: req.type = RDM.SUBMIT_MERGED; // merged is default for retrieve and delete
476: else
477: req.type = RDM.SUBMIT_MERGED; // merged is default for retrieve and delete
478: req.type = RDM.SUBMIT_NONPERSISTENT; // default for insert, update, modify
479: }
480:
481: }
482:
483: catch (Exception e) {
484: response(req, null);
485: // log - Bad submit header: {0}
486: SearchLogger.getLogger().log(Level.WARNING,
487: "PSSH_CSPSRDMR0060", e.getMessage());
488: error(req, "-", e.getMessage());
489: throw new RDMgrException();
490: }
491:
492: response(req, null);
493:
494: }
495:
496: }
|