001: /*
002: * Copyright 2001 Sun Microsystems, Inc. All rights reserved.
003: * PROPRIETARY/CONFIDENTIAL. Use of this product is subject to license terms.
004: */
005:
006: package com.sun.portal.search.admin;
007:
008: import java.net.URL;
009: import java.util.*;
010: import java.util.logging.Logger;
011: import java.util.logging.Level;
012: import java.io.*;
013:
014: import com.sun.portal.search.robot.ProcessConfig;
015: import com.sun.portal.log.common.PortalLogger;
016:
017: import javax.servlet.http.HttpServletRequest;
018:
019: import com.iplanet.jato.RequestContext;
020:
021: import com.iplanet.jato.view.event.DisplayEvent;
022: import com.iplanet.jato.view.event.RequestInvocationEvent;
023:
024: import com.iplanet.jato.view.html.StaticTextField;
025: import com.iplanet.jato.view.html.TextField;
026: import com.iplanet.jato.view.html.CheckBox;
027: import com.iplanet.jato.view.html.Button;
028: import com.iplanet.jato.view.html.ComboBox;
029: import com.iplanet.jato.view.html.RadioButtonGroup;
030:
031: import com.iplanet.jato.view.html.Option;
032: import com.iplanet.jato.view.html.OptionList;
033:
034: import com.iplanet.jato.view.View;
035: import com.iplanet.jato.view.ViewBean;
036: import com.iplanet.jato.view.ViewBeanBase;
037:
038: import com.iplanet.jato.ViewBeanManager;
039:
040: import com.iplanet.jato.model.ModelControlException;
041:
042: import com.iplanet.am.console.components.view.html.IPlanetButton;
043:
044: import sun.misc.BASE64Encoder;
045: import sun.misc.BASE64Decoder;
046:
047: public class CrawlingViewBean extends CSViewBeanBase {
048:
049: // Create a logger for this class
050: private static Logger debugLogger = PortalLogger
051: .getLogger(CrawlingViewBean.class);
052:
053: /**
054: * constructor
055: *
056: * @param PageName of this view bean
057: * @param displayURL default display URL
058: */
059: public CrawlingViewBean() {
060: super (PAGE_NAME);
061: setDefaultDisplayURL(DEFAULT_DISPLAY_URL);
062: registerChildren();
063: filterConfigurator.readFilterToArray();
064: }
065:
066: /**
067: * register child component
068: */
069: protected void registerChildren() {
070: //
071: // Speed
072: //
073: registerChild(SERVERDELAY_COMBO, ComboBox.class);
074: registerChild(MAXCONRET_COMBO, ComboBox.class);
075: registerChild(MAXCONPER_COMBO, ComboBox.class);
076: registerChild(SENDRDS_COMBO, ComboBox.class);
077: //
078: // Completion Actions
079: //
080: registerChild(SCRIPTLAUNCH_COMBO, ComboBox.class);
081: registerChild(AFTERPROC_COMBO, ComboBox.class);
082: registerChild(CONTACTEMAIL_TEXT, TextField.class);
083: //
084: // Logfile Settings
085: //
086: registerChild(LOGLEV_COMBO, ComboBox.class);
087: //
088: // Standards Compliance
089: //
090:
091: registerChild(USERAGENT_TEXT, TextField.class);
092: registerChild(IGNORERBTSTXT_CHECK, CheckBox.class);
093: //
094: // Authentication Parameters
095: //
096: registerChild(PERFAUTH_RADIO, RadioButtonGroup.class);
097: registerChild(ROBOUSER_TEXT, TextField.class);
098: registerChild(ROBOPASS_TEXT, TextField.class);
099: registerChild(PROXUSER_TEXT, TextField.class);
100: registerChild(PROXPASS_TEXT, TextField.class);
101: //
102: // Proxying
103: //
104: registerChild(TYPEPROX_RADIO, RadioButtonGroup.class);
105: registerChild(ACTYPE_RADIO, RadioButtonGroup.class);
106: registerChild(ACLOCAL_TEXT, TextField.class);
107: registerChild(MCHTTPPROX_TEXT, TextField.class);
108: registerChild(MCHTTPSPROX_TEXT, TextField.class);
109: registerChild(MCFTPPROX_TEXT, TextField.class);
110:
111: // Advanced Settings
112: //
113: registerChild(USECOOKIES_CHECK, CheckBox.class);
114: registerChild(IPASSOURCE_CHECK, CheckBox.class);
115: registerChild(SMTHOST_CHECK, CheckBox.class);
116: registerChild(CNAMERES_CHECK, CheckBox.class);
117: registerChild(DEFAULTSTARTDEPTH_COMBO, TextField.class);
118: registerChild(WORKDIR_TEXT, TextField.class);
119: registerChild(STATEDIR_TEXT, TextField.class);
120: registerChild(ANYHOST_CHECK, CheckBox.class);
121: //
122: // Extracting
123: //
124: registerChild(HTMLLINKS_CHECK, CheckBox.class);
125: registerChild(HTMLLINKS_TEXT, TextField.class);
126: registerChild(PLAINLINKS_CHECK, CheckBox.class);
127: registerChild(PLAINLINKS_TEXT, TextField.class);
128: //
129: // Standard stuff
130: //
131: registerChild(RESET_BUTTON, IPlanetButton.class);
132: registerChild(SUBMIT_BUTTON, IPlanetButton.class);
133: registerChild(SITES_VIEW, SitesListView.class);
134:
135: }
136:
137: /**
138: * create child component
139: *
140: * @param name of component
141: * @return child component
142: */
143: protected View createChild(String name) {
144: View Headerchild = super .createChild(name);
145: if (Headerchild != null)
146: return Headerchild;
147: //
148: // Speed
149: //
150: if (name.equals(SERVERDELAY_COMBO)) {
151: ComboBox child = new ComboBox(this , SERVERDELAY_COMBO, "");
152: OptionList ServerDelayOptions = new OptionList(
153: getLocalizedStringArray(
154: "robot.crawling.speed.servdelaylistlabel",
155: ","), getLocalizedStringArray(
156: "robot.crawling.speed.servdelaylistvalue",
157: ","));
158: child.setOptions(ServerDelayOptions);
159: return child;
160: }
161: if (name.equals(MAXCONRET_COMBO)) {
162: ComboBox child = new ComboBox(this , MAXCONRET_COMBO, "");
163: OptionList MaxConRetOptions = new OptionList(
164: getLocalizedStringArray(
165: "robot.crawling.speed.maxconretlistlabel",
166: ","), getLocalizedStringArray(
167: "robot.crawling.speed.maxconretlistvalue",
168: ","));
169: child.setOptions(MaxConRetOptions);
170: return child;
171: }
172: if (name.equals(MAXCONPER_COMBO)) {
173: ComboBox child = new ComboBox(this , MAXCONPER_COMBO, "");
174: OptionList MaxConPerOptions = new OptionList(
175: getLocalizedStringArray(
176: "robot.crawling.speed.maxconsitelistlabel",
177: ","), getLocalizedStringArray(
178: "robot.crawling.speed.maxconsitelistvalue",
179: ","));
180: child.setOptions(MaxConPerOptions);
181: return child;
182: }
183: if (name.equals(SENDRDS_COMBO)) {
184: ComboBox child = new ComboBox(this , SENDRDS_COMBO, "");
185: OptionList SendRDsOptions = new OptionList(
186: getLocalizedStringArray(
187: "robot.crawling.speed.indexlistlabel", ","),
188: getLocalizedStringArray(
189: "robot.crawling.speed.indexlistvalue", ","));
190: child.setOptions(SendRDsOptions);
191: return child;
192: }
193: //
194: // Completion Actions
195: //
196: if (name.equals(SCRIPTLAUNCH_COMBO)) {
197: ComboBox child = new ComboBox(this , SCRIPTLAUNCH_COMBO, "");
198: child
199: .setLabelForNoneSelected(getLocalizedString("robot.crawling.completion.scriptlaunch.default"));
200: child.setOptions(getScriptLaunchOptions());
201: return child;
202: }
203: if (name.equals(AFTERPROC_COMBO)) {
204: ComboBox child = new ComboBox(this , AFTERPROC_COMBO, "");
205: OptionList AfterProcOptions = new OptionList(
206: getLocalizedStringArray(
207: "robot.crawling.completion.afterprocesslistlabel",
208: ","),
209: getLocalizedStringArray(
210: "robot.crawling.completion.afterprocesslistvalue",
211: ","));
212: child.setOptions(AfterProcOptions);
213: return child;
214: }
215: if (name.equals(CONTACTEMAIL_TEXT)) {
216: return new TextField(this , CONTACTEMAIL_TEXT, "");
217: }
218: //
219: // Logfile Settings
220: //
221: if (name.equals(LOGLEV_COMBO)) {
222: ComboBox child = new ComboBox(this , LOGLEV_COMBO, "1");
223: OptionList LogLevOptions = new OptionList(
224: getLocalizedStringArray(
225: "robot.crawling.log.levellistlabel", "|"),
226: getLocalizedStringArray(
227: "robot.crawling.log.levellistvalue", ","));
228: child.setOptions(LogLevOptions);
229: return child;
230: }
231: //
232: // Standards Compliance
233: //
234: if (name.equals(USERAGENT_TEXT)) {
235: return new TextField(this , USERAGENT_TEXT, "");
236: }
237: if (name.equals(IGNORERBTSTXT_CHECK)) {
238: return new CheckBox(this , IGNORERBTSTXT_CHECK, "false",
239: "true", false);
240: }
241: //
242: // Authentication Parameters
243: //
244: if (name.equals(PERFAUTH_RADIO)) {
245: RadioButtonGroup child = new RadioButtonGroup(this ,
246: PERFAUTH_RADIO, "");
247: child.setHorizontalLayout(true);
248: OptionList PerfAuthOptions = new OptionList(
249: getLocalizedStringArray(
250: "robot.crawling.authentication.performradiolabel",
251: ","),
252: getLocalizedStringArray(
253: "robot.crawling.authentication.performradiovalue",
254: ","));
255: child.setOptions(PerfAuthOptions);
256: return child;
257: }
258: if (name.equals(ROBOUSER_TEXT)) {
259: return new TextField(this , ROBOUSER_TEXT, "");
260: }
261: if (name.equals(ROBOPASS_TEXT)) {
262: return new TextField(this , ROBOPASS_TEXT, "");
263: }
264: if (name.equals(PROXUSER_TEXT)) {
265: return new TextField(this , PROXUSER_TEXT, "");
266: }
267: if (name.equals(PROXPASS_TEXT)) {
268: // String Proxyusername = new BASE64Encoder().decode(PROXPASS_TEXT);
269: return new TextField(this , PROXPASS_TEXT, "");
270: }
271: //
272: // Proxying
273: //
274: if (name.equals(TYPEPROX_RADIO)) {
275: RadioButtonGroup child = new RadioButtonGroup(this ,
276: TYPEPROX_RADIO, "");
277: OptionList TypeProxOptions = new OptionList(
278: getLocalizedStringArray(
279: "robot.crawling.proxy.typeradiolabel", ","),
280: getLocalizedStringArray(
281: "robot.crawling.proxy.typeradiovalue", ","));
282: child.setOptions(TypeProxOptions);
283: return child;
284: }
285: if (name.equals(ACTYPE_RADIO)) {
286: RadioButtonGroup child = new RadioButtonGroup(this ,
287: ACTYPE_RADIO, "");
288: child.setHorizontalLayout(true);
289: OptionList ACTypeOptions = new OptionList(
290: getLocalizedStringArray(
291: "robot.crawling.proxy.autoconfig.typeradiolabel",
292: ","),
293: getLocalizedStringArray(
294: "robot.crawling.proxy.autoconfig.typeradiovalue",
295: ","));
296: child.setOptions(ACTypeOptions);
297: return child;
298: }
299: if (name.equals(ACLOCAL_TEXT)) {
300: return new TextField(this , ACLOCAL_TEXT, "");
301: }
302: if (name.equals(MCHTTPPROX_TEXT)) {
303: return new TextField(this , MCHTTPPROX_TEXT, "");
304: }
305: if (name.equals(MCHTTPSPROX_TEXT)) {
306: return new TextField(this , MCHTTPSPROX_TEXT, "");
307: }
308: if (name.equals(MCFTPPROX_TEXT)) {
309: return new TextField(this , MCFTPPROX_TEXT, "");
310: }
311: //
312: // Advanced Settings
313: //
314: if (name.equals(USECOOKIES_CHECK)) {
315: return new CheckBox(this , USECOOKIES_CHECK, "true",
316: "false", false);
317: }
318: if (name.equals(IPASSOURCE_CHECK)) {
319: return new CheckBox(this , IPASSOURCE_CHECK, "true",
320: "false", true);
321: }
322: if (name.equals(SMTHOST_CHECK)) {
323: return new CheckBox(this , SMTHOST_CHECK, "true", "false",
324: false);
325: }
326: if (name.equals(CNAMERES_CHECK)) {
327: return new CheckBox(this , CNAMERES_CHECK, "true", "false",
328: false);
329: }
330: if (name.equals(ANYHOST_CHECK)) {
331: return new CheckBox(this , ANYHOST_CHECK, "true", "false",
332: false);
333: }
334: if (name.equals(DEFAULTSTARTDEPTH_COMBO)) {
335: ComboBox child = new ComboBox(this ,
336: DEFAULTSTARTDEPTH_COMBO, "");
337: OptionList DefaultStartDepthOptions = new OptionList(
338: getLocalizedStringArray(
339: "robot.crawling.advanced.defaultstartlistlabel",
340: ","),
341: getLocalizedStringArray(
342: "robot.crawling.advanced.defaultstartlistvalue",
343: ","));
344: child.setOptions(DefaultStartDepthOptions);
345: return child;
346: }
347: if (name.equals(WORKDIR_TEXT)) {
348: return new TextField(this , WORKDIR_TEXT, "");
349: }
350: if (name.equals(STATEDIR_TEXT)) {
351: return new TextField(this , STATEDIR_TEXT, "");
352: }
353: //
354: // Extracting
355: //
356: if (name.equals(HTMLLINKS_CHECK)) {
357: return new CheckBox(this , HTMLLINKS_CHECK, "true", "false",
358: true);
359: }
360: if (name.equals(HTMLLINKS_TEXT)) {
361: return new TextField(this , HTMLLINKS_TEXT, "");
362: }
363: if (name.equals(PLAINLINKS_CHECK)) {
364: return new CheckBox(this , PLAINLINKS_CHECK, "true",
365: "false", true);
366: }
367: if (name.equals(PLAINLINKS_TEXT)) {
368: return new TextField(this , PLAINLINKS_TEXT, "");
369: }
370: if (name.equals(SUBMIT_BUTTON)) {
371: return new IPlanetButton(this , SUBMIT_BUTTON, "");
372: }
373: if (name.equals(RESET_BUTTON)) {
374: return new IPlanetButton(this , RESET_BUTTON, "");
375: }
376: if (name.equals(SITES_VIEW)) {
377: return new SitesListView(this , SITES_VIEW);
378: }
379:
380: throw new IllegalArgumentException("Invalid child name ["
381: + name + "]");
382: }
383:
384: /** begin displaying page. we set the required information
385: *
386: * @param event display event
387: * @throws ModelControlException if problem access value of component
388: */
389: public void beginDisplay(DisplayEvent event) {
390: setPageEncoding();
391: setDisplayFieldValue(SUBMIT_BUTTON,
392: getLocalizedString("submit.text"));
393: setDisplayFieldValue(RESET_BUTTON,
394: getLocalizedString("reset.text"));
395:
396: String value;
397: ProcessConfig processConf = CSConfig.getRobotConfig().processConf;
398: // set my decoder function
399: BASE64Decoder dec = new BASE64Decoder();
400:
401: //server delay
402: String serv_delay = (String) processConf.get("server-delay");
403: this .setDisplayFieldValue(this .SERVERDELAY_COMBO, serv_delay);
404: // Max concurrent retrieval
405: String max_con_ret = (String) processConf
406: .get("max-connections");
407: this .setDisplayFieldValue(this .MAXCONRET_COMBO, max_con_ret);
408: // Max connections per site
409: String max_con_per = (String) processConf
410: .get("site-max-connections");
411: this .setDisplayFieldValue(this .MAXCONPER_COMBO, max_con_per);
412: // Send RDs to Index
413: String send_rd = (String) processConf
414: .get("index-after-ngenerated");
415: this .setDisplayFieldValue(this .SENDRDS_COMBO, send_rd);
416: // Script to Launch
417: value = (String) processConf.get("cmd-Hook");
418: this .setDisplayFieldValue(this .SCRIPTLAUNCH_COMBO, value);
419:
420: // After Processing URLs
421: String after_proc = (String) processConf.get("onCompletion");
422: this .setDisplayFieldValue(this .AFTERPROC_COMBO, after_proc);
423: // Contact Email
424: String con_email = (String) processConf.get("email");
425: this .setDisplayFieldValue(this .CONTACTEMAIL_TEXT, con_email);
426: // Log Level
427: String log_lev = (String) processConf.get("loglevel");
428: if (log_lev != null) {
429: this .setDisplayFieldValue(this .LOGLEV_COMBO, log_lev);
430: }
431: // User-Agent
432: String use_agt = (String) processConf.get("user-agent");
433: this .setDisplayFieldValue(this .USERAGENT_TEXT, use_agt);
434: // Ignorance of Robots.txt
435: String rbt_txt = (String) processConf.get("enable-robots-txt");
436: this .setDisplayFieldValue(this .IGNORERBTSTXT_CHECK, rbt_txt);
437: // Perform authentication
438: String perf_auth = (String) processConf
439: .get("perform-authentication");
440: this .setDisplayFieldValue(this .PERFAUTH_RADIO, perf_auth);
441: // Robot Username
442: this .setDisplayFieldValue(ROBOUSER_TEXT, (String) processConf
443: .get("username"));
444: //String rbt_usrn = this.getDisplayFieldStringValue(this.ROBOUSER_TEXT);
445: //processConf.put("username", rbt_usrn);
446: // Robot Password
447: String RobotPassDecoded = null;
448: try {
449: RobotPassDecoded = new String(dec
450: .decodeBuffer((String) processConf.get("password")));
451: } catch (Exception e) {
452: }
453:
454: this .setDisplayFieldValue(ROBOPASS_TEXT, RobotPassDecoded);
455: //String rbt_pass = this.getDisplayFieldStringValue(this.ROBOPASS_TEXT);
456: //processConf.put("password", rbt_pass);
457: // Proxy Username
458: this .setDisplayFieldValue(PROXUSER_TEXT, (String) processConf
459: .get("proxy-username"));
460: //String prox_usrn = this.getDisplayFieldStringValue(this.PROXUSER_TEXT);
461: //processConf.put("proxy-username", prox_usrn);
462: // Proxy Password
463: String userPassDecoded = null;
464: try {
465: userPassDecoded = new String(dec
466: .decodeBuffer((String) processConf
467: .get("proxy-password")));
468: } catch (Exception e) {
469: }
470: this .setDisplayFieldValue(PROXPASS_TEXT, userPassDecoded);
471: //String prox_pass = this.getDisplayFieldStringValue(this.PROXPASS_TEXT);
472: //processConf.put("proxy-password", prox_pass);
473:
474: // Type of Proxy (direct, auto, manual)
475: this .setDisplayFieldValue(TYPEPROX_RADIO, (String) processConf
476: .get("proxy-type"));
477: //String type_prox = this.getDisplayFieldStringValue(this.TYPEPROX_RADIO);
478: //processConf.put("proxy-type", type_prox);
479: // Auto type
480: this .setDisplayFieldValue(ACTYPE_RADIO, (String) processConf
481: .get("proxy-loc"));
482: //String auto_type = this.getDisplayFieldStringValue(this.ACTYPE_RADIO);
483: //processConf.put("proxy-loc", auto_type);
484: // Auto Location
485: this .setDisplayFieldValue(ACLOCAL_TEXT, (String) processConf
486: .get("auto_serv"));
487: //String auto_loc = this.getDisplayFieldStringValue(this.ACLOCAL_TEXT);
488: //processConf.put("auto-proxy", auto_loc);
489: // HTTP Proxy
490: this .setDisplayFieldValue(MCHTTPPROX_TEXT, (String) processConf
491: .get("http_proxy"));
492: //String http_prox = this.getDisplayFieldStringValue(this.MCHTTPPROX_TEXT);
493: //processConf.put("http_proxy", http_prox);
494: // HTTPS Proxy
495: this .setDisplayFieldValue(MCHTTPSPROX_TEXT,
496: (String) processConf.get("https_proxy"));
497: //String https_pass = this.getDisplayFieldStringValue(this.MCHTTPSPROX_TEXT);
498: //processConf.put("https_proxy", https_pass);
499: // FTP PROXY
500: this .setDisplayFieldValue(MCFTPPROX_TEXT, (String) processConf
501: .get("ftp_proxy"));
502: //String ftp_pass = this.getDisplayFieldStringValue(this.MCFTPPROX_TEXT);
503: //processConf.put("ftp_proxy", ftp_pass);
504: // Use Cookies
505: this .setDisplayFieldValue(USECOOKIES_CHECK,
506: (String) processConf.get("enable-cookie"));
507: //String use_cook = this.getDisplayFieldStringValue(this.USECOOKIES_CHECK);
508: //processConf.put("enable-cookie", use_cook);
509: // Use IP as Source
510: this .setDisplayFieldValue(IPASSOURCE_CHECK,
511: (String) processConf.get("enable-ip"));
512: //String ip_source = this.getDisplayFieldStringValue(this.IPASSOURCE_CHECK);
513: //processConf.put("enable-ip", ip_source);
514: // Smart Host Heuristics
515: this .setDisplayFieldValue(SMTHOST_CHECK, (String) processConf
516: .get("smart-host-heuristics"));
517: //String smt_host = this.getDisplayFieldStringValue(this.SMTHOST_CHECK);
518: //processConf.put("smart-host-heuristics", smt_host);
519: // CNAME Resolution
520: this .setDisplayFieldValue(CNAMERES_CHECK, (String) processConf
521: .get("use-dns-cname"));
522: //String cname_res = this.getDisplayFieldStringValue(this.CNAMERES_CHECK);
523: //processConf.put("use-dns-cname", cname_res);
524: // Default Starting Point
525: this .setDisplayFieldValue(DEFAULTSTARTDEPTH_COMBO,
526: (String) processConf.get("depth"));
527: //String def_start = this.getDisplayFieldStringValue(this.DEFAULTSTARTDEPTH_COMBO);
528: //processConf.put("depth", def_start);
529: // Work Directory
530: this .setDisplayFieldValue(WORKDIR_TEXT, (String) processConf
531: .get("tmpdir"));
532: //String work_dir = this.getDisplayFieldStringValue(this.WORKDIR_TEXT);
533: //processConf.put("tmpdir", work_dir);
534: // State Directory
535: this .setDisplayFieldValue(STATEDIR_TEXT, (String) processConf
536: .get("robot-state-dir"));
537: //String state_dir = this.getDisplayFieldStringValue(this.STATEDIR_TEXT);
538: //processConf.put("robot-state-dir", state_dir);
539: // Command Priviledges
540: this .setDisplayFieldValue(ANYHOST_CHECK, (String) processConf
541: .get("remote-access"));
542: //String com_priv = this.getDisplayFieldStringValue(this.ANYHOST_CHECK);
543: //processConf.put("remote-access", com_priv);
544:
545: // Extracting
546: this .setDisplayFieldValue(HTMLLINKS_CHECK, filterConfigurator
547: .getFilterConfVars("Enumerate", "enumerate-urls",
548: "enable", "true"));
549: this .setDisplayFieldValue(HTMLLINKS_TEXT, filterConfigurator
550: .getFilterConfVars("Enumerate", "enumerate-urls",
551: "max", "1024"));
552: this .setDisplayFieldValue(PLAINLINKS_CHECK, filterConfigurator
553: .getFilterConfVars("Enumerate",
554: "enumerate-urls-from-text", "enable", "false"));
555: this .setDisplayFieldValue(PLAINLINKS_TEXT, filterConfigurator
556: .getFilterConfVars("Enumerate",
557: "enumerate-urls-from-text", "max", "1024"));
558:
559: }
560:
561: public void handleSubmitButtonRequest(RequestInvocationEvent event)
562: throws ModelControlException {
563: String value;
564: ProcessConfig processConf = CSConfig.getRobotConfig().processConf;
565: //Server delay
566: String serv_delay = this
567: .getDisplayFieldStringValue(this .SERVERDELAY_COMBO);
568: processConf.put("server-delay", serv_delay);
569: // Max concurrent retrieval
570: String max_con_ret = this
571: .getDisplayFieldStringValue(this .MAXCONRET_COMBO);
572: processConf.put("max-connections", max_con_ret);
573: // Max connections per site
574: String max_con_per = this
575: .getDisplayFieldStringValue(this .MAXCONPER_COMBO);
576: processConf.put("site-max-connections", max_con_per);
577: // Send RDs to Index
578: String send_rd = this
579: .getDisplayFieldStringValue(this .SENDRDS_COMBO);
580: processConf.put("index-after-ngenerated", send_rd);
581: // Script to Launch
582: value = this
583: .getDisplayFieldStringValue(this .SCRIPTLAUNCH_COMBO);
584: processConf.put("cmd-Hook", value);
585: // After Processing URLs
586: String after_proc = this
587: .getDisplayFieldStringValue(this .AFTERPROC_COMBO);
588: processConf.put("onCompletion", after_proc);
589: // Contact Email
590: String con_email = this
591: .getDisplayFieldStringValue(this .CONTACTEMAIL_TEXT);
592: processConf.put("email", con_email);
593: // Log Level
594: String log_lev = this
595: .getDisplayFieldStringValue(this .LOGLEV_COMBO);
596: processConf.put("loglevel", log_lev);
597: // User-Agent
598: String use_agt = this
599: .getDisplayFieldStringValue(this .USERAGENT_TEXT);
600: processConf.put("user-agent", use_agt);
601: // Ignorance of Robots.txt
602: String rbt_txt = this
603: .getDisplayFieldStringValue(this .IGNORERBTSTXT_CHECK);
604: processConf.put("enable-robots-txt", rbt_txt);
605: // Perform authentication
606: String perf_auth = this
607: .getDisplayFieldStringValue(this .PERFAUTH_RADIO);
608: processConf.put("perform-authentication", perf_auth);
609: // Robot Username
610: String rbt_usrn = this
611: .getDisplayFieldStringValue(this .ROBOUSER_TEXT);
612: processConf.put("username", rbt_usrn);
613: // Robot Password
614: String passwdStr = this
615: .getDisplayFieldStringValue(this .ROBOPASS_TEXT);
616: String passwdBase64 = "";
617: if (passwdStr != null && passwdStr.trim().length() > 0) {
618: passwdBase64 = new BASE64Encoder().encode(passwdStr.trim()
619: .getBytes());
620: }
621: processConf.put("password", passwdBase64);
622: // Proxy Username
623: String prox_usrn = this
624: .getDisplayFieldStringValue(this .PROXUSER_TEXT);
625: processConf.put("proxy-username", prox_usrn);
626: // Proxy Password
627: passwdStr = this .getDisplayFieldStringValue(this .PROXPASS_TEXT);
628: passwdBase64 = "";
629: if (passwdStr != null && passwdStr.trim().length() > 0) {
630: passwdBase64 = new BASE64Encoder().encode(passwdStr.trim()
631: .getBytes());
632: }
633: // String storepass = new BASE64Encoder().encode(prox_pass.getBytes());
634: processConf.put("proxy-password", passwdBase64);
635:
636: // Type of Proxy Proxy (direct, auto, manual)
637: String type_prox = this
638: .getDisplayFieldStringValue(this .TYPEPROX_RADIO);
639: processConf.put("proxy-type", type_prox);
640: // Auto type
641: String auto_type = this
642: .getDisplayFieldStringValue(this .ACTYPE_RADIO);
643: processConf.put("proxy-loc", auto_type);
644: // Auto Location
645: String auto_loc = this
646: .getDisplayFieldStringValue(this .ACLOCAL_TEXT);
647: processConf.put("auto_serv", auto_loc);
648: // HTTP Proxy
649: String http_prox = this
650: .getDisplayFieldStringValue(this .MCHTTPPROX_TEXT);
651: processConf.put("http_proxy", http_prox);
652: // HTTPS Proxy
653: String https_pass = this
654: .getDisplayFieldStringValue(this .MCHTTPSPROX_TEXT);
655: processConf.put("https_proxy", https_pass);
656: // FTP PROXY
657: String ftp_pass = this
658: .getDisplayFieldStringValue(this .MCFTPPROX_TEXT);
659: processConf.put("ftp_proxy", ftp_pass);
660: // Use Cookies
661: String use_cook = this
662: .getDisplayFieldStringValue(this .USECOOKIES_CHECK);
663: processConf.put("enable-cookie", use_cook);
664: // Use IP as Source
665: String ip_source = this
666: .getDisplayFieldStringValue(this .IPASSOURCE_CHECK);
667: processConf.put("enable-ip", ip_source);
668: // Smart Host Heuristics
669: String smt_host = this
670: .getDisplayFieldStringValue(this .SMTHOST_CHECK);
671: processConf.put("smart-host-heuristics", smt_host);
672: // CNAME Resolution
673: String cname_res = this
674: .getDisplayFieldStringValue(this .CNAMERES_CHECK);
675: processConf.put("use-dns-cname", cname_res);
676: // Default Starting Point
677: String def_start = this
678: .getDisplayFieldStringValue(this .DEFAULTSTARTDEPTH_COMBO);
679: processConf.put("depth", def_start);
680: // Work Directory
681: String work_dir = this
682: .getDisplayFieldStringValue(this .WORKDIR_TEXT);
683: processConf.put("tmpdir", work_dir);
684: // State Directory
685: String state_dir = this
686: .getDisplayFieldStringValue(this .STATEDIR_TEXT);
687: processConf.put("robot-state-dir", state_dir);
688: // Command Priviledges
689: String com_priv = this
690: .getDisplayFieldStringValue(this .ANYHOST_CHECK);
691: processConf.put("remote-access", com_priv);
692:
693: // Extracting
694: String ext_htmlchk = this
695: .getDisplayFieldStringValue(this .HTMLLINKS_CHECK);
696: filterConfigurator.setFilterConfVars("Enumerate",
697: "enumerate-urls", "enable", ext_htmlchk);
698: String ext_htmltxt = this
699: .getDisplayFieldStringValue(this .HTMLLINKS_TEXT);
700: filterConfigurator.setFilterConfVars("Enumerate",
701: "enumerate-urls", "max", ext_htmltxt);
702: String ext_txtchk = this
703: .getDisplayFieldStringValue(this .PLAINLINKS_CHECK);
704: filterConfigurator.setFilterConfVars("Enumerate",
705: "enumerate-urls-from-text", "enable", ext_txtchk);
706: String ext_txttxt = this
707: .getDisplayFieldStringValue(this .PLAINLINKS_TEXT);
708: filterConfigurator.setFilterConfVars("Enumerate",
709: "enumerate-urls-from-text", "max", ext_txttxt);
710:
711: // callup the write the robots.pac operator.
712: try {
713: setRobotPacFile();
714: } catch (Exception e) {
715: }
716: // Auto proxy
717: processConf.put("auto-proxy", ap);
718:
719: filterConfigurator.updateFilterConf();
720: processConf.updateFile();
721: this .forwardTo();
722: }
723:
724: public void handleResetButtonRequest(RequestInvocationEvent event)
725: throws ModelControlException {
726: clearPageSessionAttributes();
727: forwardTo();
728: }
729:
730: private OptionList getScriptLaunchOptions() {
731: if (ScriptLaunchOptions != null) {
732: return ScriptLaunchOptions;
733: }
734: ScriptLaunchOptions = new OptionList();
735: try {
736: File pathName = new File(CSConfig.getBinPath());
737: String[] scriptsAvail = pathName.list();
738: //
739: // here we pause to play with my little find all the scripts problem
740: //
741: // first add the default option
742: //ScriptLaunchOptions.add(new Option("nothing (default)",""));
743: // now parse out the name list
744: for (int i = 0; i < scriptsAvail.length; i++) {
745: // undersized strings seem to blow things up.
746: if (scriptsAvail[i].length() >= 7) {
747: if (scriptsAvail[i].substring(0, 7).equals(
748: "cmdHook")) {
749: // have a hit, now let's pull the name.
750: try {
751: //A dreamy way to do it that just isn't going to work.
752: // Properties p = new Properties();
753: // p.load(new FileInputStream(CSConfig.getBinPath() + File.separator + scriptsAvail[i]));
754: BufferedReader br = new BufferedReader(
755: new FileReader(CSConfig
756: .getBinPath()
757: + File.separator
758: + scriptsAvail[i]));
759: String line = null;
760: String descLine = null;
761: // this is a really manual way to extract the description, but I cannot think of another way to do it.
762: // it makes me feel kind of retarded to do it this way, though.
763: while ((line = br.readLine()) != null) {
764: if (line.startsWith("#")) {
765: line = line.substring(1);
766: line = line.trim();
767: if (line.startsWith("description")) {
768: descLine = line.substring(12);
769: }
770: }
771: }
772: br.close();
773: ScriptLaunchOptions
774: .add(new Option(
775: descLine,
776: (CSConfig.getBinPath()
777: + File.separator + scriptsAvail[i])));
778: } catch (Exception e) {
779: ScriptLaunchOptions.add(new Option(("["
780: + scriptsAvail[i] + "]"),
781: scriptsAvail[i]));
782: debugLogger.log(Level.INFO,
783: "PSSH_CSPSA0011", new String[] {
784: CSConfig.getBinPath(),
785: File.separator,
786: scriptsAvail[i],
787: e.getMessage() });
788: }
789: }
790: }
791: }
792: } catch (Exception e) {
793: debugLogger.log(Level.INFO, "PSSH_CSPSA0012", e
794: .getMessage());
795: }
796: return ScriptLaunchOptions;
797: // end of setScriptOptions
798: }
799:
800: public void setRobotPacFile() throws IOException {
801: File robotPac = new File(server_root + File.separator
802: + "config" + File.separator + "robot.pac");
803: // Set me strings
804: String tp = this
805: .getDisplayFieldStringValue(this .TYPEPROX_RADIO);
806: String ac = this .getDisplayFieldStringValue(this .ACTYPE_RADIO);
807: String as = this .getDisplayFieldStringValue(this .ACLOCAL_TEXT);
808:
809: // First test to for auto --> local and ignore the file, set proxy to file:auto_proxy
810: if (tp.equals("auto") && ac.equals("local")) {
811: ap = "file:" + as;
812: }
813:
814: // check for Proxy direct and remove the file, set proxy to null
815: if (tp.equals("direct")) {
816: robotPac.delete();
817: ap = "";
818: }
819:
820: // check for Proxy proxy auto --> remote and remove the file, set proxy to auto_proxy
821: if (tp.equals("auto") && ac.equals("server")) {
822: robotPac.delete();
823: ap = as;
824: }
825:
826: // check for Manual Proxy and show your love for the file. Set proxy to file:robot.pac
827: if (tp.equals("manual")) {
828: PrintWriter out = null;
829: try {
830: FileOutputStream rpac = new FileOutputStream(robotPac);
831: out = new PrintWriter(rpac, true);
832: } catch (Exception e) {
833: debugLogger.log(Level.INFO, "PSSH_CSPSA0013", e
834: .getMessage());
835: return;
836: }
837: out
838: .print("function FindProxyForURL(url, host, method) {\n");
839: out
840: .print("var protocol = url.split(\":\")[0].toLowerCase();\n\n");
841:
842: // First the HTTP proxy
843: out.print("if(\"http\" == protocol) ");
844: if (this .getDisplayFieldStringValue(this .MCHTTPPROX_TEXT) != null) {
845: out
846: .print("return \"PROXY "
847: + this
848: .getDisplayFieldStringValue(this .MCHTTPPROX_TEXT)
849: + "\";\n");
850: } else {
851: out.print("return \"DIRECT\";\n");
852: }
853: // Then the HTTPS proxy
854: out.print("if(\"https\" == protocol) ");
855: if (this .getDisplayFieldStringValue(this .MCHTTPSPROX_TEXT) != null) {
856: out
857: .print("return \"PROXY "
858: + this
859: .getDisplayFieldStringValue(this .MCHTTPSPROX_TEXT)
860: + "\";\n");
861: } else {
862: out.print("return \"DIRECT\";\n");
863: }
864: // First the HTTP proxy
865: out.print("if(\"ftp\" == protocol) ");
866: if (this .getDisplayFieldStringValue(this .MCFTPPROX_TEXT) != null) {
867: out
868: .print("return \"PROXY "
869: + this
870: .getDisplayFieldStringValue(this .MCFTPPROX_TEXT)
871: + "\";\n");
872: } else {
873: out.print("return \"DIRECT\";\n");
874: }
875: out.print("else return \"DIRECT\";\n}\n");
876: out.close();
877: ap = "file:" + robotPac;
878: }
879:
880: // end of setRobotPacFile
881: }
882:
883: private String server_root = CSConfig.getServerRoot();
884: public static final String DEFAULT_DISPLAY_URL = "/ps/searchadmin/Crawling.jsp";
885: public static final String PAGE_NAME = "Crawling";
886: // declaring some strings for the script launch list
887: private String ap = null;
888:
889: //
890: // Speed
891: //
892: public static final String SERVERDELAY_COMBO = "ServerDelayCombo";
893: public static final String MAXCONRET_COMBO = "MaxConRetCombo";
894: public static final String MAXCONPER_COMBO = "MaxConPerCombo";
895: public static final String SENDRDS_COMBO = "SendRDsCombo";
896: //
897: // Completion Actions
898: //
899: public static final String SCRIPTLAUNCH_COMBO = "ScriptLaunchCombo";
900: public static final String AFTERPROC_COMBO = "AfterProcCombo";
901: public static final String CONTACTEMAIL_TEXT = "ContactEmailText";
902: //
903: // Logfile Settings
904: //
905: public static final String LOGLEV_COMBO = "LogLevCombo";
906: //
907: // Standards Compliance
908: //
909: public static final String USERAGENT_TEXT = "UserAgentText";
910: public static final String IGNORERBTSTXT_CHECK = "IgnoreRbtsTxtCheck";
911: //
912: // Authentication Parameters
913: //
914: public static final String PERFAUTH_RADIO = "PerfAuthRadio";
915: public static final String ROBOUSER_TEXT = "RoboUserText";
916: public static final String ROBOPASS_TEXT = "RoboPassText";
917: public static final String PROXUSER_TEXT = "ProxUserText";
918: public static final String PROXPASS_TEXT = "ProxPassText";
919: //
920: // Proxying
921: //
922: public static final String TYPEPROX_RADIO = "TypeProxRadio";
923: public static final String ACTYPE_RADIO = "ACTypeRadio";
924: public static final String ACLOCAL_TEXT = "ACLocalText";
925: public static final String MCHTTPPROX_TEXT = "MCHttpProxText";
926: public static final String MCHTTPSPROX_TEXT = "MCHttpsProxText";
927: public static final String MCFTPPROX_TEXT = "MCFtpProxText";
928: //
929: // Advanced Settings
930: //
931: public static final String USECOOKIES_CHECK = "UseCookiesCheck";
932: public static final String IPASSOURCE_CHECK = "IpAsSourceCheck";
933: public static final String SMTHOST_CHECK = "SmtHostCheck";
934: public static final String CNAMERES_CHECK = "CnameResCheck";
935: public static final String DEFAULTSTARTDEPTH_COMBO = "DefaultStartDepthCombo";
936: public static final String WORKDIR_TEXT = "WorkDirText";
937: public static final String STATEDIR_TEXT = "StateDirText";
938: public static final String ANYHOST_CHECK = "AnyHostCheck";
939: //
940: // Extracting
941: //
942: public static final String HTMLLINKS_CHECK = "HTMLLinksCheck";
943: public static final String HTMLLINKS_TEXT = "HTMLLinksText";
944: public static final String PLAINLINKS_CHECK = "PlainLinksCheck";
945: public static final String PLAINLINKS_TEXT = "PlainLinksText";
946: //
947: // standard stuff
948: //
949: public static final String RESET_BUTTON = "ResetButton";
950: public static final String SUBMIT_BUTTON = "SubmitButton";
951: public static final String SITES_VIEW = "SitesList";
952:
953: //
954: // optionlists
955: //
956: private OptionList ScriptLaunchOptions = null;
957:
958: }
|