001: package net.matuschek.jobo;
002:
003: import java.awt.*;
004: import java.net.MalformedURLException;
005: import java.net.URL;
006: import java.text.DecimalFormat;
007: import java.util.Date;
008: import javax.swing.*;
009:
010: import net.matuschek.http.HttpTool;
011: import net.matuschek.http.HttpToolCallback;
012: import net.matuschek.http.cookie.Cookie;
013: import net.matuschek.http.cookie.CookieException;
014: import net.matuschek.spider.WebRobotCallback;
015: import net.matuschek.swing.OptionPanel;
016: import net.matuschek.swing.VerticalAlignPanel;
017:
018: import org.apache.log4j.BasicConfigurator;
019:
020: /**
021: * The JoBo interface with Swing GUI.
022: *
023: * @author Daniel Matuschek
024: * @version $Revision: 1.52 $
025: */
026: public class JoBoSwing extends JFrame implements HttpToolCallback,
027: WebRobotCallback {
028: /** JoBo version */
029: final static String VERSION = "1.4";
030:
031: //private static Category log = Category.getInstance("");
032: private static LogFrame logFrame;
033: private static LogFrameAppender lfAppend = null;
034: private JoBoBase jobobase = null;
035: private Thread robotThread = null;
036: private Date docDownloadStarted = new Date();
037:
038: private long httpToolDocSize = 0;
039: private int robotCount = 0;
040: private long robotSize = 0;
041: private int robotQueueSize = 0;
042:
043: // Swing variables declaration
044: private JTextField urlField;
045: private JTextField directoryField;
046: private JProgressBar progressBar;
047: private JTextField currentUrlField;
048: private JTextField queuedField;
049: private JTextField retrievedField;
050:
051: private JButton runStopButton;
052: private JButton sleepButton;
053: // End of Swing variables
054:
055: // Other frames
056:
057: private RobotConfigFrame robotConfigFrame = null;
058: private URLCheckConfigFrame urlCheckConfigFrame = null;
059: private FilterConfigFrame filterConfigFrame = null;
060: private AllowedListFrame allowedURLsFrame = null;
061:
062: // End of other frames
063:
064: /** Creates new form JoBoSwing */
065: public JoBoSwing() {
066:
067: try {
068: jobobase = JoBoBase.createFromXML();
069: } catch (ClassNotFoundException e) {
070: System.out.println("Could not initialize WebRobot: " + e);
071: System.exit(1);
072: }
073:
074: jobobase.registerHttpToolCallback(this );
075: jobobase.registerWebRobotCallback(this );
076:
077: initComponents();
078: pack();
079:
080: updateDialogFromRobot();
081: }
082:
083: private void initComponents() {
084: getContentPane().setLayout(new BorderLayout());
085:
086: setTitle("JoBo/Swing: http://www.matuschek.net/jobo/");
087: addWindowListener(new java.awt.event.WindowAdapter() {
088: public void windowClosing(java.awt.event.WindowEvent evt) {
089: exitApp();
090: }
091: });
092:
093: /** Heading **/
094: VerticalAlignPanel headPanel = new VerticalAlignPanel();
095:
096: JLabel heading1 = new JLabel();
097: heading1.setText("JoBo " + VERSION);
098: heading1.setHorizontalAlignment(JTextField.CENTER);
099: heading1.setFont(new java.awt.Font("Dialog", 0, 24));
100:
101: JLabel heading2 = new JLabel();
102: heading2.setText("look at http://www.matuschek.net/jobo/");
103: heading2.setHorizontalAlignment(JTextField.CENTER);
104: heading2.setFont(new java.awt.Font("Dialog", 0, 12));
105:
106: JLabel heading3 = new JLabel();
107: heading3.setText("for more details");
108: heading3.setHorizontalAlignment(JTextField.CENTER);
109: heading3.setFont(new java.awt.Font("Dialog", 0, 12));
110:
111: headPanel.add(heading1);
112: headPanel.add(heading2);
113: headPanel.add(heading3);
114:
115: getContentPane().add(headPanel, BorderLayout.NORTH);
116:
117: /* fields */
118: OptionPanel optPanel = new OptionPanel(2);
119:
120: urlField = new JTextField();
121: urlField.setColumns(40);
122: urlField.setText("http://");
123: optPanel.add("URL:", urlField);
124:
125: directoryField = new JTextField();
126: directoryField.setText(jobobase.getStorageDirectory());
127: directoryField.setColumns(40);
128: optPanel.add("Storage directory:", directoryField);
129:
130: currentUrlField = new JTextField();
131: currentUrlField.setEditable(false);
132: currentUrlField.setColumns(40);
133: currentUrlField.setText("-");
134: optPanel.add("Current URL:", currentUrlField);
135:
136: retrievedField = new JTextField();
137: retrievedField.setEditable(false);
138: retrievedField.setColumns(40);
139: optPanel.add("Retrieved:", retrievedField);
140:
141: queuedField = new JTextField();
142: queuedField.setEditable(false);
143: queuedField.setColumns(40);
144: optPanel.add("Queued:", queuedField);
145:
146: progressBar = new JProgressBar();
147: progressBar.setStringPainted(true);
148: optPanel.add("Progress:", progressBar,
149: GridBagConstraints.HORIZONTAL);
150:
151: getContentPane().add(optPanel);
152:
153: /** Buttons **/
154: JPanel buttonPanel = new JPanel();
155: JButton butt = null;
156:
157: runStopButton = new JButton();
158: runStopButton.setText("Run");
159: runStopButton
160: .addActionListener(new java.awt.event.ActionListener() {
161: public void actionPerformed(
162: java.awt.event.ActionEvent evt) {
163: runStopButtonActionPerformed();
164: }
165: });
166: buttonPanel.add(runStopButton);
167:
168: sleepButton = new JButton();
169: sleepButton.setText("Sleep");
170: sleepButton
171: .addActionListener(new java.awt.event.ActionListener() {
172: public void actionPerformed(
173: java.awt.event.ActionEvent evt) {
174: sleepButtonActionPerformed();
175: }
176: });
177: buttonPanel.add(sleepButton);
178:
179: butt = new JButton();
180: butt.setText("Robot settings");
181: butt.addActionListener(new java.awt.event.ActionListener() {
182: public void actionPerformed(java.awt.event.ActionEvent evt) {
183: doRobotSettingsDialog();
184: }
185: });
186: buttonPanel.add(butt);
187:
188: butt = new JButton();
189: butt.setText("Save");
190: butt.addActionListener(new java.awt.event.ActionListener() {
191: public void actionPerformed(java.awt.event.ActionEvent evt) {
192: saveSettings();
193: }
194: });
195: // buttonPanel.add(butt);
196:
197: butt = new JButton();
198: butt.setText("URLCheck");
199: butt.addActionListener(new java.awt.event.ActionListener() {
200: public void actionPerformed(java.awt.event.ActionEvent evt) {
201: configureURLCheck();
202: }
203: });
204: // buttonPanel.add(butt);
205:
206: butt = new JButton();
207: butt.setText("Filter configuration");
208: butt.addActionListener(new java.awt.event.ActionListener() {
209: public void actionPerformed(java.awt.event.ActionEvent evt) {
210: configureFilters();
211: }
212: });
213:
214: // buttonPanel.add(butt);
215:
216: butt = new JButton();
217: butt.setText("Allowed URLs");
218: butt.addActionListener(new java.awt.event.ActionListener() {
219: public void actionPerformed(java.awt.event.ActionEvent evt) {
220: configureAllowedURLs();
221: }
222: });
223: buttonPanel.add(butt);
224:
225: butt = new JButton();
226: butt.setText("Add cookie");
227: butt.addActionListener(new java.awt.event.ActionListener() {
228: public void actionPerformed(java.awt.event.ActionEvent evt) {
229: addCookie();
230: }
231: });
232: buttonPanel.add(butt);
233:
234: butt = new JButton();
235: butt.setText("Log");
236: butt.addActionListener(new java.awt.event.ActionListener() {
237: public void actionPerformed(java.awt.event.ActionEvent evt) {
238: logButtonActionPerformed();
239: }
240: });
241: buttonPanel.add(butt);
242:
243: butt = new JButton();
244: butt.setText("Exit");
245: butt.addActionListener(new java.awt.event.ActionListener() {
246: public void actionPerformed(java.awt.event.ActionEvent evt) {
247: exitApp();
248: }
249: });
250: buttonPanel.add(butt);
251:
252: getContentPane().add(buttonPanel, BorderLayout.SOUTH);
253: }
254:
255: private void sleepButtonActionPerformed() {
256:
257: // is there a robotThread running ?
258: if ((robotThread != null) && (robotThread.isAlive())) {
259:
260: if (jobobase.getRobot().isSleeping()) {
261: jobobase.getRobot().setSleep(false);
262: sleepButton.setText("Sleep");
263: } else {
264: jobobase.getRobot().setSleep(true);
265: sleepButton.setText("Wake up");
266: }
267: }
268: }
269:
270: private void logButtonActionPerformed() {
271: logFrame.setVisible(true);
272: }
273:
274: private void runStopButtonActionPerformed() {
275:
276: // is there another robotThread running ?
277: if ((robotThread != null) && (robotThread.isAlive())) {
278: jobobase.getRobot().stopRobot();
279: } else {
280: if (updateRobotFromDialog()) {
281: runStopButton.setText("Stop");
282: // clear cookies from last run
283: //jobobase.getRobot().clearCookies();
284: robotThread = new Thread(jobobase.getRobot());
285: robotThread.start();
286: }
287: }
288: }
289:
290: /**
291: * open the robot settings dialog
292: */
293: private void doRobotSettingsDialog() {
294: if (robotConfigFrame == null) {
295: robotConfigFrame = new RobotConfigFrame(jobobase);
296: }
297: robotConfigFrame.setVisible(true);
298: }
299:
300: /**
301: * configure URLCheck
302: */
303: private void configureURLCheck() {
304: if (urlCheckConfigFrame == null) {
305: urlCheckConfigFrame = new URLCheckConfigFrame(jobobase
306: .getURLCheck());
307: }
308: urlCheckConfigFrame.setVisible(true);
309: }
310:
311: /**
312: * configure URLCheck
313: */
314: private void addCookie() {
315: String cookieStr = JOptionPane.showInputDialog(this ,
316: "Cookie string:");
317: String domain = JOptionPane.showInputDialog(this , "Domain:");
318: URL url;
319: try {
320: url = new URL("http://" + domain);
321: } catch (MalformedURLException e1) {
322: JOptionPane.showMessageDialog(this , "Domain invalid: "
323: + e1.getMessage());
324: return;
325: }
326: try {
327: if (cookieStr.startsWith("Set-Cookie")) {
328: Cookie cookie;
329: cookie = new Cookie(cookieStr, url);
330: jobobase.getRobot().getCookieManager().add(cookie);
331: JOptionPane.showMessageDialog(this , "Cookie added");
332: } else {
333: Cookie[] cookies = Cookie.cookieStringToCookies(
334: cookieStr, domain);
335: for (int i = 0; i < cookies.length; i++) {
336: System.out.println(cookies[i]);
337: jobobase.getRobot().getCookieManager().add(
338: cookies[i]);
339: }
340: JOptionPane.showMessageDialog(this , cookies.length
341: + " cookies added");
342: }
343: } catch (CookieException e) {
344: JOptionPane.showMessageDialog(this ,
345: "Cookie string invalid: " + e.getMessage());
346: }
347:
348: }
349:
350: /**
351: * configure document filters
352: */
353: private void configureFilters() {
354: if (filterConfigFrame == null) {
355: filterConfigFrame = new FilterConfigFrame();
356: }
357: filterConfigFrame.setVisible(true);
358: }
359:
360: /**
361: * configure allowed URls
362: */
363: private void configureAllowedURLs() {
364: if (allowedURLsFrame == null) {
365: allowedURLsFrame = new AllowedListFrame(jobobase.getRobot()
366: .getAllowedURLs());
367: }
368: allowedURLsFrame.setVisible(true);
369: }
370:
371: /**
372: * store current settings
373: */
374: private void saveSettings() {
375: jobobase.saveConfig("test.xml");
376: }
377:
378: /**
379: * Exit the Application
380: */
381: private void exitApp() {
382: System.exit(0);
383: }
384:
385: /**
386: * @param args the command line arguments
387: */
388: public static void main(String[] args) {
389:
390: // create a new frame for logging
391: logFrame = new LogFrame();
392: logFrame.addMsg("Starting ...");
393:
394: // configure Log4J subsystem
395: lfAppend = new LogFrameAppender(logFrame);
396: BasicConfigurator.configure(lfAppend);
397:
398: // other command line arguments
399: for (int i = 0; i < args.length; i++) {
400: if (args[i].equals("-debug")) {
401: // send all log output to standard out (including level debug)
402: BasicConfigurator.configure();
403: }
404: }
405:
406: new JoBoSwing().setVisible(true);
407: }
408:
409: /**
410: * update the WebRobot setting using the values of the input fields
411: * @return true if everything is ok, false otherwise
412: */
413: private boolean updateRobotFromDialog() {
414:
415: // start URL
416: String startUrl = urlField.getText();
417: try {
418: URL u = new URL(startUrl);
419: jobobase.getRobot().setStartURL(u);
420: } catch (MalformedURLException e) {
421: JOptionPane.showMessageDialog(this , "URL " + startUrl
422: + " is invalid");
423: return false;
424: }
425:
426: // storage directory
427: String storageDir = directoryField.getText();
428: jobobase.setStorageDirectory(storageDir);
429:
430: return true;
431: }
432:
433: /**
434: * update the dialog field from the current setting in the webrobot
435: * will only be called on startup !
436: */
437: private void updateDialogFromRobot() {
438:
439: // start URL
440: URL u = jobobase.getRobot().getStartURL();
441: if (u != null) {
442: urlField.setText(u.toString());
443: } else {
444: urlField.setText("");
445: }
446:
447: }
448:
449: /***********************************************************************
450: ***********************************************************************
451:
452: Callback interface for httpTool
453:
454: ***********************************************************************
455: ***********************************************************************/
456:
457: public void setHttpToolDocUrl(String url) {
458: try {
459: currentUrlField.setText(urlToString(new URL(url)));
460: } catch (MalformedURLException e) {
461: }
462: }
463:
464: public void setHttpToolDocSize(int size) {
465: this .httpToolDocSize = size;
466: progressBar.setMaximum(size);
467: }
468:
469: public void setHttpToolDocCurrentSize(int size) {
470: StringBuffer progressStr = new StringBuffer();
471: progressBar.setValue(size);
472: int kB = size / 1024;
473: long longsize = size;
474:
475: Date current = new Date();
476:
477: long millisecs = (current.getTime() - docDownloadStarted
478: .getTime());
479: long ratio = 0;
480: if (millisecs > 0) {
481: ratio = ((longsize * 1000) / millisecs);
482: } else {
483: ratio = 0;
484: }
485:
486: progressStr.append(kB);
487: progressStr.append(" kB");
488:
489: // document size known ?
490: if (this .httpToolDocSize > 0) {
491: progressStr.append(" of ");
492: progressStr.append(httpToolDocSize / 1024);
493: progressStr.append(" kB");
494: }
495:
496: // ratio
497: DecimalFormat myFormat = new DecimalFormat("####0.000");
498: progressStr.append(", ");
499: progressStr.append(myFormat.format((float) (ratio) / 1000));
500: progressStr.append(" kB/s");
501:
502: // time left
503: if (this .httpToolDocSize > 0) {
504: long secleft;
505: if (ratio > 0)
506: secleft = (this .httpToolDocSize - longsize) / ratio;
507: else
508: secleft = 100000;
509:
510: long min = secleft / 60;
511: int sec = (int) (secleft % 60);
512: String secStr = Integer.toString(sec);
513: if (secStr.length() < 2) {
514: secStr = "0" + secStr;
515: }
516: progressStr.append(", ");
517: progressStr.append(min);
518: progressStr.append(":");
519: progressStr.append(secStr);
520: progressStr.append(" min left");
521: }
522:
523: progressBar.setString(progressStr.toString());
524: }
525:
526: public void setHttpToolStatus(int status) {
527: if (status == HttpTool.STATUS_CONNECTING) {
528: progressBar.setString("Connecting");
529: this .setHttpToolDocSize(0);
530: } else if (status == HttpTool.STATUS_CONNECTED) {
531: progressBar.setString("Connected");
532: docDownloadStarted = new Date();
533: } else if (status == HttpTool.STATUS_RETRIEVING) {
534: progressBar.setString("Retrieving");
535: } else if (status == HttpTool.STATUS_DONE) {
536: progressBar.setString("Finished");
537: } else if (status == HttpTool.STATUS_DENIEDBYRULE) {
538: progressBar.setString("Denied by rule");
539: } else {
540: progressBar.setString("Unknown status (" + status + ")");
541: }
542: }
543:
544: /**
545: * converts a URL to a textual representation where the authentication
546: * info is hidden
547: * @param u
548: * @return a textual representation
549: */
550: protected String urlToString(URL u) {
551: if (u == null) {
552: return null;
553: }
554:
555: String userInfo = u.getUserInfo();
556: if ((userInfo != null) && (!userInfo.equals(""))) {
557: userInfo = "authenticated@";
558: } else {
559: userInfo = "";
560: }
561:
562: return u.getProtocol() + "://" + userInfo + u.getHost()
563: + u.getPath();
564: }
565:
566: /***********************************************************************
567: ***********************************************************************
568:
569: End of callback interface for httpTool
570:
571: ***********************************************************************
572: ***********************************************************************/
573:
574: public void webRobotRetrievedDoc(String url, int size) {
575: robotCount++;
576: robotSize += size;
577: updateControls();
578: }
579:
580: public void webRobotUpdateQueueStatus(int length) {
581: robotQueueSize = length;
582: updateControls();
583: }
584:
585: public void webRobotDone() {
586: progressBar.setString("Download completed");
587: robotCount = 0;
588: robotSize = 0;
589: runStopButton.setText("Run");
590: }
591:
592: public void webRobotSleeping(boolean sleeping) {
593: if (sleeping) {
594: progressBar.setString("sleeping");
595: } else {
596: progressBar.setString("");
597: }
598: }
599:
600: /***********************************************************************
601: ***********************************************************************
602:
603: End of callback interface for webRobot
604:
605: ***********************************************************************
606: ***********************************************************************/
607:
608: /**
609: * update status of dynamic elements
610: */
611: protected void updateControls() {
612: DecimalFormat myFormat = new DecimalFormat("###,###,###.0");
613: String retrievedContent = robotCount + " files with "
614: + myFormat.format((float) robotSize / 1024) + " kB";
615: String queuedContent = robotQueueSize + "";
616: retrievedField.setText(retrievedContent);
617: queuedField.setText(queuedContent);
618: }
619:
620: }
|