001: // loaderThreads.java
002: // ---------------------------
003: // (C) by Michael Peter Christen; mc@anomic.de
004: // first published on http://www.anomic.de
005: // Frankfurt, Germany, 2004
006: // last major change: 28.09.2004
007: //
008: // This program is free software; you can redistribute it and/or modify
009: // it under the terms of the GNU General Public License as published by
010: // the Free Software Foundation; either version 2 of the License, or
011: // (at your option) any later version.
012: //
013: // This program is distributed in the hope that it will be useful,
014: // but WITHOUT ANY WARRANTY; without even the implied warranty of
015: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016: // GNU General Public License for more details.
017: //
018: // You should have received a copy of the GNU General Public License
019: // along with this program; if not, write to the Free Software
020: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
021: //
022: // Using this software in any meaning (reading, learning, copying, compiling,
023: // running) means that you agree that the Author(s) is (are) not responsible
024: // for cost, loss of data or any harm that may be caused directly or indirectly
025: // by usage of this softare or this documentation. The usage of this software
026: // is on your own risk. The installation and usage (starting/running) of this
027: // software may allow other people or application to access your computer and
028: // any attached devices and is highly dependent on the configuration of the
029: // software which must be done by the user of the software; the author(s) is
030: // (are) also not responsible for proper configuration and usage of the
031: // software, even if provoked by documentation provided together with
032: // the software.
033: //
034: // Any changes to this file according to the GPL as documented in the file
035: // gpl.txt aside this file in the shipment you received can be done to the
036: // lines that follows this copyright notice here, but changes must not be
037: // done inside the copyright notive above. A re-distribution must contain
038: // the intact and unchanged copyright notice.
039: // Contributions and changes to the program code must be marked as such.
040:
041: package de.anomic.tools;
042:
043: import java.net.MalformedURLException;
044: import java.util.ArrayList;
045: import java.util.Hashtable;
046:
047: import de.anomic.http.httpRemoteProxyConfig;
048: import de.anomic.http.httpc;
049: import de.anomic.yacy.yacyURL;
050:
051: public class loaderThreads {
052:
053: // global values for loader threads
054: protected int timeout;
055: protected String user;
056: protected String password;
057: protected httpRemoteProxyConfig remoteProxyConfig;
058:
059: // management objects for collection of threads
060: Hashtable<String, Thread> threads;
061: int completed, failed;
062:
063: public loaderThreads() {
064: this (null);
065: }
066:
067: public loaderThreads(httpRemoteProxyConfig theremoteProxyConfig) {
068: this (10000, null, null, theremoteProxyConfig);
069: }
070:
071: public loaderThreads(int timeout, String user, String password,
072: httpRemoteProxyConfig theremoteProxyConfig) {
073: this .timeout = timeout;
074: this .user = user;
075: this .password = password;
076: this .remoteProxyConfig = theremoteProxyConfig;
077: this .threads = new Hashtable<String, Thread>();
078: this .completed = 0;
079: this .failed = 0;
080: }
081:
082: public void newPropLoaderThread(String name, yacyURL url) {
083: newThread(name, url, new propLoader());
084: }
085:
086: public void newThread(String name, yacyURL url,
087: loaderProcess process) {
088: Thread t = new loaderThread(url, process);
089: threads.put(name, t);
090: t.start();
091: }
092:
093: public void terminateThread(String name) {
094: loaderThread t = (loaderThread) threads.get(name);
095: if (t == null)
096: throw new RuntimeException("no such thread: " + name);
097: else
098: t.terminate();
099: }
100:
101: public int threadCompleted(String name) {
102: loaderThread t = (loaderThread) threads.get(name);
103: if (t == null)
104: throw new RuntimeException("no such thread: " + name);
105: else
106: return t.completed();
107: }
108:
109: public int threadStatus(String name) {
110: loaderThread t = (loaderThread) threads.get(name);
111: if (t == null)
112: throw new RuntimeException("no such thread: " + name);
113: else
114: return t.status();
115: }
116:
117: public int completed() {
118: return completed;
119: }
120:
121: public int failed() {
122: return failed;
123: }
124:
125: public int count() {
126: return threads.size();
127: }
128:
129: public Exception threadError(String name) {
130: loaderThread t = (loaderThread) threads.get(name);
131: if (t == null)
132: throw new RuntimeException("no such thread: " + name);
133: else
134: return t.error();
135: }
136:
137: protected class loaderThread extends Thread {
138: private yacyURL url;
139: private Exception error;
140: private loaderProcess process;
141: private byte[] page;
142: private boolean loaded;
143:
144: public loaderThread(yacyURL url, loaderProcess process) {
145: this .url = url;
146: this .process = process;
147: this .error = null;
148: this .page = null;
149: this .loaded = false;
150: }
151:
152: public void run() {
153: try {
154: page = httpc.wget(url, url.getHost(), timeout, user,
155: password, remoteProxyConfig, null, null);
156: loaded = true;
157: process.feed(page);
158: if (process.status() == loaderCore.STATUS_FAILED) {
159: error = process.error();
160: }
161: if ((process.status() == loaderCore.STATUS_COMPLETED)
162: || (process.status() == loaderCore.STATUS_FINALIZED))
163: completed++;
164: if ((process.status() == loaderCore.STATUS_ABORTED)
165: || (process.status() == loaderCore.STATUS_FAILED))
166: failed++;
167: } catch (Exception e) {
168: error = e;
169: failed++;
170: }
171: }
172:
173: public void terminate() {
174: process.terminate();
175: }
176:
177: public boolean loaded() {
178: return loaded;
179: }
180:
181: public int completed() {
182: if (process.status() == loaderCore.STATUS_READY)
183: return 1;
184: if (process.status() == loaderCore.STATUS_RUNNING)
185: return 9 + ((process.completed() * 9) / 10);
186: if (process.status() == loaderCore.STATUS_COMPLETED)
187: return 100;
188: return 0;
189: }
190:
191: public int status() {
192: return process.status(); // see constants in loaderCore
193: }
194:
195: public Exception error() {
196: return error;
197: }
198:
199: }
200:
201: public class propLoader extends loaderCore implements loaderProcess {
202:
203: public propLoader() {
204: this .status = STATUS_READY;
205: }
206:
207: public synchronized void feed(byte[] v) {
208: this .status = STATUS_RUNNING;
209: this .completion = 1;
210: int line = 0;
211: String s, key, value;
212: int p;
213: ArrayList<String> lines = nxTools.strings(v);
214: try {
215: while ((this .run) && (line < lines.size())) {
216: // parse line and construct a property
217: s = (String) lines.get(line);
218: if ((s != null) && ((p = s.indexOf('=')) > 0)) {
219: key = s.substring(0, p).trim();
220: value = s.substring(p + 1).trim();
221: if (key.length() > 0)
222: result.put(key, value);
223: }
224: // update thread information
225: line++;
226: this .completion = 100 * line / lines.size();
227: }
228: if (line == lines.size()) {
229: this .status = STATUS_COMPLETED;
230: return;
231: } else {
232: this .status = STATUS_ABORTED;
233: return;
234: }
235: } catch (Exception e) {
236: this .status = STATUS_FAILED;
237: this .error = e;
238: return;
239: }
240: }
241: }
242:
243: public static void main(String[] args) {
244: httpRemoteProxyConfig proxyConfig = httpRemoteProxyConfig.init(
245: "192.168.1.122", 3128);
246: loaderThreads loader = new loaderThreads(proxyConfig);
247: try {
248: loader.newPropLoaderThread("load1", new yacyURL(
249: "http://www.anomic.de/superseed.txt", null));
250: } catch (MalformedURLException e) {
251:
252: }
253: }
254:
255: }
|