001: //icapd.java
002: //-----------------------
003: //(C) by Michael Peter Christen; mc@anomic.de
004: //first published on http://www.anomic.de
005: //Frankfurt, Germany, 2004
006: //
007: //This file is contributed by Martin Thelian
008: //last major change: $LastChangedDate: 2007-10-29 01:43:20 +0000 (Mo, 29 Okt 2007) $ by $LastChangedBy: orbiter $
009: //Revision: $LastChangedRevision: 4181 $
010: //
011: //This program is free software; you can redistribute it and/or modify
012: //it under the terms of the GNU General Public License as published by
013: //the Free Software Foundation; either version 2 of the License, or
014: //(at your option) any later version.
015: //
016: //This program is distributed in the hope that it will be useful,
017: //but WITHOUT ANY WARRANTY; without even the implied warranty of
018: //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
019: //GNU General Public License for more details.
020: //
021: //You should have received a copy of the GNU General Public License
022: //along with this program; if not, write to the Free Software
023: //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
024: //
025: //Using this software in any meaning (reading, learning, copying, compiling,
026: //running) means that you agree that the Author(s) is (are) not responsible
027: //for cost, loss of data or any harm that may be caused directly or indirectly
028: //by usage of this softare or this documentation. The usage of this software
029: //is on your own risk. The installation and usage (starting/running) of this
030: //software may allow other people or application to access your computer and
031: //any attached devices and is highly dependent on the configuration of the
032: //software which must be done by the user of the software; the author(s) is
033: //(are) also not responsible for proper configuration and usage of the
034: //software, even if provoked by documentation provided together with
035: //the software.
036: //
037: //Any changes to this file according to the GPL as documented in the file
038: //gpl.txt aside this file in the shipment you received can be done to the
039: //lines that follows this copyright notice here, but changes must not be
040: //done inside the copyright notive above. A re-distribution must contain
041: //the intact and unchanged copyright notice.
042: //Contributions and changes to the program code must be marked as such.
043:
044: package de.anomic.icap;
045:
046: import java.io.BufferedOutputStream;
047: import java.io.BufferedReader;
048: import java.io.ByteArrayInputStream;
049: import java.io.ByteArrayOutputStream;
050: import java.io.File;
051: import java.io.IOException;
052: import java.io.InputStream;
053: import java.io.InputStreamReader;
054: import java.io.OutputStream;
055: import java.net.InetAddress;
056: import java.util.Date;
057: import java.util.Properties;
058:
059: import de.anomic.http.httpChunkedInputStream;
060: import de.anomic.http.httpHeader;
061: import de.anomic.http.httpc;
062: import de.anomic.plasma.plasmaHTCache;
063: import de.anomic.plasma.plasmaParser;
064: import de.anomic.plasma.plasmaSwitchboard;
065: import de.anomic.plasma.cache.IResourceInfo;
066: import de.anomic.plasma.cache.http.ResourceInfo;
067: import de.anomic.server.serverCore;
068: import de.anomic.server.serverFileUtils;
069: import de.anomic.server.serverHandler;
070: import de.anomic.server.logging.serverLog;
071: import de.anomic.server.serverCore.Session;
072: import de.anomic.yacy.yacyURL;
073:
074: /**
075: * @author theli
076: */
077: public class icapd implements serverHandler {
078:
079: private serverCore.Session session; // holds the session object of the calling class
080:
081: // the connection properties
082: private final Properties prop = new Properties();
083:
084: // the address of the client
085: private InetAddress userAddress;
086: private String clientIP;
087: private int keepAliveRequestCount = 0;
088:
089: // needed for logging
090: private final serverLog log = new serverLog("ICAPD");
091:
092: private static plasmaSwitchboard switchboard = null;
093: private static String virtualHost = null;
094: private static boolean keepAliveSupport = true;
095:
096: public icapd() {
097: if (switchboard == null) {
098: switchboard = plasmaSwitchboard.getSwitchboard();
099: virtualHost = switchboard
100: .getConfig("fileHost", "localhost");
101: }
102:
103: }
104:
105: public Object clone() {
106: return new icapd();
107: }
108:
109: public void initSession(Session session) throws IOException {
110: this .session = session;
111: this .userAddress = session.userAddress; // client InetAddress
112: this .clientIP = this .userAddress.getHostAddress();
113: if (this .userAddress.isAnyLocalAddress())
114: this .clientIP = "localhost";
115: if (this .clientIP.equals("0:0:0:0:0:0:0:1"))
116: this .clientIP = "localhost";
117: if (this .clientIP.equals("127.0.0.1"))
118: this .clientIP = "localhost";
119: }
120:
121: public String greeting() {
122: // TODO Auto-generated method stub
123: return null;
124: }
125:
126: public String error(Throwable e) {
127: // TODO Auto-generated method stub
128: return null;
129: }
130:
131: public void reset() {
132: }
133:
134: public Boolean EMPTY(String arg) throws IOException {
135: // TODO Auto-generated method stub
136: return serverCore.TERMINATE_CONNECTION;
137: }
138:
139: public Boolean UNKNOWN(String requestLine) throws IOException {
140: // TODO Auto-generated method stub
141: return serverCore.TERMINATE_CONNECTION;
142: }
143:
144: public icapHeader getDefaultHeaders() {
145: icapHeader newHeaders = new icapHeader();
146:
147: newHeaders.put(icapHeader.SERVER, "YaCy/"
148: + switchboard.getConfig("vString", ""));
149: newHeaders.put(icapHeader.DATE, httpc.dateString(httpc
150: .nowDate()));
151: newHeaders.put(icapHeader.ISTAG, "\""
152: + switchboard.getConfig("vString", "") + "\"");
153:
154: return newHeaders;
155: }
156:
157: public Boolean OPTIONS(String arg) throws IOException {
158:
159: BufferedOutputStream out = new BufferedOutputStream(
160: this .session.out);
161:
162: // parsing the http request line
163: parseRequestLine(icapHeader.METHOD_OPTIONS, arg);
164:
165: // reading the headers
166: icapHeader icapReqHeader = icapHeader.readHeader(this .prop,
167: this .session);
168:
169: // determines if the connection should be kept alive
170: boolean persistent = handlePersistentConnection(icapReqHeader);
171:
172: // setting the icap response headers
173: icapHeader resHeader = getDefaultHeaders();
174: resHeader.put(icapHeader.ALLOW, "204");
175: resHeader.put(icapHeader.ENCAPSULATED, "null-body=0");
176: resHeader.put(icapHeader.MAX_CONNECTIONS, "1000");
177: resHeader.put(icapHeader.OPTIONS_TTL, "300");
178: resHeader.put(icapHeader.SERVICE_ID, "???");
179: resHeader.put(icapHeader.PREVIEW, "30");
180: resHeader.put(icapHeader.TRANSFER_COMPLETE, "*");
181: //resHeader.put(icapHeader.TRANSFER_PREVIEW, "*");
182: if (!persistent)
183: resHeader.put(icapHeader.CONNECTION, "close");
184:
185: // determining the requested service and call it or send back an error message
186: String reqService = this .prop.getProperty(
187: icapHeader.CONNECTION_PROP_PATH, "");
188: if (reqService.equalsIgnoreCase("/resIndexing")) {
189: resHeader.put(icapHeader.SERVICE,
190: "YaCy ICAP Indexing Service 1.0");
191: resHeader
192: .put(icapHeader.METHODS, icapHeader.METHOD_RESPMOD);
193:
194: String transferIgnoreList = plasmaParser.getMediaExtList();
195: transferIgnoreList = transferIgnoreList.substring(1,
196: transferIgnoreList.length() - 1);
197: resHeader.put(icapHeader.TRANSFER_IGNORE,
198: transferIgnoreList);
199: } else {
200: resHeader.put(icapHeader.SERVICE, "YaCy ICAP Service 1.0");
201: }
202:
203: StringBuffer headerStringBuffer = resHeader.toHeaderString(
204: "ICAP/1.0", 200, null);
205: out.write(headerStringBuffer.toString().getBytes());
206: out.flush();
207:
208: return this .prop.getProperty(
209: icapHeader.CONNECTION_PROP_PERSISTENT).equals(
210: "keep-alive") ? serverCore.RESUME_CONNECTION
211: : serverCore.TERMINATE_CONNECTION;
212: }
213:
214: public Boolean REQMOD(String arg) {
215: return serverCore.TERMINATE_CONNECTION;
216: }
217:
218: public Boolean RESPMOD(String arg) {
219: try {
220: InputStream in = this .session.in;
221: OutputStream out = this .session.out;
222:
223: // parsing the icap request line
224: parseRequestLine(icapHeader.METHOD_RESPMOD, arg);
225:
226: // reading the icap request header
227: icapHeader icapReqHeader = icapHeader.readHeader(this .prop,
228: this .session);
229:
230: // determines if the connection should be kept alive
231: handlePersistentConnection(icapReqHeader);
232:
233: // determining the requested service and call it or send back an error message
234: String reqService = this .prop.getProperty(
235: icapHeader.CONNECTION_PROP_PATH, "");
236: if (reqService.equalsIgnoreCase("/resIndexing")) {
237: indexingService(icapReqHeader, in, out);
238: } else {
239: icapHeader icapResHeader = getDefaultHeaders();
240: icapResHeader.put(icapHeader.ENCAPSULATED,
241: icapReqHeader.get(icapHeader.ENCAPSULATED));
242: icapResHeader.put(icapHeader.SERVICE,
243: "YaCy ICAP Service 1.0");
244: // icapResHeader.put(icapHeader.CONNECTION, "close");
245:
246: StringBuffer headerStringBuffer = icapResHeader
247: .toHeaderString("ICAP/1.0", 404, null);
248: out.write((new String(headerStringBuffer)).getBytes());
249: out.flush();
250: }
251:
252: } catch (Exception e) {
253: e.printStackTrace();
254: } finally {
255:
256: }
257: return this .prop.getProperty(
258: icapHeader.CONNECTION_PROP_PERSISTENT).equals(
259: "keep-alive") ? serverCore.RESUME_CONNECTION
260: : serverCore.TERMINATE_CONNECTION;
261: }
262:
263: /*
264: private void blacklistService(icapHeader reqHeader, InputStream in, OutputStream out) {
265: try {
266:
267: } catch (Exception e) {
268: e.printStackTrace();
269: }
270: }
271: */
272:
273: private void indexingService(icapHeader reqHeader, InputStream in,
274: OutputStream out) {
275: try {
276:
277: /* =========================================================================
278: * Reading the various message parts into buffers
279: * ========================================================================= */
280: ByteArrayInputStream reqHdrStream = null, resHdrStream = null, resBodyStream = null;
281: String[] encapsulated = ((String) reqHeader
282: .get(icapHeader.ENCAPSULATED)).split(",");
283: int prevLength = 0, currLength = 0;
284: for (int i = 0; i < encapsulated.length; i++) {
285: // reading the request header
286: if (encapsulated[i].indexOf("req-hdr") >= 0) {
287: prevLength = currLength;
288: currLength = Integer.parseInt(encapsulated[i + 1]
289: .split("=")[1]);
290:
291: byte[] buffer = new byte[currLength - prevLength];
292: in.read(buffer, 0, buffer.length);
293:
294: reqHdrStream = new ByteArrayInputStream(buffer);
295:
296: // reading the response header
297: } else if (encapsulated[i].indexOf("res-hdr") >= 0) {
298: prevLength = currLength;
299: currLength = Integer.parseInt(encapsulated[i + 1]
300: .split("=")[1]);
301:
302: byte[] buffer = new byte[currLength - prevLength];
303: in.read(buffer, 0, buffer.length);
304:
305: resHdrStream = new ByteArrayInputStream(buffer);
306:
307: // reading the response body
308: } else if (encapsulated[i].indexOf("res-body") >= 0) {
309: httpChunkedInputStream chunkedIn = new httpChunkedInputStream(
310: in);
311: ByteArrayOutputStream bout = new ByteArrayOutputStream();
312: int l = 0, len = 0;
313: byte[] buffer = new byte[2048];
314: while ((l = chunkedIn.read(buffer)) >= 0) {
315: len += l;
316: bout.write(buffer, 0, l);
317: }
318: resBodyStream = new ByteArrayInputStream(bout
319: .toByteArray());
320: }
321: }
322:
323: /* =========================================================================
324: * sending back the icap status
325: * ========================================================================= */
326: icapHeader icapResHeader = getDefaultHeaders();
327: if (reqHeader.allow(204)) {
328: icapResHeader.put(icapHeader.ENCAPSULATED, reqHeader
329: .get(icapHeader.ENCAPSULATED));
330: icapResHeader.put(icapHeader.SERVICE,
331: "YaCy ICAP Service 1.0");
332: // resHeader.put(icapHeader.CONNECTION, "close");
333:
334: StringBuffer headerStringBuffer = icapResHeader
335: .toHeaderString("ICAP/1.0", 204, null);
336: out.write((new String(headerStringBuffer)).getBytes());
337: out.flush();
338: } else {
339: icapResHeader.put(icapHeader.ENCAPSULATED, reqHeader
340: .get(icapHeader.ENCAPSULATED));
341: icapResHeader.put(icapHeader.SERVICE,
342: "YaCy ICAP Service 1.0");
343: // icapResHeader.put(icapHeader.CONNECTION, "close");
344:
345: StringBuffer headerStringBuffer = icapResHeader
346: .toHeaderString("ICAP/1.0", 503, null);
347: out.write((new String(headerStringBuffer)).getBytes());
348: out.flush();
349: }
350:
351: /* =========================================================================
352: * Parsing request data
353: * ========================================================================= */
354: // reading the requestline
355: BufferedReader reader = new BufferedReader(
356: new InputStreamReader(reqHdrStream));
357: String httpRequestLine = reader.readLine();
358:
359: // parsing the requestline
360: Properties httpReqProps = new Properties();
361: httpHeader.parseRequestLine(httpRequestLine, httpReqProps,
362: virtualHost);
363:
364: if (!httpReqProps.getProperty(
365: httpHeader.CONNECTION_PROP_METHOD).equals(
366: httpHeader.METHOD_GET)) {
367: this .log
368: .logInfo("Wrong http request method for indexing:"
369: + "\nRequest Method: "
370: + httpReqProps
371: .getProperty(httpHeader.CONNECTION_PROP_METHOD)
372: + "\nRequest Line: "
373: + httpRequestLine);
374: reader.close();
375: reqHdrStream.close();
376: return;
377: }
378:
379: // reading all request headers
380: httpHeader httpReqHeader = httpHeader
381: .readHttpHeader(reader);
382: reader.close();
383: reqHdrStream.close();
384:
385: // handle transparent proxy support: this function call is needed to set the host property properly
386: httpHeader.handleTransparentProxySupport(httpReqHeader,
387: httpReqProps, virtualHost, true);
388:
389: // getting the request URL
390: yacyURL httpRequestURL = httpHeader
391: .getRequestURL(httpReqProps);
392:
393: /* =========================================================================
394: * Parsing response data
395: * ========================================================================= */
396: // getting the response status
397: reader = new BufferedReader(new InputStreamReader(
398: resHdrStream));
399: String httpRespStatusLine = reader.readLine();
400:
401: Object[] httpRespStatus = httpHeader
402: .parseResponseLine(httpRespStatusLine);
403:
404: if (!(httpRespStatus[1].equals(new Integer(200)) || httpRespStatus[1]
405: .equals(new Integer(203)))) {
406: this .log.logInfo("Wrong status code for indexing:"
407: + "\nStatus Code: " + httpRespStatus[1]
408: + "\nRequest Line: " + httpRequestLine
409: + "\nResponse Line: " + httpRespStatusLine);
410: reader.close();
411: resHdrStream.close();
412: return;
413: }
414:
415: // reading all response headers
416: httpHeader httpResHeader = httpHeader
417: .readHttpHeader(reader);
418: reader.close();
419: resHdrStream.close();
420:
421: if (!plasmaParser.supportedContent(
422: plasmaParser.PARSER_MODE_ICAP, httpRequestURL,
423: httpResHeader.mime())) {
424: this .log
425: .logInfo("Wrong mimeType or fileExtension for indexing:"
426: + "\nMimeType: "
427: + httpResHeader.mime()
428: + "\nRequest Line:" + httpRequestLine);
429: return;
430: }
431:
432: /* =========================================================================
433: * Prepare data for indexing
434: * ========================================================================= */
435:
436: // generating a htcache entry object
437: IResourceInfo resInfo = new ResourceInfo(httpRequestURL,
438: httpReqHeader, httpResHeader);
439: plasmaHTCache.Entry cacheEntry = plasmaHTCache.newEntry(
440: new Date(), 0, httpRequestURL, "",
441: httpRespStatusLine, resInfo, null,
442: switchboard.defaultProxyProfile);
443:
444: // getting the filename/path to store the response body
445: File cacheFile = plasmaHTCache.getCachePath(httpRequestURL);
446:
447: // if the file already exits we delete it
448: if (cacheFile.isFile()) {
449: plasmaHTCache.deleteURLfromCache(httpRequestURL);
450: }
451: // we write the new cache entry to file system directly
452: cacheFile.getParentFile().mkdirs();
453:
454: // copy the response body into the file
455: serverFileUtils.copy(resBodyStream, cacheFile);
456: resBodyStream.close();
457: resBodyStream = null;
458:
459: // indexing the response
460: plasmaHTCache.push(cacheEntry);
461: } catch (Exception e) {
462: e.printStackTrace();
463: }
464: }
465:
466: private final void parseRequestLine(String cmd, String s) {
467: // parsing the requestlin
468: icapHeader.parseRequestLine(cmd, s, this .prop, virtualHost);
469:
470: // adding the client ip prop
471: this .prop.setProperty(icapHeader.CONNECTION_PROP_CLIENTIP,
472: this .clientIP);
473:
474: // counting the amount of received requests within this permanent conneciton
475: this .prop.setProperty(
476: icapHeader.CONNECTION_PROP_KEEP_ALIVE_COUNT, Integer
477: .toString(++this .keepAliveRequestCount));
478: }
479:
480: private boolean handlePersistentConnection(icapHeader header) {
481:
482: if (!keepAliveSupport) {
483: this .prop.put(icapHeader.CONNECTION_PROP_PERSISTENT,
484: "close");
485: return false;
486: }
487:
488: boolean persistent = true;
489: if (((String) header.get(icapHeader.CONNECTION, "keep-alive"))
490: .toLowerCase().equals("close")) {
491: persistent = false;
492: }
493:
494: this .prop.put(icapHeader.CONNECTION_PROP_PERSISTENT,
495: persistent ? "keep-alive" : "close");
496: return persistent;
497: }
498:
499: }
|