0001: // httpc.java
0002: // -------------------------------------
0003: // (C) by Michael Peter Christen; mc@anomic.de
0004: // first published on http://www.anomic.de
0005: // Frankfurt, Germany, 2004
0006: // last major change: 26.02.2004
0007: //
0008: // This program is free software; you can redistribute it and/or modify
0009: // it under the terms of the GNU General Public License as published by
0010: // the Free Software Foundation; either version 2 of the License, or
0011: // (at your option) any later version.
0012: //
0013: // This program is distributed in the hope that it will be useful,
0014: // but WITHOUT ANY WARRANTY; without even the implied warranty of
0015: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
0016: // GNU General Public License for more details.
0017: //
0018: // You should have received a copy of the GNU General Public License
0019: // along with this program; if not, write to the Free Software
0020: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0021: //
0022: // Using this software in any meaning (reading, learning, copying, compiling,
0023: // running) means that you agree that the Author(s) is (are) not responsible
0024: // for cost, loss of data or any harm that may be caused directly or indirectly
0025: // by usage of this softare or this documentation. The usage of this software
0026: // is on your own risk. The installation and usage (starting/running) of this
0027: // software may allow other people or application to access your computer and
0028: // any attached devices and is highly dependent on the configuration of the
0029: // software which must be done by the user of the software; the author(s) is
0030: // (are) also not responsible for proper configuration and usage of the
0031: // software, even if provoked by documentation provided together with
0032: // the software.
0033: //
0034: // Any changes to this file according to the GPL as documented in the file
0035: // gpl.txt aside this file in the shipment you received can be done to the
0036: // lines that follows this copyright notice here, but changes must not be
0037: // done inside the copyright notive above. A re-distribution must contain
0038: // the intact and unchanged copyright notice.
0039: // Contributions and changes to the program code must be marked as such.
0040:
0041: package de.anomic.http;
0042:
0043: import java.io.File;
0044: import java.io.FileNotFoundException;
0045: import java.io.FileOutputStream;
0046: import java.io.IOException;
0047: import java.io.InputStream;
0048: import java.io.InputStreamReader;
0049: import java.io.OutputStream;
0050: import java.io.OutputStreamWriter;
0051: import java.io.PushbackInputStream;
0052: import java.io.Writer;
0053: import java.net.InetAddress;
0054: import java.net.InetSocketAddress;
0055: import java.net.MalformedURLException;
0056: import java.net.Socket;
0057: import java.net.SocketException;
0058: import java.net.UnknownHostException;
0059: import java.text.SimpleDateFormat;
0060: import java.util.ArrayList;
0061: import java.util.Arrays;
0062: import java.util.Comparator;
0063: import java.util.Date;
0064: import java.util.HashMap;
0065: import java.util.HashSet;
0066: import java.util.Iterator;
0067: import java.util.Locale;
0068: import java.util.Map;
0069: import java.util.TimeZone;
0070: import java.util.zip.GZIPInputStream;
0071: import java.util.zip.GZIPOutputStream;
0072:
0073: import javax.net.ssl.HostnameVerifier;
0074: import javax.net.ssl.HttpsURLConnection;
0075: import javax.net.ssl.SSLContext;
0076: import javax.net.ssl.SSLSocketFactory;
0077: import javax.net.ssl.TrustManager;
0078: import javax.net.ssl.X509TrustManager;
0079:
0080: import de.anomic.kelondro.kelondroBase64Order;
0081: import de.anomic.server.serverByteBuffer;
0082: import de.anomic.server.serverCore;
0083: import de.anomic.server.serverDomains;
0084: import de.anomic.server.serverFileUtils;
0085: import de.anomic.server.serverObjects;
0086: import de.anomic.server.logging.serverLog;
0087: import de.anomic.tools.nxTools;
0088: import de.anomic.yacy.yacyURL;
0089:
0090: /**
0091: * This class implements an http client. While http access is built-in in java
0092: * libraries, it is still necessary to implement the network interface since
0093: * otherwise there is no access to the HTTP/1.0 / HTTP/1.1 header information
0094: * that comes along each connection.
0095: */
0096:
0097: public final class httpc {
0098:
0099: // some constants
0100: /**
0101: * Specifies that the httpc is allowed to use gzip content encoding for
0102: * http post requests
0103: * @see #POST(String, httpHeader, serverObjects, HashMap)
0104: */
0105: public static final String GZIP_POST_BODY = "GZIP_POST_BODY";
0106:
0107: // final statics
0108: private static final String vDATE = "20040602";
0109: private static final int terminalMaxLength = 30000;
0110: private static final SimpleDateFormat HTTPGMTFormatter = new SimpleDateFormat(
0111: "EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
0112: private static final HashMap<String, String> reverseMappingCache = new HashMap<String, String>();
0113: private static final HashSet<httpc> activeConnections = new HashSet<httpc>(); // all connections are stored here and deleted when they are finished
0114: private static final long minimumTime_before_activeConnections_cleanup = 3600000; // 1 Hour
0115: private static final long minimumTime_before_idleConnections_cleanup = 120000; // 2 Minutes
0116: private static final int activeConnections_maximum = 64;
0117: public static final connectionTimeComparator connectionTimeComparatorInstance = new connectionTimeComparator();
0118:
0119: private static int objCounter = 0; // will be increased with each object and is used to return a hash code
0120:
0121: // defined during set-up of switchboard
0122: public static boolean yacyDebugMode = false;
0123:
0124: // statics to be defined in static section below
0125: private static SSLSocketFactory theSSLSockFactory = null;
0126: public static String systemOST;
0127: public static String userAgent;
0128:
0129: static {
0130: // set the time zone
0131: HTTPGMTFormatter.setTimeZone(TimeZone.getTimeZone("GMT")); // The GMT standard date format used in the HTTP protocol
0132:
0133: // set time-out of InetAddress.getByName cache ttl
0134: java.security.Security.setProperty("networkaddress.cache.ttl",
0135: "60");
0136: java.security.Security.setProperty(
0137: "networkaddress.cache.negative.ttl", "0");
0138:
0139: // initializing a dummy trustManager to enable https connections
0140:
0141: // Create a trust manager that does not validate certificate chains
0142: TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
0143: public java.security.cert.X509Certificate[] getAcceptedIssuers() {
0144: return null;
0145: }
0146:
0147: public void checkClientTrusted(
0148: java.security.cert.X509Certificate[] certs,
0149: String authType) {
0150: }
0151:
0152: public void checkServerTrusted(
0153: java.security.cert.X509Certificate[] certs,
0154: String authType) {
0155: }
0156: } };
0157:
0158: // Install the all-trusting trust manager
0159: try {
0160: SSLContext sc = SSLContext.getInstance("SSL");
0161: // Create empty HostnameVerifier
0162: HostnameVerifier hv = new HostnameVerifier() {
0163: public boolean verify(String urlHostName,
0164: javax.net.ssl.SSLSession session) {
0165: // logger.info("Warning: URL Host: "+urlHostName+"
0166: // vs."+session.getPeerHost());
0167: return true;
0168: }
0169: };
0170:
0171: sc.init(null, trustAllCerts,
0172: new java.security.SecureRandom());
0173: HttpsURLConnection
0174: .setDefaultSSLSocketFactory(theSSLSockFactory = sc
0175: .getSocketFactory());
0176: HttpsURLConnection.setDefaultHostnameVerifier(hv);
0177: } catch (Exception e) {
0178: }
0179:
0180: // provide system information for client identification
0181: String loc = System.getProperty("user.timezone", "nowhere");
0182: int p = loc.indexOf("/");
0183: if (p > 0)
0184: loc = loc.substring(0, p);
0185: loc = loc + "/" + System.getProperty("user.language", "dumb");
0186: systemOST = System.getProperty("os.arch", "no-os-arch") + " "
0187: + System.getProperty("os.name", "no-os-name") + " "
0188: + System.getProperty("os.version", "no-os-version")
0189: + "; " + "java "
0190: + System.getProperty("java.version", "no-java-version")
0191: + "; " + loc;
0192: userAgent = "yacy (www.yacy.net; v" + vDATE + "; " + systemOST
0193: + ")";
0194: }
0195:
0196: // class variables
0197: private Socket socket = null; // client socket for commands
0198: public String adressed_host = null;
0199: public int adressed_port = 80;
0200: private String target_virtual_host = null;
0201:
0202: // output and input streams for client control connection
0203: private PushbackInputStream clientInput = null;
0204: private OutputStream clientOutput = null;
0205:
0206: private httpdByteCountInputStream clientInputByteCount = null;
0207: private httpdByteCountOutputStream clientOutputByteCount = null;
0208:
0209: private boolean remoteProxyUse = false;
0210: private httpRemoteProxyConfig remoteProxyConfig = null;
0211:
0212: private String requestPath = null;
0213: private boolean allowContentEncoding = true;
0214:
0215: public boolean ssl;
0216: public long initTime, lastIO;
0217: public String command;
0218: public int timeout;
0219:
0220: private int hashIndex;
0221:
0222: /**
0223: * Initialize the httpc-instance with the given data.
0224: *
0225: * @param remoteProxyHost
0226: * @param remoteProxyPort
0227: * @throws IOException
0228: */
0229: public httpc(String server, String vhost, int port, int timeout,
0230: boolean ssl, httpRemoteProxyConfig theRemoteProxyConfig,
0231: String incomingByteCountAccounting,
0232: String outgoingByteCountAccounting) throws IOException {
0233:
0234: // remove old connections
0235: // do NOT remove this check; in case that everything works fine this call does nothing
0236: // but if in any arror case connections stay open, this will ensure that the peer keeps running and the host server is not blocked from working
0237: checkIdleConnections();
0238: assert timeout != 0;
0239:
0240: // register new connection
0241: this .hashIndex = objCounter;
0242: objCounter++;
0243: synchronized (activeConnections) {
0244: activeConnections.add(this );
0245: }
0246: //System.out.println("*** DEBUG init httpc: " + activeConnections.size() + " connections online");
0247:
0248: this .ssl = ssl;
0249: this .initTime = Long.MAX_VALUE;
0250: this .lastIO = Long.MAX_VALUE;
0251: this .command = null;
0252: this .timeout = timeout;
0253:
0254: if ((theRemoteProxyConfig == null)
0255: || (!theRemoteProxyConfig.useProxy())) {
0256: initN(server, vhost, port, timeout, ssl,
0257: incomingByteCountAccounting,
0258: outgoingByteCountAccounting);
0259: return;
0260: }
0261:
0262: if (port == -1) {
0263: port = (ssl) ? 443 : 80;
0264: }
0265:
0266: String remoteProxyHost = theRemoteProxyConfig.getProxyHost();
0267: int remoteProxyPort = theRemoteProxyConfig.getProxyPort();
0268:
0269: this .initN(remoteProxyHost, vhost, remoteProxyPort, timeout,
0270: ssl, incomingByteCountAccounting,
0271: outgoingByteCountAccounting);
0272:
0273: this .remoteProxyUse = true;
0274: this .adressed_host = server;
0275: this .adressed_port = port;
0276: this .target_virtual_host = vhost;
0277: this .remoteProxyConfig = theRemoteProxyConfig;
0278: }
0279:
0280: /**
0281: * Convert the status of this class into an String object to output it.
0282: */
0283: public String toString() {
0284: return (this .adressed_host == null) ? "Disconnected"
0285: : "Connected to "
0286: + this .adressed_host
0287: + ((this .remoteProxyUse) ? " via "
0288: + adressed_host : "");
0289: }
0290:
0291: /**
0292: * Sets wether the content is allowed to be unzipped while getting?
0293: * FIXME: The name of this method seems misleading, if I read the usage of
0294: * this method correctly?
0295: *
0296: * @param status true, if the content is allowed to be decoded on the fly?
0297: */
0298: public void setAllowContentEncoding(boolean status) {
0299: this .allowContentEncoding = status;
0300: }
0301:
0302: /**
0303: * Check wether the connection of this instance is closed.
0304: *
0305: * @return true if the connection is no longer open.
0306: */
0307: public boolean isClosed() {
0308: if (this .socket == null)
0309: return true;
0310: return (!this .socket.isConnected()) || (this .socket.isClosed());
0311: }
0312:
0313: /**
0314: * Returns the given date in an HTTP-usable format.
0315: * (according to RFC822)
0316: *
0317: * @param date The Date-Object to be converted.
0318: * @return String with the date.
0319: */
0320: public static String dateString(Date date) {
0321: if (date == null)
0322: return "";
0323:
0324: /*
0325: * This synchronized is needed because SimpleDateFormat
0326: * is not thread-safe.
0327: * See: http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6231579
0328: */
0329: synchronized (HTTPGMTFormatter) {
0330: return HTTPGMTFormatter.format(date);
0331: }
0332: }
0333:
0334: /**
0335: * Returns the current date as Date-Object.
0336: *
0337: * @return Date-object with the current time.
0338: */
0339: public static Date nowDate() {
0340: return new Date();
0341: }
0342:
0343: public int hashCode() {
0344: // return a hash code so it is possible to store objects of httpc objects in a HashSet
0345: return this .hashIndex;
0346: }
0347:
0348: /**
0349: * Initialize the https-instance with the given data. Opens the sockets to
0350: * the remote server and creats input and output streams.
0351: *
0352: * @param server Hostname of the server to connect to.
0353: * @param port On which port should we connect.
0354: * @param timeout How long do we wait for answers?
0355: * @param ssl Wether we should use SSL.
0356: * @throws IOException
0357: */
0358: private void initN(String server, String vhost, int port,
0359: int timeout, boolean ssl,
0360: String incomingByteCountAccounting,
0361: String outgoingByteCountAccounting) throws IOException {
0362: //serverLog.logDebug("HTTPC", handle + " initialized");
0363: this .remoteProxyUse = false;
0364:
0365: try {
0366: if (port == -1) {
0367: port = (ssl) ? 443 : 80;
0368: }
0369:
0370: this .adressed_host = server;
0371: this .adressed_port = port;
0372: this .target_virtual_host = vhost;
0373:
0374: // creating a socket
0375: this .socket = (ssl) ? theSSLSockFactory.createSocket()
0376: : new Socket();
0377:
0378: // creating a socket address
0379: InetSocketAddress address = null;
0380: if (!this .remoteProxyUse) {
0381: // only try to resolve the address if we are not using a proxy
0382: InetAddress hostip = serverDomains.dnsResolve(server);
0383: if (hostip == null)
0384: throw new UnknownHostException(server);
0385: address = new InetSocketAddress(hostip, port);
0386: } else {
0387: address = new InetSocketAddress(server, port);
0388: }
0389:
0390: // trying to establish a connection to the address
0391: this .initTime = System.currentTimeMillis();
0392: this .lastIO = System.currentTimeMillis();
0393: this .socket.setKeepAlive(false);
0394: // setting socket timeout and keep alive behaviour
0395: this .socket.setSoTimeout(timeout); // waiting time for read
0396: // get the connection
0397: this .socket.connect(address, timeout);
0398:
0399: if (incomingByteCountAccounting != null) {
0400: this .clientInputByteCount = new httpdByteCountInputStream(
0401: this .socket.getInputStream(),
0402: incomingByteCountAccounting);
0403: }
0404: if (outgoingByteCountAccounting != null) {
0405: this .clientOutputByteCount = new httpdByteCountOutputStream(
0406: this .socket.getOutputStream(),
0407: outgoingByteCountAccounting);
0408: }
0409:
0410: // getting input and output streams
0411: this .clientInput = new PushbackInputStream(
0412: (this .clientInputByteCount != null) ? this .clientInputByteCount
0413: : this .socket.getInputStream());
0414: this .clientOutput = this .socket.getOutputStream();
0415:
0416: // if we reached this point, we should have a connection
0417: } catch (UnknownHostException e) {
0418: serverLog.logFine("HTTPC", "Couldn't find host " + server);
0419: close();
0420: throw new IOException("unknown host: " + server);
0421: } catch (IOException e) {
0422: // There was an error while connecting the socket, probably a SocketTimeoutException
0423: // we have to close the httpc, otherwise it would stay in activeConnections forever
0424: serverLog.logFine("HTTPC", "Couldn't open socket to: "
0425: + this .adressed_host + ":" + this .adressed_port);
0426: close();
0427:
0428: // TODO do we need to hand it over to the caller?
0429: throw e;
0430: }
0431: }
0432:
0433: public long getInputStreamByteCount() {
0434: return (this .clientInputByteCount == null) ? 0
0435: : this .clientInputByteCount.getCount();
0436: }
0437:
0438: public long getOutputStreamByteCount() {
0439: return (this .clientOutputByteCount == null) ? 0
0440: : this .clientOutputByteCount.getCount();
0441: }
0442:
0443: public static int checkIdleConnections() {
0444: // try to find and close all connections that did not find a target server and are idle waiting for a server socket
0445:
0446: httpc[] a = allConnections(); // put set into array to avoid ConcurrentModificationExceptions
0447: int tbd = 0;
0448: int c = 0;
0449: if (a.length > activeConnections_maximum) {
0450: // delete some connections; choose the oldest
0451: Arrays.sort(a, httpc.connectionTimeComparatorInstance);
0452: tbd = a.length - activeConnections_maximum;
0453: for (int i = 0; i < tbd; i++) {
0454: if (a[i] != null) {
0455: a[i].close();
0456: c++;
0457: }
0458: }
0459: }
0460: for (int i = tbd; i < a.length; i++) {
0461: httpc clientConnection = a[i];
0462: if (clientConnection != null) {
0463: if ((clientConnection.initTime != Long.MAX_VALUE)
0464: && (clientConnection.initTime
0465: + Math
0466: .max(
0467: minimumTime_before_activeConnections_cleanup,
0468: clientConnection.timeout) < System
0469: .currentTimeMillis())) {
0470: // the time-out limit is reached. close the connection
0471: clientConnection.close();
0472: c++;
0473: }
0474: if ((clientConnection.lastIO != Long.MAX_VALUE)
0475: && (clientConnection.lastIO
0476: + Math
0477: .max(
0478: minimumTime_before_idleConnections_cleanup,
0479: clientConnection.timeout) < System
0480: .currentTimeMillis())) {
0481: // the time-out limit is reached. close the connection
0482: clientConnection.close();
0483: c++;
0484: }
0485: }
0486: }
0487: return c;
0488: }
0489:
0490: public static int closeAllConnections() {
0491: httpc[] a = allConnections(); // put set into array to avoid ConcurrentModificationExceptions
0492: int c = 0;
0493: for (int i = 0; i < a.length; i++) {
0494: httpc clientConnection = a[i];
0495: if (clientConnection != null) {
0496: clientConnection.close();
0497: c++;
0498: }
0499: }
0500: return c;
0501: }
0502:
0503: public static httpc[] allConnections() {
0504: httpc[] a = null;
0505: synchronized (activeConnections) {
0506: a = new httpc[activeConnections.size()];
0507: Iterator<httpc> i = httpc.activeConnections.iterator();
0508: int c = 0;
0509: while (i.hasNext()) {
0510: a[c++] = i.next();
0511: }
0512: }
0513: return a;
0514: }
0515:
0516: public static class connectionTimeComparator implements
0517: Comparator<httpc> {
0518:
0519: public connectionTimeComparator() {
0520: super ();
0521: }
0522:
0523: public int compare(httpc o1, httpc o2) {
0524: long l1 = System.currentTimeMillis() - o1.lastIO;
0525: long l2 = System.currentTimeMillis() - o2.lastIO;
0526: if (l1 < l2)
0527: return 1;
0528: if (l1 > l2)
0529: return -1;
0530: return 0;
0531: }
0532: }
0533:
0534: public void finalize() {
0535: this .close();
0536: }
0537:
0538: public void close() {
0539: synchronized (activeConnections) {
0540: activeConnections.remove(this );
0541: }
0542: //System.out.println("*** DEBUG close httpc: " + activeConnections.size() + " connections online");
0543:
0544: if (this .clientInput != null) {
0545: try {
0546: this .clientInput.close();
0547: } catch (Exception e) {
0548: }
0549: this .clientInput = null;
0550: }
0551:
0552: if (this .clientOutput != null) {
0553: try {
0554: this .clientOutput.close();
0555: } catch (Exception e) {
0556: }
0557: this .clientOutput = null;
0558: }
0559:
0560: if (this .socket != null) {
0561: try {
0562: this .socket.close();
0563: } catch (Exception e) {
0564: }
0565: this .socket = null;
0566: }
0567:
0568: if (this .clientInputByteCount != null) {
0569: this .clientInputByteCount.finish();
0570: this .clientInputByteCount = null;
0571: }
0572:
0573: if (this .clientOutputByteCount != null) {
0574: this .clientOutputByteCount.finish();
0575: this .clientOutputByteCount = null;
0576: }
0577:
0578: this .adressed_host = null;
0579: this .target_virtual_host = null;
0580:
0581: this .remoteProxyConfig = null;
0582: this .requestPath = null;
0583: }
0584:
0585: /**
0586: * This method invokes a call to the given server.
0587: *
0588: * @param method
0589: * Which method should be called? GET, POST, HEAD or CONNECT
0590: * @param path
0591: * String with the path on the server to be get.
0592: * @param header
0593: * The prefilled header (if available) from the calling browser.
0594: * @param zipped
0595: * Is encoded content (gzip) allowed or not?
0596: * @throws IOException
0597: */
0598: private void send(String method, String path, httpHeader header,
0599: boolean zipped) throws IOException {
0600: // scheduled request through request-response objects/threads
0601:
0602: // check and correct path
0603: if ((path == null) || (path.length() == 0))
0604: path = "/";
0605:
0606: // for debuggug:
0607: this .requestPath = path;
0608:
0609: // prepare header
0610: if (header == null)
0611: header = new httpHeader();
0612:
0613: // set some standard values
0614: if (!(header.containsKey(httpHeader.ACCEPT)))
0615: header
0616: .put(
0617: httpHeader.ACCEPT,
0618: "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
0619: if (!(header.containsKey(httpHeader.ACCEPT_CHARSET)))
0620: header.put(httpHeader.ACCEPT_CHARSET,
0621: "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
0622: if (!(header.containsKey(httpHeader.ACCEPT_LANGUAGE)))
0623: header.put(httpHeader.ACCEPT_LANGUAGE, "en-us,en;q=0.5");
0624: if (!(header.containsKey(httpHeader.KEEP_ALIVE)))
0625: header.put(httpHeader.KEEP_ALIVE, "300");
0626:
0627: // set user agent. The user agent is only set if the value does not yet exists.
0628: // this gives callers the opportunity, to change the user agent themselves, and
0629: // it will not be changed.
0630: if (!(header.containsKey(httpHeader.USER_AGENT)))
0631: header.put(httpHeader.USER_AGENT, userAgent);
0632:
0633: // set the host attribute. This is in particular necessary, if we contact another proxy
0634: // the host is mandatory, if we use HTTP/1.1
0635: if (!(header.containsKey(httpHeader.HOST))) {
0636: if (this .remoteProxyUse) {
0637: header.put(httpHeader.HOST, this .adressed_host);
0638: } else {
0639: header.put(httpHeader.HOST, this .target_virtual_host);
0640: }
0641: }
0642:
0643: if (this .remoteProxyUse) {
0644: String remoteProxyUser = this .remoteProxyConfig
0645: .getProxyUser();
0646: String remoteProxyPwd = this .remoteProxyConfig
0647: .getProxyPwd();
0648: if ((remoteProxyUser != null)
0649: && (remoteProxyUser.length() > 0)) {
0650: header.put(httpHeader.PROXY_AUTHORIZATION, "Basic "
0651: + kelondroBase64Order.standardCoder
0652: .encodeString(remoteProxyUser + ":"
0653: + remoteProxyPwd));
0654: }
0655: }
0656:
0657: if (!(header.containsKey(httpHeader.CONNECTION))) {
0658: header.put(httpHeader.CONNECTION, "close");
0659: }
0660:
0661: // stimulate zipping or not
0662: // we can unzip, and we will return it always as unzipped, unless not wanted
0663: if (header.containsKey(httpHeader.ACCEPT_ENCODING)) {
0664: String encoding = (String) header
0665: .get(httpHeader.ACCEPT_ENCODING);
0666: if (zipped) {
0667: if (encoding.indexOf("gzip") < 0) {
0668: // add the gzip encoding
0669: //System.out.println("!!! adding gzip encoding");
0670: header.put(httpHeader.ACCEPT_ENCODING,
0671: "gzip,deflate"
0672: + ((encoding.length() == 0) ? ""
0673: : (";" + encoding)));
0674: }
0675: } else {
0676: int pos = encoding.indexOf("gzip");
0677: if (pos >= 0) {
0678: // remove the gzip encoding
0679: //System.out.println("!!! removing gzip encoding");
0680: // ex: "gzip,deflate" => pos == 0, but we need to remove the "," as well => substring(pos+5),
0681: // ex: "gzip" => pos == 0, but substring(pos+5) would exceed boundaries
0682: String enc = encoding.substring(0, pos)
0683: + (encoding.length() > (pos + 5) ? encoding
0684: .substring(pos + 5) : "");
0685: header.put(httpHeader.ACCEPT_ENCODING, enc);
0686: }
0687: }
0688: } else {
0689: if (zipped)
0690: header.put(httpHeader.ACCEPT_ENCODING, "gzip,deflate");
0691: }
0692:
0693: //header = new httpHeader(); header.put("Host", this.host); // debug
0694:
0695: StringBuffer sb = new StringBuffer();
0696: // send request
0697: if ((this .remoteProxyUse)
0698: && (!(method.equals(httpHeader.METHOD_CONNECT))))
0699: path = ((this .adressed_port == 443) ? "https://"
0700: : "http://")
0701: + this .adressed_host
0702: + ":"
0703: + this .adressed_port
0704: + path;
0705: sb.append(method + " " + path + " HTTP/1.0"
0706: + serverCore.CRLF_STRING); // TODO if set to HTTP/1.1, servers give time-outs?
0707:
0708: // send header
0709: //System.out.println("***HEADER for path " + path + ": PROXY TO SERVER = " + header.toString()); // DEBUG
0710: Iterator<String> i = header.keySet().iterator();
0711: String key;
0712: int count;
0713: char tag;
0714: while (i.hasNext()) {
0715: key = i.next();
0716: tag = key.charAt(0);
0717: if ((tag != '*') && (tag != '#')) {
0718: count = header.keyCount(key);
0719: for (int j = 0; j < count; j++) {
0720: sb.append(key
0721: + ": "
0722: + ((String) header.getSingle(key, j))
0723: .trim() + serverCore.CRLF_STRING);
0724: }
0725: //System.out.println("#" + key + ": " + value);
0726: }
0727: }
0728:
0729: // add terminating line
0730: sb.append(serverCore.CRLF_STRING);
0731: serverCore.send(this .clientOutput, sb.toString());
0732: this .clientOutput.flush();
0733:
0734: // this is the place where www.stern.de refuses to answer ..???
0735: }
0736:
0737: /**
0738: * This method GETs a page from the server.
0739: *
0740: * @param path The path to the page which should be GET.
0741: * @param requestHeader Prefilled httpHeader.
0742: * @return Instance of response with the content.
0743: * @throws IOException
0744: */
0745: public response GET(String path, httpHeader requestHeader)
0746: throws IOException {
0747: //serverLog.logDebug("HTTPC", handle + " requested GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
0748: this .command = "GET " + path;
0749: try {
0750: boolean zipped = (!this .allowContentEncoding) ? false
0751: : httpd.shallTransportZipped(path);
0752: send(httpHeader.METHOD_GET, path, requestHeader, zipped);
0753: response r = new response(zipped);
0754: //serverLog.logDebug("HTTPC", handle + " returned GET '" + path + "', time = " + (System.currentTimeMillis() - handle));
0755: return r;
0756: } catch (Exception e) {
0757: if (e.getMessage().indexOf("heap space") > 0) {
0758: e.printStackTrace();
0759: }
0760: throw new IOException(e.getMessage());
0761: }
0762: }
0763:
0764: /**
0765: * This method gets only the header of a page.
0766: *
0767: * @param path The path to the page whose header should be get.
0768: * @param requestHeader Prefilled httpHeader.
0769: * @return Instance of response with the content.
0770: * @throws IOException
0771: */
0772: public response HEAD(String path, httpHeader requestHeader)
0773: throws IOException {
0774: this .command = "HEAD " + path;
0775: try {
0776: send(httpHeader.METHOD_HEAD, path, requestHeader, false);
0777: return new response(false);
0778: // in this case the caller should not read the response body,
0779: // since there is none...
0780: } catch (SocketException e) {
0781: throw new IOException(e.getMessage());
0782: }
0783: }
0784:
0785: /**
0786: * This method POSTs some data to a page.
0787: *
0788: * @param path The path to the page which the post is sent to.
0789: * @param requestHeader Prefilled httpHeader.
0790: * @param ins InputStream with the data to be posted to the server.
0791: * @return Instance of response with the content.
0792: * @throws IOException
0793: */
0794: public response POST(String path, httpHeader requestHeader,
0795: InputStream ins) throws IOException {
0796: this .command = "POST " + path;
0797: try {
0798: send(httpHeader.METHOD_POST, path, requestHeader, false);
0799: // if there is a body to the call, we would have a CONTENT-LENGTH tag in the requestHeader
0800: String cl = (String) requestHeader
0801: .get(httpHeader.CONTENT_LENGTH);
0802: int len, c;
0803: byte[] buffer = new byte[512];
0804: if (cl != null) {
0805: len = Integer.parseInt(cl);
0806: // transfer len bytes from ins to the server
0807: while ((len > 0) && ((c = ins.read(buffer)) >= 0)) {
0808: this .clientOutput.write(buffer, 0, c);
0809: len -= c;
0810: }
0811: } else {
0812: len = 0;
0813: while ((c = ins.read(buffer)) >= 0) {
0814: this .clientOutput.write(buffer, 0, c);
0815: len += c;
0816: }
0817:
0818: // TODO: we can not set the header here. This ist too late
0819: requestHeader.put(httpHeader.CONTENT_LENGTH, Integer
0820: .toString(len));
0821: }
0822: this .clientOutput.flush();
0823: return new response(false);
0824: } catch (SocketException e) {
0825: throw new IOException(e.getMessage());
0826: }
0827: }
0828:
0829: /**
0830: * Call the server with the CONNECT-method.
0831: * This is used to establish https-connections through a https-proxy
0832: *
0833: * @param host To which host should a connection be made?
0834: * @param port Which port should be connected?
0835: * @param requestHeader prefilled httpHeader.
0836: * @return Instance of response with the content.
0837: */
0838:
0839: public response CONNECT(String remotehost, int remoteport,
0840: httpHeader requestHeader) throws IOException {
0841: this .command = "CONNECT " + remotehost + ":" + remoteport;
0842: try {
0843: send(httpHeader.METHOD_CONNECT, remotehost + ":"
0844: + remoteport, requestHeader, false);
0845: return new response(false);
0846: } catch (SocketException e) {
0847: throw new IOException(e.getMessage());
0848: }
0849: }
0850:
0851: /**
0852: * This method sends several files at once via a POST request. Only those
0853: * files in the Hashtable files are written whose names are contained in
0854: * args.
0855: *
0856: * @param path The path to the page which the post is sent to.
0857: * @param requestHeader Prefilled httpHeader.
0858: * @param args serverObjects with the names of the files to send.
0859: * @param files HashMap with the names of the files as key and the content
0860: * of the files as value.
0861: * @return Instance of response with the content.
0862: * @throws IOException
0863: */
0864: public response POST(String path, httpHeader requestHeader,
0865: serverObjects args, HashMap<String, byte[]> files)
0866: throws IOException {
0867: // make shure, the header has a boundary information like
0868: // CONTENT-TYPE=multipart/form-data; boundary=----------0xKhTmLbOuNdArY
0869: if (requestHeader == null)
0870: requestHeader = new httpHeader();
0871: String boundary = (String) requestHeader
0872: .get(httpHeader.CONTENT_TYPE);
0873: if (boundary == null) {
0874: // create a boundary
0875: boundary = "multipart/form-data; boundary=----------"
0876: + java.lang.System.currentTimeMillis();
0877: requestHeader.put(httpHeader.CONTENT_TYPE, boundary);
0878: }
0879: // extract the boundary string
0880: int pos = boundary.toUpperCase().indexOf("BOUNDARY=");
0881: if (pos < 0) {
0882: // again, create a boundary
0883: boundary = "multipart/form-data; boundary=----------"
0884: + java.lang.System.currentTimeMillis();
0885: requestHeader.put(httpHeader.CONTENT_TYPE, boundary);
0886: pos = boundary.indexOf("boundary=");
0887: }
0888: boundary = "--"
0889: + boundary.substring(pos + "boundary=".length());
0890:
0891: boolean zipContent = args.containsKey(GZIP_POST_BODY);
0892: args.remove(GZIP_POST_BODY);
0893:
0894: OutputStream out;
0895: GZIPOutputStream zippedOut;
0896: serverByteBuffer buf = new serverByteBuffer();
0897: if (zipContent) {
0898: zippedOut = new GZIPOutputStream(buf);
0899: out = zippedOut;
0900: } else {
0901: out = buf;
0902: }
0903:
0904: // in contrast to GET and HEAD, this method also transports a message body
0905: // the body consists of repeated boundaries and values in between
0906: if (args.size() != 0) {
0907: // we have values for the POST, start with one boundary
0908: String key, value;
0909: Iterator<String> e = args.keySet().iterator();
0910: while (e.hasNext()) {
0911: // start with a boundary
0912: out.write(boundary.getBytes("UTF-8"));
0913: out.write(serverCore.CRLF);
0914: // write value
0915: key = e.next();
0916: value = args.get(key, "");
0917: if ((files != null) && (files.containsKey(key))) {
0918: // we are about to write a file
0919: out.write(("Content-Disposition: form-data; name="
0920: + '"' + key + '"' + "; filename=" + '"'
0921: + value + '"').getBytes("UTF-8"));
0922: out.write(serverCore.CRLF);
0923: out.write(serverCore.CRLF);
0924: out.write((byte[]) files.get(key));
0925: out.write(serverCore.CRLF);
0926: } else {
0927: // write a single value
0928: out.write(("Content-Disposition: form-data; name="
0929: + '"' + key + '"').getBytes("UTF-8"));
0930: out.write(serverCore.CRLF);
0931: out.write(serverCore.CRLF);
0932: out.write(value.getBytes("UTF-8"));
0933: out.write(serverCore.CRLF);
0934: }
0935: }
0936: // finish with a boundary
0937: out.write(boundary.getBytes("UTF-8"));
0938: out.write(serverCore.CRLF);
0939: }
0940: // create body array
0941: out.close();
0942: byte[] body = buf.getBytes();
0943: buf = null;
0944: out = null;
0945:
0946: //System.out.println("DEBUG: PUT BODY=" + new String(body));
0947: if (zipContent) {
0948: requestHeader.put(httpHeader.CONTENT_ENCODING, "gzip");
0949:
0950: //TODO: should we also set the content length here?
0951: } else {
0952: // size of that body
0953: requestHeader.put(httpHeader.CONTENT_LENGTH, Integer
0954: .toString(body.length));
0955: }
0956:
0957: // send the header
0958: send(httpHeader.METHOD_POST, path, requestHeader, false);
0959:
0960: // send the body
0961: serverCore.send(this .clientOutput, body);
0962:
0963: return new response(false);
0964: }
0965:
0966: public static byte[] singleGET(String realhost, String virtualhost,
0967: int port, String path, int timeout, String user,
0968: String password, boolean ssl,
0969: httpRemoteProxyConfig theRemoteProxyConfig,
0970: httpHeader requestHeader, File download) throws IOException {
0971: // if download == null, the get result is stored to a byte[] and returned,
0972: // otherwise the get is streamed to the file and null is returned
0973: if (requestHeader == null)
0974: requestHeader = new httpHeader();
0975:
0976: // setting host authorization header
0977: if ((user != null) && (password != null)
0978: && (user.length() != 0)) {
0979: requestHeader.put(httpHeader.AUTHORIZATION,
0980: kelondroBase64Order.standardCoder.encodeString(user
0981: + ":" + password));
0982: }
0983:
0984: httpc con = new httpc(realhost, virtualhost, port, timeout,
0985: ssl, theRemoteProxyConfig, null, null);
0986:
0987: httpc.response res = con.GET(path, requestHeader);
0988: if (res.status.startsWith("2")) {
0989: if (download == null) {
0990: // stream to byte[]
0991: serverByteBuffer sbb = new serverByteBuffer();
0992: res.writeContent(sbb, null);
0993: con.close();
0994: return sbb.getBytes();
0995: } else {
0996: // stream to file and return null
0997: res.writeContent(null, download);
0998: con.close();
0999: return null;
1000: }
1001: }
1002: return res.status.getBytes();
1003: }
1004:
1005: public static byte[] singleGET(yacyURL u, String vhost,
1006: int timeout, String user, String password,
1007: httpRemoteProxyConfig theRemoteProxyConfig, File download)
1008: throws IOException {
1009: int port = u.getPort();
1010: boolean ssl = u.getProtocol().equals("https");
1011: if (port < 0)
1012: port = (ssl) ? 443 : 80;
1013: String path = u.getPath();
1014: String query = u.getQuery();
1015: if ((query != null) && (query.length() > 0))
1016: path = path + "?" + query;
1017: return singleGET(u.getHost(), vhost, port, path, timeout, user,
1018: password, ssl, theRemoteProxyConfig, null, download);
1019: }
1020:
1021: public static byte[] singlePOST(String realhost,
1022: String virtualhost, int port, String path, int timeout,
1023: String user, String password, boolean ssl,
1024: httpRemoteProxyConfig theRemoteProxyConfig,
1025: httpHeader requestHeader, serverObjects props,
1026: HashMap<String, byte[]> files) throws IOException {
1027:
1028: if (requestHeader == null)
1029: requestHeader = new httpHeader();
1030: if ((user != null) && (password != null)
1031: && (user.length() != 0)) {
1032: requestHeader.put(httpHeader.AUTHORIZATION,
1033: kelondroBase64Order.standardCoder.encodeString(user
1034: + ":" + password));
1035: }
1036:
1037: httpc con = new httpc(realhost, virtualhost, port, timeout,
1038: ssl, theRemoteProxyConfig, null, null);
1039: httpc.response res = con
1040: .POST(path, requestHeader, props, files);
1041:
1042: //System.out.println("response=" + res.toString());
1043: if (!(res.status.startsWith("2"))) {
1044: byte[] status = res.status.getBytes();
1045: con.close();
1046: return status;
1047: }
1048:
1049: // read connection body and return body
1050: serverByteBuffer sbb = new serverByteBuffer();
1051: res.writeContent(sbb, null);
1052: con.close();
1053: return sbb.getBytes();
1054: }
1055:
1056: public static byte[] singlePOST(yacyURL u, String vhost,
1057: int timeout, String user, String password,
1058: httpRemoteProxyConfig theRemoteProxyConfig,
1059: serverObjects props, HashMap<String, byte[]> files)
1060: throws IOException {
1061: int port = u.getPort();
1062: boolean ssl = u.getProtocol().equals("https");
1063: if (port < 0)
1064: port = (ssl) ? 443 : 80;
1065: String path = u.getPath();
1066: String query = u.getQuery();
1067: if ((query != null) && (query.length() > 0))
1068: path = path + "?" + query;
1069: return singlePOST(u.getHost(), vhost, port, path, timeout,
1070: user, password, ssl, theRemoteProxyConfig, null, props,
1071: files);
1072: }
1073:
1074: public static byte[] wget(yacyURL url, String vhost, int timeout,
1075: String user, String password,
1076: httpRemoteProxyConfig theRemoteProxyConfig,
1077: httpHeader requestHeader, File download) throws IOException {
1078:
1079: int port = url.getPort();
1080: boolean ssl = url.getProtocol().equals("https");
1081: if (port < 0)
1082: port = (ssl) ? 443 : 80;
1083: String path = url.getPath();
1084: String query = url.getQuery();
1085: if ((query != null) && (query.length() > 0))
1086: path = path + "?" + query;
1087:
1088: // splitting of the byte array into lines
1089: byte[] a = singleGET(url.getHost(), vhost, port, path, timeout,
1090: user, password, ssl, theRemoteProxyConfig,
1091: requestHeader, download);
1092:
1093: if (a == null)
1094: return null;
1095:
1096: // support of gzipped data (requested by roland)
1097: a = serverFileUtils.uncompressGZipArray(a);
1098:
1099: // return result
1100: return a;
1101: }
1102:
1103: public static Map<String, String> loadHashMap(yacyURL url,
1104: httpRemoteProxyConfig proxy) {
1105: try {
1106: // should we use the proxy?
1107: boolean useProxy = (proxy != null) && (proxy.useProxy())
1108: && (proxy.useProxy4Yacy());
1109:
1110: // sending request
1111: final HashMap<String, String> result = nxTools.table(httpc
1112: .wget(url, url.getHost(), 8000, null, null,
1113: (useProxy) ? proxy : null, null, null),
1114: "UTF-8");
1115:
1116: if (result == null)
1117: return new HashMap<String, String>();
1118: return result;
1119: } catch (Exception e) {
1120: return new HashMap<String, String>();
1121: }
1122: }
1123:
1124: public static httpHeader whead(yacyURL url, String vhost,
1125: int timeout, String user, String password,
1126: httpRemoteProxyConfig theRemoteProxyConfig)
1127: throws IOException {
1128: return whead(url, vhost, timeout, user, password,
1129: theRemoteProxyConfig, null);
1130: }
1131:
1132: public static httpHeader whead(yacyURL url, String vhost,
1133: int timeout, String user, String password,
1134: httpRemoteProxyConfig theRemoteProxyConfig,
1135: httpHeader requestHeader) throws IOException {
1136: // generate request header
1137: if (requestHeader == null)
1138: requestHeader = new httpHeader();
1139: if ((user != null) && (password != null)
1140: && (user.length() != 0)) {
1141: requestHeader.put(httpHeader.AUTHORIZATION,
1142: kelondroBase64Order.standardCoder.encodeString(user
1143: + ":" + password));
1144: }
1145: // parse query
1146:
1147: int port = url.getPort();
1148: boolean ssl = url.getProtocol().equals("https");
1149: if (port < 0)
1150: port = (ssl) ? 443 : 80;
1151: String path = url.getPath();
1152: String query = url.getQuery();
1153: if ((query != null) && (query.length() > 0))
1154: path = path + "?" + query;
1155: String realhost = url.getHost();
1156:
1157: // start connection
1158: httpc con = new httpc(realhost, vhost, port, timeout, ssl,
1159: theRemoteProxyConfig, null, null);
1160: httpc.response res = con.HEAD(path, requestHeader);
1161: httpHeader h = res.responseHeader;
1162: con.close();
1163: return h;
1164: }
1165:
1166: public static byte[] wput(yacyURL url, String vhost, int timeout,
1167: String user, String password,
1168: httpRemoteProxyConfig theRemoteProxyConfig,
1169: serverObjects props, HashMap<String, byte[]> files)
1170: throws IOException {
1171: // splitting of the byte array into lines
1172: byte[] a = singlePOST(url, vhost, timeout, user, password,
1173: theRemoteProxyConfig, props, files);
1174:
1175: if (a == null)
1176: return null;
1177:
1178: // support of gzipped data
1179: a = serverFileUtils.uncompressGZipArray(a);
1180:
1181: // return result
1182: return a;
1183:
1184: //System.out.println("wput-out=" + new String(a));
1185: //return nxTools.strings(a);
1186: }
1187:
1188: public static void main(String[] args) {
1189: System.out.println("ANOMIC.DE HTTP CLIENT v" + vDATE);
1190: String url = args[0];
1191: if (!(url.toUpperCase().startsWith("HTTP://")))
1192: url = "http://" + url;
1193: ArrayList<String> text = new ArrayList<String>();
1194: if (args.length == 4) {
1195: int timeout = Integer.parseInt(args[1]);
1196: String proxyHost = args[2];
1197: int proxyPort = Integer.parseInt(args[3]);
1198:
1199: httpRemoteProxyConfig theRemoteProxyConfig = httpRemoteProxyConfig
1200: .init(proxyHost, proxyPort);
1201: try {
1202: yacyURL u = new yacyURL(url, null);
1203: text = nxTools.strings(wget(u, u.getHost(), timeout,
1204: null, null, theRemoteProxyConfig, null, null));
1205: } catch (MalformedURLException e) {
1206: System.out.println("The url '" + url + "' is wrong.");
1207: } catch (IOException e) {
1208: System.out.println("Error loading url '" + url + "': "
1209: + e.getMessage());
1210: }
1211: } /*else {
1212: serverObjects post = new serverObjects();
1213: int p;
1214: for (int i = 1; i < args.length; i++) {
1215: p = args[i].indexOf("=");
1216: if (p > 0) post.put(args[i].substring(0, p), args[i].substring(p + 1));
1217: }
1218: text = wput(url, post);
1219: }*/
1220: Iterator<String> i = text.listIterator();
1221: while (i.hasNext())
1222: System.out.println(i.next());
1223: }
1224:
1225: /**
1226: * Inner Class to get the response of an http-request and parse it.
1227: */
1228: public final class response {
1229: // Response-Header = Date | Pragma | Allow | Content-Encoding | Content-Length | Content-Type |
1230: // Expires | Last-Modified | HTTP-header
1231: /*
1232: Status-Line = HTTP-Version SP Status-Code SP Reason-Phrase CRLF
1233: 1xx: Informational - Not used, but reserved for future use
1234: 2xx: Success - The action was successfully received, understood, and accepted.
1235: 3xx: Redirection - Further action must be taken in order to complete the request
1236: 4xx: Client Error - The request contains bad syntax or cannot be fulfilled
1237: 5xx: Server Error - The server failed to fulfill an apparently valid request
1238: */
1239:
1240: // header information
1241: public httpHeader responseHeader = null;
1242: public String httpVer = "HTTP/0.9";
1243: public String status; // the success/failure response string starting with status-code
1244: public int statusCode = 503;
1245: public String statusText = "internal error";
1246: private boolean gzip; // for gunzipping on-the-fly
1247: private long gzippedLength = -1; // reported content length if content-encoding is set
1248:
1249: /**
1250: * Constructor for this class. Reads in the content for the given outer
1251: * instance and parses it.
1252: *
1253: * @param zipped true, if the content of this response is gzipped.
1254: * @throws IOException
1255: */
1256: public response(boolean zipped) throws IOException {
1257:
1258: // lets start with worst-case attributes as set-up
1259: this .responseHeader = new httpHeader(reverseMappingCache);
1260: this .statusCode = 503;
1261: this .statusText = "internal httpc error";
1262: this .status = Integer.toString(this .statusCode) + " "
1263: + this .statusText;
1264: this .gzip = false;
1265:
1266: // check connection status
1267: if (httpc.this .clientInput == null) {
1268: // the server has meanwhile disconnected
1269: this .statusCode = 503;
1270: this .statusText = "lost connection to server";
1271: this .status = Integer.toString(this .statusCode) + " "
1272: + this .statusText;
1273: return; // in bad mood
1274: }
1275:
1276: // reads in the http header, right now, right here
1277: byte[] b = serverCore.receive(httpc.this .clientInput,
1278: terminalMaxLength, false);
1279: if (b == null) {
1280: // the server has meanwhile disconnected
1281: this .statusCode = 503;
1282: this .statusText = "server has closed connection";
1283: this .status = Integer.toString(this .statusCode) + " "
1284: + this .statusText;
1285: return; // in bad mood
1286: }
1287:
1288: // parsing the response status line
1289: String buffer = new String(b);
1290: Object[] responseInfo = httpHeader
1291: .parseResponseLine(buffer);
1292: this .httpVer = (String) responseInfo[0];
1293: this .statusCode = ((Integer) responseInfo[1]).intValue();
1294: this .statusText = (String) responseInfo[2];
1295: this .status = this .statusCode + " " + this .statusText;
1296:
1297: if ((this .statusCode == 500)
1298: && (this .statusText
1299: .equals("status line parse error"))) {
1300: // flush in anything that comes without parsing
1301: while ((b != null) && (b.length != 0))
1302: b = serverCore.receive(httpc.this .clientInput,
1303: terminalMaxLength, false);
1304: return; // in bad mood
1305: }
1306:
1307: // check validity
1308: if (this .statusCode == 400) {
1309: // bad request
1310: // flush in anything that comes without parsing
1311: while ((b = serverCore.receive(httpc.this .clientInput,
1312: terminalMaxLength, false)).length != 0) {
1313: }
1314: return; // in bad mood
1315: }
1316:
1317: // at this point we should have a valid response. read in the header properties
1318: String key = "";
1319: while ((b = serverCore.receive(httpc.this .clientInput,
1320: terminalMaxLength, false)) != null) {
1321: if (b.length == 0)
1322: break;
1323: buffer = new String(b);
1324: buffer = buffer.trim();
1325: //System.out.println("#H#" + buffer); // debug
1326: // RFC2616 4.2: headers beginning with LWS are appended to the previous line
1327: if (b[0] == serverCore.SP || b[0] == serverCore.HT) {
1328: // use old entry
1329: if (key.length() == 0)
1330: throw new IOException(
1331: "header corrupted - input error");
1332: // attach new line
1333: if (!(this .responseHeader.containsKey(key)))
1334: throw new IOException(
1335: "header corrupted - internal error");
1336: this .responseHeader.put(key,
1337: (String) this .responseHeader.get(key) + " "
1338: + buffer.trim());
1339: } else {
1340: // create new entry
1341: int p = buffer.indexOf(":");
1342: if (p > 0) {
1343: this .responseHeader
1344: .add(buffer.substring(0, p).trim(),
1345: buffer.substring(p + 1).trim());
1346: } else {
1347: serverLog.logSevere("HTTPC",
1348: "RESPONSE PARSE ERROR: HOST='"
1349: + httpc.this .adressed_host
1350: + "', PATH='"
1351: + httpc.this .requestPath
1352: + "', STATUS='" + this .status
1353: + "'");
1354: serverLog.logSevere("HTTPC",
1355: "..............BUFFER: " + buffer);
1356: throw new IOException(this .status);
1357: }
1358: }
1359: }
1360: // finished with reading header
1361:
1362: // we will now manipulate the header if the content is gzip encoded, because
1363: // reading the content with "writeContent" will gunzip on-the-fly
1364: this .gzip = ((zipped) && (this .responseHeader.gzip()));
1365:
1366: if (this .gzip) {
1367: if (this .responseHeader
1368: .containsKey(httpHeader.CONTENT_LENGTH)) {
1369: this .gzippedLength = this .responseHeader
1370: .contentLength();
1371: }
1372: this .responseHeader.remove(httpHeader.CONTENT_ENCODING); // we fake that we don't have encoding, since what comes out does not have gzip and we also don't know what was encoded
1373: this .responseHeader.remove(httpHeader.CONTENT_LENGTH); // we cannot use the length during gunzippig yet; still we can hope that it works
1374: }
1375: }
1376:
1377: public long getGzippedLength() {
1378: return this .gzippedLength;
1379: }
1380:
1381: public boolean isGzipped() {
1382: return this .gzip;
1383: }
1384:
1385: /**
1386: * Converts an instance of this class into a readable string.
1387: *
1388: * @return String with some information about this instance.
1389: */
1390: public String toString() {
1391: StringBuffer toStringBuffer = new StringBuffer();
1392: toStringBuffer.append(
1393: (this .status == null) ? "Status: Unknown"
1394: : "Status: " + this .status).append(
1395: " | Headers: ").append(
1396: (this .responseHeader == null) ? "none"
1397: : this .responseHeader.toString());
1398: return new String(toStringBuffer);
1399: }
1400:
1401: /**
1402: * Returns wether this request was successful or not. Stati beginning
1403: * with 2 or 3 are considered successful.
1404: *
1405: * @return True, if the request was successful.
1406: */
1407: public boolean success() {
1408: return ((this .status.charAt(0) == '2') || (this .status
1409: .charAt(0) == '3'));
1410: }
1411:
1412: /**
1413: * If the response was encoded using <code>Content-Encoding: gzip</code>
1414: * a {@link GZIPInputStream} is returned. If the <code>Content-Length</code> header was set,
1415: * a {@link httpContentLengthInputStream} is returned which returns <code>-1</code> if the end of the
1416: * response body was reached.
1417: *
1418: * @return a {@link InputStream} to read the response body
1419: * @throws IOException
1420: */
1421: public InputStream getContentInputStream() throws IOException {
1422: if (this .gzip) {
1423: // use a gzip input stream for Content-Encoding: gzip
1424: return new GZIPInputStream(httpc.this .clientInput);
1425: } else if (this .responseHeader.contentLength() != -1) {
1426: // use a httpContentLengthInputStream to read until the end of the response body is reached
1427: return new httpContentLengthInputStream(
1428: httpc.this .clientInput, this .responseHeader
1429: .contentLength());
1430: }
1431: // no Content-Lenght was set. In this case we can read until EOF
1432: return httpc.this .clientInput;
1433: }
1434:
1435: /**
1436: * This method outputs the found content into an byte-array and
1437: * additionally outputs it to procOS.
1438: *
1439: * @param procOS
1440: * @return the found content
1441: * @throws IOException
1442: */
1443: public byte[] writeContent(Object procOS,
1444: boolean returnByteArray) throws IOException {
1445: serverByteBuffer sbb = null;
1446:
1447: if (returnByteArray) {
1448: int contentLength = (int) this .responseHeader
1449: .contentLength();
1450: sbb = new serverByteBuffer((contentLength == -1) ? 8192
1451: : contentLength);
1452: }
1453:
1454: if (procOS instanceof OutputStream) {
1455: writeX(this .getContentInputStream(),
1456: (OutputStream) procOS, sbb);
1457: } else if (procOS instanceof Writer) {
1458: String charSet = this .responseHeader
1459: .getCharacterEncoding();
1460: if (charSet == null)
1461: charSet = httpHeader.DEFAULT_CHARSET;
1462: writeX(this .getContentInputStream(), charSet,
1463: (Writer) procOS, sbb, charSet);
1464: } else {
1465: throw new IllegalArgumentException(
1466: "Invalid procOS object type '"
1467: + procOS.getClass().getName() + "'");
1468: }
1469:
1470: return (sbb == null) ? null : sbb.getBytes();
1471: }
1472:
1473: /**
1474: * This method writes the input stream to either another output stream
1475: * or a file or both.
1476: * In case that an exception occurrs, the stream reading is just teminated
1477: * and content received so far is returned
1478: *
1479: * @param procOS
1480: * @param file
1481: */
1482: public void writeContent(Object procOS, File file) {
1483: // this writes the input stream to either another output stream or
1484: // a file or both.
1485: FileOutputStream bufferOS = null;
1486: if (file != null)
1487: try {
1488: bufferOS = new FileOutputStream(file);
1489: } catch (FileNotFoundException e) {
1490: file = null;
1491: }
1492: try {
1493: InputStream is = this .getContentInputStream();
1494: if (procOS == null) {
1495: writeX(is, null, bufferOS);
1496: } else if (procOS instanceof OutputStream) {
1497: writeX(is, (OutputStream) procOS, bufferOS);
1498: //writeContentX(httpc.this.clientInput, this.gzip, this.responseHeader.contentLength(), procOS, bufferOS);
1499: } else if (procOS instanceof Writer) {
1500: String charSet = this .responseHeader
1501: .getCharacterEncoding();
1502: if (charSet == null)
1503: charSet = httpHeader.DEFAULT_CHARSET;
1504: writeX(is, charSet, (Writer) procOS, bufferOS,
1505: charSet);
1506: } else {
1507: throw new IllegalArgumentException(
1508: "Invalid procOS object type '"
1509: + procOS.getClass().getName() + "'");
1510: }
1511: } catch (IOException e) {
1512: }
1513:
1514: if (bufferOS != null) {
1515: try {
1516: bufferOS.flush();
1517: bufferOS.close();
1518: } catch (IOException e) {
1519: }
1520: if (file.length() == 0)
1521: file.delete();
1522: }
1523: }
1524:
1525: public void writeX(InputStream source, OutputStream procOS,
1526: OutputStream bufferOS) {
1527: byte[] buffer = new byte[2048];
1528: int l, c = 0;
1529: lastIO = System.currentTimeMillis();
1530:
1531: io: while (true)
1532: try {
1533: l = source.read(buffer, 0, buffer.length);
1534: if (l < 0)
1535: break;
1536: if (l == 0)
1537: try {
1538: if (System.currentTimeMillis() - lastIO > 30000)
1539: break;
1540: this .wait(300);
1541: continue io;
1542: } catch (InterruptedException e) {
1543: } // may happen without EOF
1544: lastIO = System.currentTimeMillis();
1545: c += l;
1546: if (procOS != null)
1547: procOS.write(buffer, 0, l);
1548: if (bufferOS != null)
1549: bufferOS.write(buffer, 0, l);
1550: } catch (IOException e) {
1551: //System.out.println("*** DEBUG: writeX/IOStream terminated with IOException, processed " + c + " bytes.");
1552: break;
1553: }
1554:
1555: // flush the streams
1556: if (procOS != null)
1557: try {
1558: procOS.flush();
1559: } catch (IOException e) {
1560: }
1561: if (bufferOS != null)
1562: try {
1563: bufferOS.flush();
1564: } catch (IOException e) {
1565: }
1566: buffer = null;
1567: }
1568:
1569: public void writeX(InputStream source, String inputCharset,
1570: Writer procOS, OutputStream bufferOS,
1571: String outputCharset) {
1572: try {
1573: InputStreamReader sourceReader = new InputStreamReader(
1574: source, inputCharset);
1575: OutputStreamWriter bufferOSWriter = (bufferOS == null) ? null
1576: : new OutputStreamWriter(bufferOS,
1577: outputCharset);
1578: char[] buffer = new char[2048];
1579: int l, c = 0;
1580: lastIO = System.currentTimeMillis();
1581:
1582: io: while (true)
1583: try {
1584: l = sourceReader.read(buffer, 0, buffer.length);
1585: if (l < 0)
1586: break;
1587: if (l == 0)
1588: try {
1589: if (System.currentTimeMillis() - lastIO > 30000)
1590: break;
1591: this .wait(300);
1592: continue io;
1593: } catch (InterruptedException e) {
1594: } // may happen without EOF
1595: lastIO = System.currentTimeMillis();
1596: c += l;
1597: if (procOS != null)
1598: procOS.write(buffer, 0, l);
1599: if (bufferOSWriter != null)
1600: bufferOSWriter.write(buffer, 0, l);
1601: } catch (IOException e) {
1602: //System.out.println("*** DEBUG: writeX/ReaderWriter terminated with IOException, processed " + c + " bytes.");
1603: break;
1604: }
1605:
1606: // flush the streams
1607: if (procOS != null)
1608: try {
1609: procOS.flush();
1610: } catch (IOException e) {
1611: }
1612: if (bufferOSWriter != null)
1613: try {
1614: bufferOSWriter.flush();
1615: } catch (IOException e) {
1616: }
1617: buffer = null;
1618: } catch (IOException e) {
1619: }
1620: }
1621:
1622: /**
1623: * This method outputs a logline to the serverlog with the current
1624: * status of this instance.
1625: */
1626: public void print() {
1627: serverLog.logInfo("HTTPC", "RESPONSE: status="
1628: + this .status + ", header="
1629: + this .responseHeader.toString());
1630: }
1631:
1632: }
1633:
1634: }
1635:
1636: /*
1637: import java.net.*;
1638: import java.io.*;
1639: import javax.net.ssl.*;
1640: import javax.security.cert.X509Certificate;
1641: import java.security.KeyStore;
1642:
1643:
1644: //The application can be modified to connect to a server outside
1645: //the firewall by following SSLSocketClientWithTunneling.java.
1646:
1647: public class SSLSocketClientWithClientAuth {
1648:
1649: public static void main(String[] args) throws Exception {
1650: String host = null;
1651: int port = -1;
1652: String path = null;
1653: for (int i = 0; i < args.length; i++)
1654: System.out.println(args[i]);
1655:
1656: if (args.length < 3) {
1657: System.out.println(
1658: "USAGE: java SSLSocketClientWithClientAuth " +
1659: "host port requestedfilepath");
1660: System.exit(-1);
1661: }
1662:
1663: try {
1664: host = args[0];
1665: port = Integer.parseInt(args[1]);
1666: path = args[2];
1667: } catch (IllegalArgumentException e) {
1668: System.out.println("USAGE: java SSLSocketClientWithClientAuth " +
1669: "host port requestedfilepath");
1670: System.exit(-1);
1671: }
1672:
1673: try {
1674:
1675: SSLSocketFactory factory = null;
1676: try {
1677: SSLContext ctx;
1678: KeyManagerFactory kmf;
1679: KeyStore ks;
1680: char[] passphrase = "passphrase".toCharArray();
1681:
1682: ctx = SSLContext.getInstance("TLS");
1683: kmf = KeyManagerFactory.getInstance("SunX509");
1684: ks = KeyStore.getInstance("JKS");
1685:
1686: ks.load(new FileInputStream("testkeys"), passphrase);
1687:
1688: kmf.init(ks, passphrase);
1689: ctx.init(kmf.getKeyManagers(), null, null);
1690:
1691: factory = ctx.getSocketFactory();
1692: } catch (Exception e) {
1693: throw new IOException(e.getMessage());
1694: }
1695:
1696: SSLSocket socket = (SSLSocket)factory.createSocket(host, port);
1697:
1698: socket.startHandshake();
1699:
1700: PrintWriter out = new PrintWriter(
1701: new BufferedWriter(
1702: new OutputStreamWriter(
1703: socket.getOutputStream())));
1704: out.println("GET " + path + " HTTP/1.1");
1705: out.println();
1706: out.flush();
1707:
1708: if (out.checkError())
1709: System.out.println(
1710: "SSLSocketClient: java.io.PrintWriter error");
1711:
1712: BufferedReader in = new BufferedReader(
1713: new InputStreamReader(
1714: socket.getInputStream()));
1715:
1716: String inputLine;
1717:
1718: while ((inputLine = in.readLine()) != null)
1719: System.out.println(inputLine);
1720:
1721: in.close();
1722: out.close();
1723: socket.close();
1724:
1725: } catch (Exception e) {
1726: e.printStackTrace();
1727: }
1728: }
1729: }
1730: */
|