0001: // yacyCore.java
0002: // -------------------------------------
0003: // (C) by Michael Peter Christen; mc@anomic.de
0004: // first published on http://www.anomic.de
0005: // Frankfurt, Germany, 2004
0006: //
0007: // $LastChangedDate: 2008-01-24 22:49:00 +0000 (Do, 24 Jan 2008) $
0008: // $LastChangedRevision: 4397 $
0009: // $LastChangedBy: orbiter $
0010: //
0011: // This program is free software; you can redistribute it and/or modify
0012: // it under the terms of the GNU General Public License as published by
0013: // the Free Software Foundation; either version 2 of the License, or
0014: // (at your option) any later version.
0015: //
0016: // This program is distributed in the hope that it will be useful,
0017: // but WITHOUT ANY WARRANTY; without even the implied warranty of
0018: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
0019: // GNU General Public License for more details.
0020: //
0021: // You should have received a copy of the GNU General Public License
0022: // along with this program; if not, write to the Free Software
0023: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0024: //
0025: // Using this software in any meaning (reading, learning, copying, compiling,
0026: // running) means that you agree that the Author(s) is (are) not responsible
0027: // for cost, loss of data or any harm that may be caused directly or indirectly
0028: // by usage of this softare or this documentation. The usage of this software
0029: // is on your own risk. The installation and usage (starting/running) of this
0030: // software may allow other people or application to access your computer and
0031: // any attached devices and is highly dependent on the configuration of the
0032: // software which must be done by the user of the software; the author(s) is
0033: // (are) also not responsible for proper configuration and usage of the
0034: // software, even if provoked by documentation provided together with
0035: // the software.
0036: //
0037: // Any changes to this file according to the GPL as documented in the file
0038: // gpl.txt aside this file in the shipment you received can be done to the
0039: // lines that follows this copyright notice here, but changes must not be
0040: // done inside the copyright notive above. A re-distribution must contain
0041: // the intact and unchanged copyright notice.
0042: // Contributions and changes to the program code must be marked as such.
0043:
0044: /*
0045: the yacy process of getting in touch of other peers starts as follows:
0046: - init seed cache. It is needed to determine the right peer for the Hello-Process
0047: - create a own seed. This can be a new one or one loaded from a file
0048: - The httpd must start up then first
0049: - the own seed is completed by performing the 'yacyHello' process. This
0050: process will result in a request back to the own peer to check if it runs
0051: in server mode. This is the reason that the httpd must be started in advance.
0052:
0053: */
0054:
0055: // contributions:
0056: // principal peer status via file generation by Alexander Schier [AS]
0057: package de.anomic.yacy;
0058:
0059: import java.io.File;
0060: import java.io.FilenameFilter;
0061: import java.io.IOException;
0062: import java.net.MalformedURLException;
0063: import java.net.URI;
0064: import java.util.Collections;
0065: import java.util.Date;
0066: import java.util.HashMap;
0067: import java.util.Iterator;
0068: import java.util.LinkedList;
0069: import java.util.List;
0070: import java.util.Map;
0071:
0072: import de.anomic.plasma.plasmaSwitchboard;
0073: import de.anomic.server.serverCore;
0074: import de.anomic.server.serverDate;
0075: import de.anomic.server.serverSemaphore;
0076: import de.anomic.server.serverSwitch;
0077: import de.anomic.server.logging.serverLog;
0078:
0079: public class yacyCore {
0080:
0081: // statics
0082: public static final ThreadGroup publishThreadGroup = new ThreadGroup(
0083: "publishThreadGroup");
0084: public static yacySeedDB seedDB = null;
0085: public static yacyNewsPool newsPool = null;
0086: public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
0087: public static yacyPeerActions peerActions = null;
0088: public static yacyDHTAction dhtAgent = null;
0089: public static serverLog log;
0090: public static long lastOnlineTime = 0;
0091: /** pseudo-random key derived from a time-interval while YaCy startup*/
0092: public static long speedKey = 0;
0093: public static File yacyDBPath;
0094: public static final Map<String, yacyAccessible> amIAccessibleDB = Collections
0095: .synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
0096: // constants for PeerPing behaviour
0097: private static final int PING_INITIAL = 10;
0098: private static final int PING_MAX_RUNNING = 3;
0099: private static final int PING_MIN_RUNNING = 1;
0100: private static final int PING_MIN_DBSIZE = 5;
0101: private static final int PING_MIN_PEERSEEN = 1; // min. accessible to force senior
0102: private static final long PING_MAX_DBAGE = 15 * 60 * 1000; // in milliseconds
0103:
0104: // public static yacyShare shareManager = null;
0105: // public static boolean terminate = false;
0106:
0107: // class variables
0108: private int lastSeedUpload_seedDBSize = 0;
0109: public long lastSeedUpload_timeStamp = System.currentTimeMillis();
0110: // private String lastSeedUpload_myPeerType = "";
0111: private String lastSeedUpload_myIP = "";
0112:
0113: private static int onlineMode = 1;
0114: private plasmaSwitchboard switchboard;
0115:
0116: public static int yacyTime() {
0117: // the time since startup of yacy in seconds
0118: return (int) ((System.currentTimeMillis() - serverCore.startupTime) / 1000);
0119: }
0120:
0121: public yacyCore(plasmaSwitchboard sb) {
0122: long time = System.currentTimeMillis();
0123:
0124: this .switchboard = sb;
0125: switchboard.setConfig("yacyStatus", "");
0126:
0127: // set log level
0128: log = new serverLog("YACY");
0129:
0130: // create a yacy db
0131: yacyDBPath = sb.getConfigPath("yacyDB", "DATA/YACYDB");
0132: if (!yacyDBPath.exists()) {
0133: yacyDBPath.mkdir();
0134: }
0135:
0136: // create or init seed cache
0137: long memDHT_time = Long.parseLong(switchboard.getConfig(
0138: "ramCacheDHT_time", "1000"));
0139: seedDB = new yacySeedDB(sb,
0140: new File(yacyDBPath, "seed2.new.db"), new File(
0141: yacyDBPath, "seed2.old.db"), new File(
0142: yacyDBPath, "seed2.pot.db"), memDHT_time);
0143:
0144: // create or init news database
0145: newsPool = new yacyNewsPool(yacyDBPath);
0146:
0147: loadSeedUploadMethods();
0148:
0149: // deploy peer actions
0150: peerActions = new yacyPeerActions(seedDB, switchboard);
0151: dhtAgent = new yacyDHTAction(seedDB);
0152: peerActions.deploy(dhtAgent);
0153: peerActions.deploy(new yacyNewsAction(newsPool));
0154:
0155: lastSeedUpload_seedDBSize = seedDB.sizeConnected();
0156:
0157: log.logConfig("CORE INITIALIZED");
0158: // ATTENTION, VERY IMPORTANT: before starting the thread, the httpd yacy server must be running!
0159:
0160: speedKey = System.currentTimeMillis() - time;
0161:
0162: // start with a seedList update to propagate out peer, if possible
0163: onlineMode = Integer.parseInt(switchboard.getConfig(
0164: "onlineMode", "1"));
0165: //lastSeedUpdate = universalTime();
0166: lastOnlineTime = 0;
0167:
0168: // cycle
0169: // within cycle: update seed file, strengthen network, pass news (new, old seed's)
0170: if (online()) {
0171: log.logConfig("you are in online mode");
0172: } else {
0173: log.logConfig("YOU ARE OFFLINE! ---");
0174: log
0175: .logConfig("--- TO START BOOTSTRAPING, YOU MUST USE THE PROXY,");
0176: log.logConfig("--- OR HIT THE BUTTON 'go online'");
0177: log.logConfig("--- ON THE STATUS PAGE http://localhost:"
0178: + serverCore.getPortNr(switchboard.getConfig(
0179: "port", "8080")) + "/Status.html");
0180: }
0181: }
0182:
0183: public synchronized void close() {
0184: seedDB.close();
0185: newsPool.close();
0186: }
0187:
0188: synchronized static public void triggerOnlineAction() {
0189: lastOnlineTime = System.currentTimeMillis();
0190: }
0191:
0192: public final boolean online() {
0193: onlineMode = Integer.parseInt(switchboard.getConfig(
0194: "onlineMode", "1"));
0195: return ((onlineMode == 2) || ((System.currentTimeMillis() - lastOnlineTime) < 10000));
0196: }
0197:
0198: public static int getOnlineMode() {
0199: return onlineMode;
0200: }
0201:
0202: public static void setOnlineMode(int newOnlineMode) {
0203: onlineMode = newOnlineMode;
0204: return;
0205: }
0206:
0207: public final void publishSeedList() {
0208: log.logFine("yacyCore.publishSeedList: Triggered Seed Publish");
0209:
0210: /*
0211: if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1")))
0212: yacyCore.log.logDebug("***DEBUG publishSeedList: oldIP is equal");
0213: if (seedCacheSizeStamp == seedDB.sizeConnected())
0214: yacyCore.log.logDebug("***DEBUG publishSeedList: sizeConnected is equal");
0215: if (canReachMyself())
0216: yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
0217: */
0218:
0219: if ((this .lastSeedUpload_myIP.equals(seedDB.mySeed().get(
0220: yacySeed.IP, "127.0.0.1")))
0221: && (this .lastSeedUpload_seedDBSize == seedDB
0222: .sizeConnected())
0223: && (canReachMyself())
0224: && (System.currentTimeMillis()
0225: - this .lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24)
0226: && (seedDB.mySeed().isPrincipal())) {
0227: log
0228: .logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
0229: return;
0230: }
0231:
0232: // getting the seed upload method that should be used ...
0233: final String seedUploadMethod = this .switchboard.getConfig(
0234: "seedUploadMethod", "");
0235:
0236: if ((!seedUploadMethod.equalsIgnoreCase("none"))
0237: || ((seedUploadMethod.equals("")) && (this .switchboard
0238: .getConfig("seedFTPPassword", "").length() > 0))
0239: || ((seedUploadMethod.equals("")) && (this .switchboard
0240: .getConfig("seedFilePath", "").length() > 0))) {
0241: if (seedUploadMethod.equals("")) {
0242: if (this .switchboard.getConfig("seedFTPPassword", "")
0243: .length() > 0) {
0244: this .switchboard.setConfig("seedUploadMethod",
0245: "Ftp");
0246: }
0247: if (this .switchboard.getConfig("seedFilePath", "")
0248: .length() > 0) {
0249: this .switchboard.setConfig("seedUploadMethod",
0250: "File");
0251: }
0252: }
0253: // we want to be a principal...
0254: saveSeedList();
0255: } else {
0256: if (seedUploadMethod.equals("")) {
0257: this .switchboard.setConfig("seedUploadMethod", "none");
0258: }
0259: log
0260: .logFine("yacyCore.publishSeedList: No uploading method configured");
0261: return;
0262: }
0263: }
0264:
0265: public final void peerPing() {
0266: if (!online()) {
0267: return;
0268: }
0269: if ((switchboard.isRobinsonMode())
0270: && (switchboard.getConfig("cluster.mode", "")
0271: .equals("privatepeer"))) {
0272: // in case this peer is a privat peer we omit the peer ping
0273: // all other robinson peer types do a peer ping:
0274: // the privatecluster does the ping to the other cluster members
0275: // the publiccluster does the ping to all peers, but prefers the own peer
0276: // the publicpeer does the ping to all peers
0277: return;
0278: }
0279:
0280: // before publishing, update some seed data
0281: peerActions.updateMySeed();
0282:
0283: // publish own seed to other peer, this can every peer, but makes only sense for senior peers
0284: if (seedDB.sizeConnected() == 0) {
0285: // reload the seed lists
0286: peerActions.loadSeedLists();
0287: log.logInfo("re-initialized seed list. received "
0288: + seedDB.sizeConnected() + " new peer(s)");
0289: }
0290: final int newSeeds = publishMySeed(false);
0291: if (newSeeds > 0) {
0292: log.logInfo("received " + newSeeds
0293: + " new peer(s), know a total of "
0294: + seedDB.sizeConnected() + " different peers");
0295: }
0296: }
0297:
0298: private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work
0299: // returns true if we can reach ourself under our known peer address
0300: // if we cannot reach ourself, we call a forced publishMySeed and return false
0301: final int urlc = yacyClient.queryUrlCount(seedDB.mySeed());
0302: if (urlc >= 0) {
0303: seedDB.mySeed().setLastSeenUTC();
0304: return true;
0305: }
0306: log.logInfo("re-connect own seed");
0307: final String oldAddress = seedDB.mySeed().getPublicAddress();
0308: /*final int newSeeds =*/publishMySeed(true);
0309: return (oldAddress != null && oldAddress.equals(seedDB.mySeed()
0310: .getPublicAddress()));
0311: }
0312:
0313: protected class publishThread extends Thread {
0314: private int added;
0315: private yacySeed seed;
0316: private final serverSemaphore sync;
0317: private final List<Thread> syncList;
0318:
0319: public publishThread(ThreadGroup tg, yacySeed seed,
0320: serverSemaphore sync, List<Thread> syncList)
0321: throws InterruptedException {
0322: super (tg, "PublishSeed_" + seed.getName());
0323:
0324: this .sync = sync;
0325: this .sync.P();
0326: this .syncList = syncList;
0327:
0328: this .seed = seed;
0329: this .added = 0;
0330: }
0331:
0332: public final void run() {
0333: try {
0334: this .added = yacyClient.publishMySeed(seed
0335: .getClusterAddress(), seed.hash);
0336: if (this .added < 0) {
0337: // no or wrong response, delete that address
0338: String cause = "peer ping to peer resulted in error response (added < 0)";
0339: log.logInfo("publish: disconnected "
0340: + this .seed.get(yacySeed.PEERTYPE,
0341: yacySeed.PEERTYPE_SENIOR)
0342: + " peer '" + this .seed.getName()
0343: + "' from " + this .seed.getPublicAddress()
0344: + ": " + cause);
0345: peerActions.peerDeparture(this .seed, cause);
0346: } else {
0347: // success! we have published our peer to a senior peer
0348: // update latest news from the other peer
0349: log.logInfo("publish: handshaked "
0350: + this .seed.get(yacySeed.PEERTYPE,
0351: yacySeed.PEERTYPE_SENIOR)
0352: + " peer '" + this .seed.getName() + "' at "
0353: + this .seed.getPublicAddress());
0354: // check if seed's lastSeen has been updated
0355: yacySeed newSeed = seedDB
0356: .getConnected(this .seed.hash);
0357: if (newSeed != null) {
0358: if (newSeed.getLastSeenUTC() < (System
0359: .currentTimeMillis() - 10000)) {
0360: // update last seed date
0361: if (newSeed.getLastSeenUTC() >= this .seed
0362: .getLastSeenUTC()) {
0363: log
0364: .logFine("publish: recently handshaked "
0365: + this .seed
0366: .get(
0367: yacySeed.PEERTYPE,
0368: yacySeed.PEERTYPE_SENIOR)
0369: + " peer '"
0370: + this .seed.getName()
0371: + "' at "
0372: + this .seed
0373: .getPublicAddress()
0374: + " with old LastSeen: '"
0375: + serverDate
0376: .formatShortSecond(new Date(
0377: newSeed
0378: .getLastSeenUTC()))
0379: + "'");
0380: newSeed.setLastSeenUTC();
0381: peerActions.peerArrival(newSeed, true);
0382: } else {
0383: log
0384: .logFine("publish: recently handshaked "
0385: + this .seed
0386: .get(
0387: yacySeed.PEERTYPE,
0388: yacySeed.PEERTYPE_SENIOR)
0389: + " peer '"
0390: + this .seed.getName()
0391: + "' at "
0392: + this .seed
0393: .getPublicAddress()
0394: + " with old LastSeen: '"
0395: + serverDate
0396: .formatShortSecond(new Date(
0397: newSeed
0398: .getLastSeenUTC()))
0399: + "', this is more recent: '"
0400: + serverDate
0401: .formatShortSecond(new Date(
0402: this .seed
0403: .getLastSeenUTC()))
0404: + "'");
0405: this .seed.setLastSeenUTC();
0406: peerActions
0407: .peerArrival(this .seed, true);
0408: }
0409: }
0410: } else {
0411: log.logFine("publish: recently handshaked "
0412: + this .seed.get(yacySeed.PEERTYPE,
0413: yacySeed.PEERTYPE_SENIOR)
0414: + " peer '" + this .seed.getName()
0415: + "' at "
0416: + this .seed.getPublicAddress()
0417: + " not in connectedDB");
0418: }
0419: }
0420: } catch (Exception e) {
0421: log.logSevere("publishThread: error with target seed "
0422: + seed.toString() + ": " + e.getMessage(), e);
0423: } finally {
0424: this .syncList.add(this );
0425: this .sync.V();
0426: }
0427: }
0428: }
0429:
0430: private int publishMySeed(boolean force) {
0431: try {
0432: // call this after the httpd was started up
0433:
0434: // we need to find out our own ip
0435: // This is not always easy, since the application may
0436: // live behind a firewall or nat.
0437: // the normal way to do this is either measure the value that java gives us,
0438: // but this is not correct if the peer lives behind a NAT/Router or has several
0439: // addresses and not the right one can be found out.
0440: // We have several alternatives:
0441: // 1. ask another peer. This should be normal and the default method.
0442: // but if no other peer lives, or we don't know them, we cannot do that
0443: // 2. ask own NAT. This is only an option if the NAT is a DI604, because this is the
0444: // only supported for address retrieval
0445: // 3. ask ip respond services in the internet. There are several, and they are all
0446: // probed until we get a valid response.
0447:
0448: // init yacyHello-process
0449: Map<String, yacySeed> seeds; // hash/yacySeed relation
0450:
0451: int attempts = seedDB.sizeConnected();
0452:
0453: // getting a list of peers to contact
0454: if (seedDB.mySeed().get(yacySeed.PEERTYPE,
0455: yacySeed.PEERTYPE_VIRGIN).equals(
0456: yacySeed.PEERTYPE_VIRGIN)) {
0457: if (attempts > PING_INITIAL) {
0458: attempts = PING_INITIAL;
0459: }
0460: Map<String, String> ch = plasmaSwitchboard
0461: .getSwitchboard().clusterhashes;
0462: seeds = seedDB.seedsByAge(true, attempts
0463: - ((ch == null) ? 0 : ch.size())); // best for fast connection
0464: // add also all peers from cluster if this is a public robinson cluster
0465: if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
0466: Iterator<Map.Entry<String, String>> i = ch
0467: .entrySet().iterator();
0468: String hash;
0469: Map.Entry<String, String> entry;
0470: yacySeed seed;
0471: while (i.hasNext()) {
0472: entry = i.next();
0473: hash = entry.getKey();
0474: seed = (yacySeed) seeds.get(hash);
0475: if (seed == null) {
0476: seed = seedDB.get(hash);
0477: if (seed == null)
0478: continue;
0479: }
0480: seed.setAlternativeAddress((String) entry
0481: .getValue());
0482: seeds.put(hash, seed);
0483: }
0484: }
0485: } else {
0486: int diff = PING_MIN_DBSIZE - amIAccessibleDB.size();
0487: if (diff > PING_MIN_RUNNING) {
0488: diff = Math.min(diff, PING_MAX_RUNNING);
0489: if (attempts > diff) {
0490: attempts = diff;
0491: }
0492: } else {
0493: if (attempts > PING_MIN_RUNNING) {
0494: attempts = PING_MIN_RUNNING;
0495: }
0496: }
0497: seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
0498: }
0499:
0500: if ((seeds == null) || seeds.size() == 0) {
0501: return 0;
0502: }
0503: if (seeds.size() < attempts) {
0504: attempts = seeds.size();
0505: }
0506:
0507: // This will try to get Peers that are not currently in amIAccessibleDB
0508: Iterator<yacySeed> si = seeds.values().iterator();
0509: yacySeed seed;
0510:
0511: // include a YaCyNews record to my seed
0512: try {
0513: final yacyNewsRecord record = newsPool.myPublication();
0514: if (record == null) {
0515: seedDB.mySeed().put("news", "");
0516: } else {
0517: seedDB.mySeed().put(
0518: "news",
0519: de.anomic.tools.crypt.simpleEncode(record
0520: .toString()));
0521: }
0522: } catch (IOException e) {
0523: log.logSevere(
0524: "publishMySeed: problem with news encoding", e);
0525: }
0526: seedDB.mySeed().setUnusedFlags();
0527:
0528: // include current citation-rank file count
0529: seedDB.mySeed().put(
0530: yacySeed.CRWCNT,
0531: Integer.toString(switchboard.rankingOwnDistribution
0532: .size()));
0533: seedDB
0534: .mySeed()
0535: .put(
0536: yacySeed.CRTCNT,
0537: Integer
0538: .toString(switchboard.rankingOtherDistribution
0539: .size()));
0540: int newSeeds = -1;
0541: //if (seeds.length > 1) {
0542: // holding a reference to all started threads
0543: int contactedSeedCount = 0;
0544: final List<Thread> syncList = Collections
0545: .synchronizedList(new LinkedList<Thread>()); // memory for threads
0546: final serverSemaphore sync = new serverSemaphore(attempts);
0547:
0548: // going through the peer list and starting a new publisher thread for each peer
0549: int i = 0;
0550: while (si.hasNext()) {
0551: seed = (yacySeed) si.next();
0552: if (seed == null) {
0553: sync.P();
0554: continue;
0555: }
0556: i++;
0557:
0558: final String address = seed.getClusterAddress();
0559: log.logFine("HELLO #" + i + " to peer '"
0560: + seed.get(yacySeed.NAME, "") + "' at "
0561: + address); // debug
0562: String seederror = seed.isProper();
0563: if ((address == null) || (seederror != null)) {
0564: // we don't like that address, delete it
0565: peerActions.peerDeparture(seed,
0566: "peer ping to peer resulted in address = "
0567: + address + "; seederror = "
0568: + seederror);
0569: sync.P();
0570: } else {
0571: // starting a new publisher thread
0572: contactedSeedCount++;
0573: (new publishThread(yacyCore.publishThreadGroup,
0574: seed, sync, syncList)).start();
0575: }
0576: }
0577:
0578: // receiving the result of all started publisher threads
0579: for (int j = 0; j < contactedSeedCount; j++) {
0580:
0581: // waiting for the next thread to finish
0582: sync.P();
0583:
0584: // if this is true something is wrong ...
0585: if (syncList.isEmpty()) {
0586: log
0587: .logWarning("PeerPing: syncList.isEmpty()==true");
0588: continue;
0589: //return 0;
0590: }
0591:
0592: // getting a reference to the finished thread
0593: final publishThread t = (publishThread) syncList
0594: .remove(0);
0595:
0596: // getting the amount of new reported seeds
0597: if (t.added >= 0) {
0598: if (newSeeds == -1) {
0599: newSeeds = t.added;
0600: } else {
0601: newSeeds += t.added;
0602: }
0603: }
0604: }
0605:
0606: int accessible = 0;
0607: int notaccessible = 0;
0608: final long cutofftime = System.currentTimeMillis()
0609: - PING_MAX_DBAGE;
0610: final int dbSize;
0611: synchronized (amIAccessibleDB) {
0612: dbSize = amIAccessibleDB.size();
0613: Iterator<String> ai = amIAccessibleDB.keySet()
0614: .iterator();
0615: while (ai.hasNext()) {
0616: yacyAccessible ya = (yacyAccessible) amIAccessibleDB
0617: .get(ai.next());
0618: if (ya.lastUpdated < cutofftime) {
0619: ai.remove();
0620: } else {
0621: if (ya.IWasAccessed) {
0622: accessible++;
0623: } else {
0624: notaccessible++;
0625: }
0626: }
0627: }
0628: log.logFine("DBSize before -> after Cleanup: " + dbSize
0629: + " -> " + amIAccessibleDB.size());
0630: }
0631: log.logInfo("PeerPing: I am accessible for " + accessible
0632: + " peer(s), not accessible for " + notaccessible
0633: + " peer(s).");
0634:
0635: if ((accessible + notaccessible) > 0) {
0636: final String newPeerType;
0637: // At least one other Peer told us our type
0638: if ((accessible >= PING_MIN_PEERSEEN)
0639: || (accessible >= notaccessible)) {
0640: // We can be reached from a majority of other Peers
0641: if (yacyCore.seedDB.mySeed().isPrincipal()) {
0642: newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
0643: } else {
0644: newPeerType = yacySeed.PEERTYPE_SENIOR;
0645: }
0646: } else {
0647: // We cannot be reached from the outside
0648: newPeerType = yacySeed.PEERTYPE_JUNIOR;
0649: }
0650: if (yacyCore.seedDB.mySeed().orVirgin().equals(
0651: newPeerType)) {
0652: log.logInfo("PeerPing: myType is "
0653: + yacyCore.seedDB.mySeed().orVirgin());
0654: } else {
0655: log.logInfo("PeerPing: changing myType from '"
0656: + yacyCore.seedDB.mySeed().orVirgin()
0657: + "' to '" + newPeerType + "'");
0658: yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE,
0659: newPeerType);
0660: }
0661: } else {
0662: log.logInfo("PeerPing: No data, staying at myType: "
0663: + yacyCore.seedDB.mySeed().orVirgin());
0664: }
0665:
0666: if (newSeeds >= 0) {
0667: // success! we have published our peer to a senior peer
0668: // update latest news from the other peer
0669: // log.logInfo("publish: handshaked " + t.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + t.seed.getName() + "' at " + t.seed.getAddress());
0670: peerActions.saveMySeed();
0671: return newSeeds;
0672: }
0673:
0674: // if we have an address, we do nothing
0675: if (seedDB.mySeed().isProper() == null && !force) {
0676: return 0;
0677: }
0678:
0679: // still no success: ask own NAT or internet responder
0680: //final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true");
0681: //final String DI604pw = switchboard.getConfig("DI604pw", "");
0682: String ip = switchboard.getConfig("staticIP", "");
0683: //if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
0684:
0685: // yacyCore.log.logDebug("DEBUG: new IP=" + ip);
0686: seedDB.mySeed().put(yacySeed.IP, ip);
0687: if (seedDB.mySeed().get(yacySeed.PEERTYPE,
0688: yacySeed.PEERTYPE_JUNIOR).equals(
0689: yacySeed.PEERTYPE_JUNIOR)) // ???????????????
0690: seedDB.mySeed().put(yacySeed.PEERTYPE,
0691: yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
0692: log
0693: .logInfo("publish: no recipient found, our address is "
0694: + ((seedDB.mySeed().getPublicAddress() == null) ? "unknown"
0695: : seedDB.mySeed()
0696: .getPublicAddress()));
0697: peerActions.saveMySeed();
0698: return 0;
0699: } catch (InterruptedException e) {
0700: try {
0701: log
0702: .logInfo("publish: Interruption detected while publishing my seed.");
0703:
0704: // consuming the theads interrupted signal
0705: Thread.interrupted();
0706:
0707: // interrupt all already started publishThreads
0708: log.logInfo("publish: Signaling shutdown to "
0709: + yacyCore.publishThreadGroup.activeCount()
0710: + " remaining publishing threads ...");
0711: yacyCore.publishThreadGroup.interrupt();
0712:
0713: // waiting some time for the publishThreads to finish execution
0714: try {
0715: Thread.sleep(500);
0716: } catch (InterruptedException ex) {
0717: }
0718:
0719: // getting the amount of remaining publishing threads
0720: int threadCount = yacyCore.publishThreadGroup
0721: .activeCount();
0722: final Thread[] threadList = new Thread[threadCount];
0723: threadCount = yacyCore.publishThreadGroup
0724: .enumerate(threadList);
0725:
0726: // we need to use a timeout here because of missing interruptable session threads ...
0727: log.logFine("publish: Trying to abort "
0728: + yacyCore.publishThreadGroup.activeCount()
0729: + " remaining publishing threads ...");
0730: for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) {
0731: Thread currentThread = threadList[currentThreadIdx];
0732:
0733: if (currentThread.isAlive()) {
0734: // TODO: this object should care of all open clien connections within this class and close them here
0735: }
0736: }
0737:
0738: // we need to use a timeout here because of missing interruptable session threads ...
0739: log
0740: .logFine("publish: Waiting for "
0741: + yacyCore.publishThreadGroup
0742: .activeCount()
0743: + " remaining publishing threads to finish shutdown ...");
0744: for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) {
0745: final Thread currentThread = threadList[currentThreadIdx];
0746:
0747: if (currentThread.isAlive()) {
0748: log
0749: .logFine("publish: Waiting for remaining publishing thread '"
0750: + currentThread.getName()
0751: + "' to finish shutdown");
0752: try {
0753: currentThread.join(500);
0754: } catch (InterruptedException ex) {
0755: }
0756: }
0757: }
0758:
0759: log
0760: .logInfo("publish: Shutdown off all remaining publishing thread finished.");
0761:
0762: } catch (Exception ee) {
0763: log
0764: .logWarning(
0765: "publish: Unexpected error while trying to shutdown all remaining publishing threads.",
0766: e);
0767: }
0768:
0769: return 0;
0770: }
0771: }
0772:
0773: @SuppressWarnings("unchecked")
0774: public static HashMap<String, String> getSeedUploadMethods() {
0775: synchronized (yacyCore.seedUploadMethods) {
0776: return (HashMap<String, String>) yacyCore.seedUploadMethods
0777: .clone();
0778: }
0779: }
0780:
0781: public static yacySeedUploader getSeedUploader(String methodname) {
0782: String className = null;
0783: synchronized (yacyCore.seedUploadMethods) {
0784: if (yacyCore.seedUploadMethods.containsKey(methodname)) {
0785: className = (String) yacyCore.seedUploadMethods
0786: .get(methodname);
0787: }
0788: }
0789:
0790: if (className == null) {
0791: return null;
0792: }
0793: try {
0794: final Class<?> uploaderClass = Class.forName(className);
0795: final Object uploader = uploaderClass.newInstance();
0796: return (yacySeedUploader) uploader;
0797: } catch (Exception e) {
0798: return null;
0799: }
0800: }
0801:
0802: public static void loadSeedUploadMethods() {
0803: final HashMap<String, String> availableUploaders = new HashMap<String, String>();
0804: try {
0805: final String uploadersPkgName = yacyCore.class.getPackage()
0806: .getName()
0807: + ".seedUpload";
0808: final String packageURI = yacyCore.class.getResource(
0809: "/" + uploadersPkgName.replace('.', '/'))
0810: .toString();
0811:
0812: // open the parser directory
0813: final File uploadersDir = new File(new URI(packageURI));
0814: if ((uploadersDir == null) || (!uploadersDir.exists())
0815: || (!uploadersDir.isDirectory())) {
0816: yacyCore.seedUploadMethods.clear();
0817: changeSeedUploadMethod("none");
0818: }
0819:
0820: final String[] uploaderClasses = uploadersDir
0821: .list(new FilenameFilter() {
0822: public boolean accept(File dir, String name) {
0823: return name.startsWith("yacySeedUpload")
0824: && name.endsWith(".class");
0825: }
0826: });
0827:
0828: final String javaClassPath = System
0829: .getProperty("java.class.path");
0830:
0831: if (uploaderClasses == null) {
0832: return;
0833: }
0834: for (int uploaderNr = 0; uploaderNr < uploaderClasses.length; uploaderNr++) {
0835: final String className = uploaderClasses[uploaderNr]
0836: .substring(0, uploaderClasses[uploaderNr]
0837: .indexOf(".class"));
0838: final String fullClassName = uploadersPkgName + "."
0839: + className;
0840: try {
0841: final Class<?> uploaderClass = Class
0842: .forName(fullClassName);
0843: final Object theUploader = uploaderClass
0844: .newInstance();
0845: if (!(theUploader instanceof yacySeedUploader)) {
0846: continue;
0847: }
0848: final String[] neededLibx = ((yacySeedUploader) theUploader)
0849: .getLibxDependencies();
0850: if (neededLibx != null) {
0851: for (int libxId = 0; libxId < neededLibx.length; libxId++) {
0852: if (javaClassPath
0853: .indexOf(neededLibx[libxId]) == -1) {
0854: throw new Exception(
0855: "Missing dependency");
0856: }
0857: }
0858: }
0859: availableUploaders.put(className
0860: .substring("yacySeedUpload".length()),
0861: fullClassName);
0862: } catch (Exception e) { /* we can ignore this for the moment */
0863: } catch (Error e) { /* we can ignore this for the moment */
0864: }
0865: }
0866: } catch (Exception e) {
0867: } finally {
0868: synchronized (yacyCore.seedUploadMethods) {
0869: yacyCore.seedUploadMethods.clear();
0870: yacyCore.seedUploadMethods.putAll(availableUploaders);
0871: }
0872: }
0873: }
0874:
0875: public static boolean changeSeedUploadMethod(String method) {
0876: if (method == null || method.length() == 0) {
0877: return false;
0878: }
0879:
0880: if (method.equalsIgnoreCase("none")) {
0881: return true;
0882: }
0883:
0884: synchronized (yacyCore.seedUploadMethods) {
0885: return yacyCore.seedUploadMethods.containsKey(method);
0886: }
0887: }
0888:
0889: public final String saveSeedList() {
0890: // return an error if this is not successful, and NULL if everything is fine
0891: return saveSeedList(this .switchboard);
0892: }
0893:
0894: public final String saveSeedList(serverSwitch sb) {
0895: try {
0896: // return an error if this is not successful, and NULL if everything is fine
0897: String logt;
0898:
0899: // be shure that we have something to say
0900: if (seedDB.mySeed().getPublicAddress() == null) {
0901: final String errorMsg = "We have no valid IP address until now";
0902: log.logWarning("SaveSeedList: " + errorMsg);
0903: return errorMsg;
0904: }
0905:
0906: // getting the configured seed uploader
0907: String seedUploadMethod = sb.getConfig("seedUploadMethod",
0908: "");
0909:
0910: // for backward compatiblity ....
0911: if (seedUploadMethod.equalsIgnoreCase("Ftp")
0912: || (seedUploadMethod.equals("") && sb.getConfig(
0913: "seedFTPPassword", "").length() > 0)) {
0914:
0915: seedUploadMethod = "Ftp";
0916: sb.setConfig("seedUploadMethod", seedUploadMethod);
0917:
0918: } else if (seedUploadMethod.equalsIgnoreCase("File")
0919: || (seedUploadMethod.equals("") && sb.getConfig(
0920: "seedFilePath", "").length() > 0)) {
0921:
0922: seedUploadMethod = "File";
0923: sb.setConfig("seedUploadMethod", seedUploadMethod);
0924: }
0925:
0926: // determine the seed uploader that should be used ...
0927: if (seedUploadMethod.equalsIgnoreCase("none")) {
0928: return "no uploader specified";
0929: }
0930:
0931: yacySeedUploader uploader = getSeedUploader(seedUploadMethod);
0932: if (uploader == null) {
0933: final String errorMsg = "Unable to get the proper uploader-class for seed uploading method '"
0934: + seedUploadMethod + "'.";
0935: log.logWarning("SaveSeedList: " + errorMsg);
0936: return errorMsg;
0937: }
0938:
0939: // ensure that the seed file url is configured properly
0940: yacyURL seedURL;
0941: try {
0942: final String seedURLStr = sb.getConfig("seedURL", "");
0943: if (seedURLStr.length() == 0) {
0944: throw new MalformedURLException(
0945: "The seed-file url must not be empty.");
0946: }
0947: if (!(seedURLStr.toLowerCase().startsWith("http://") || seedURLStr
0948: .toLowerCase().startsWith("https://"))) {
0949: throw new MalformedURLException(
0950: "Unsupported protocol.");
0951: }
0952: seedURL = new yacyURL(seedURLStr, null);
0953: } catch (MalformedURLException e) {
0954: final String errorMsg = "Malformed seed file URL '"
0955: + sb.getConfig("seedURL", "") + "'. "
0956: + e.getMessage();
0957: log.logWarning("SaveSeedList: " + errorMsg);
0958: return errorMsg;
0959: }
0960:
0961: // upload the seed-list using the configured uploader class
0962: String prevStatus = seedDB.mySeed().get(yacySeed.PEERTYPE,
0963: yacySeed.PEERTYPE_JUNIOR);
0964: if (prevStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
0965: prevStatus = yacySeed.PEERTYPE_SENIOR;
0966: }
0967:
0968: try {
0969: seedDB.mySeed().put(yacySeed.PEERTYPE,
0970: yacySeed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
0971:
0972: log
0973: .logFine("SaveSeedList: Using seed uploading method '"
0974: + seedUploadMethod
0975: + "' for seed-list uploading."
0976: + "\n\tPrevious peerType is '"
0977: + seedDB.mySeed().get(
0978: yacySeed.PEERTYPE,
0979: yacySeed.PEERTYPE_JUNIOR)
0980: + "'.");
0981:
0982: // logt = seedDB.uploadCache(seedFTPServer, seedFTPAccount, seedFTPPassword, seedFTPPath, seedURL);
0983: logt = seedDB
0984: .uploadCache(uploader, sb, seedDB, seedURL);
0985: if (logt != null) {
0986: if (logt.indexOf("Error") >= 0) {
0987: seedDB.mySeed().put(yacySeed.PEERTYPE,
0988: prevStatus);
0989: final String errorMsg = "SaveSeedList: seed upload failed using "
0990: + uploader.getClass().getName()
0991: + " (error): "
0992: + logt
0993: .substring(logt
0994: .indexOf("Error") + 6);
0995: log.logSevere(errorMsg);
0996: return errorMsg;
0997: }
0998: log.logInfo(logt);
0999: }
1000:
1001: // finally, set the principal status
1002: sb.setConfig("yacyStatus", yacySeed.PEERTYPE_PRINCIPAL);
1003: return null;
1004: } catch (Exception e) {
1005: seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
1006: sb.setConfig("yacyStatus", prevStatus);
1007: final String errorMsg = "SaveSeedList: Seed upload failed (IO error): "
1008: + e.getMessage();
1009: log.logInfo(errorMsg, e);
1010: return errorMsg;
1011: }
1012: } finally {
1013: this .lastSeedUpload_seedDBSize = seedDB.sizeConnected();
1014: this .lastSeedUpload_timeStamp = System.currentTimeMillis();
1015:
1016: this .lastSeedUpload_myIP = seedDB.mySeed().get(yacySeed.IP,
1017: "127.0.0.1");
1018: //this.lastSeedUpload_myPeerType = seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
1019: }
1020: }
1021:
1022: }
|