001: // yacyVersion.java
002: // ----------------
003: // (C) 2007 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
004: // first published 27.04.2007 on http://yacy.net
005: //
006: // This is a part of YaCy, a peer-to-peer based web search engine
007: //
008: // $LastChangedDate: 2008-01-31 23:40:47 +0000 (Do, 31 Jan 2008) $
009: // $LastChangedRevision: 4424 $
010: // $LastChangedBy: orbiter $
011: //
012: // LICENSE
013: //
014: // This program is free software; you can redistribute it and/or modify
015: // it under the terms of the GNU General Public License as published by
016: // the Free Software Foundation; either version 2 of the License, or
017: // (at your option) any later version.
018: //
019: // This program is distributed in the hope that it will be useful,
020: // but WITHOUT ANY WARRANTY; without even the implied warranty of
021: // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
022: // GNU General Public License for more details.
023: //
024: // You should have received a copy of the GNU General Public License
025: // along with this program; if not, write to the Free Software
026: // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
027:
028: package de.anomic.yacy;
029:
030: import java.io.File;
031: import java.io.IOException;
032: import java.util.ArrayList;
033: import java.util.Comparator;
034: import java.util.HashMap;
035: import java.util.Iterator;
036: import java.util.Map;
037: import java.util.TreeSet;
038: import java.util.regex.Matcher;
039: import java.util.regex.Pattern;
040:
041: import de.anomic.htmlFilter.htmlFilterContentScraper;
042: import de.anomic.http.httpc;
043: import de.anomic.plasma.plasmaSwitchboard;
044: import de.anomic.server.serverCore;
045: import de.anomic.server.serverSystem;
046: import de.anomic.server.logging.serverLog;
047:
048: public final class yacyVersion implements Comparator<yacyVersion>,
049: Comparable<yacyVersion> {
050:
051: // general release info
052: public static final float YACY_SUPPORTS_PORT_FORWARDING = (float) 0.383;
053: public static final float YACY_SUPPORTS_GZIP_POST_REQUESTS = (float) 0.40300772;
054: public static final float YACY_ACCEPTS_RANKING_TRANSMISSION = (float) 0.414;
055: public static final float YACY_HANDLES_COLLECTION_INDEX = (float) 0.486;
056: public static final float YACY_POVIDES_REMOTECRAWL_LISTS = (float) 0.550;
057: public static final float YACY_STANDARDREL_IS_PRO = (float) 0.557;
058:
059: // information about latest release, retrieved by other peers release version
060: public static double latestRelease = 0.1; // this value is overwritten when a peer with later version appears
061:
062: // information about latest release, retrieved from download pages
063: // this static information should be overwritten by network-specific locations
064: // for details see yacy.network.unit
065: private static HashMap<yacyURL, DevMain> latestReleases = new HashMap<yacyURL, DevMain>();
066: public static ArrayList<yacyURL> latestReleaseLocations = new ArrayList<yacyURL>(); // will be initialized with value in yacy.network.unit
067:
068: // private static release info about this release; is generated only once and can be retrieved by thisVersion()
069: private static yacyVersion this Version = null;
070:
071: // class variables
072: public float releaseNr;
073: public String dateStamp;
074: public int svn;
075: public boolean mainRelease;
076: public yacyURL url;
077: public String name;
078:
079: public yacyVersion(yacyURL url) {
080: this (url.getFileName());
081: this .url = url;
082: }
083:
084: public yacyVersion(String release) {
085: // parse a release file name
086: // the have the following form:
087: // yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
088: // yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
089: // i.e. yacy_v0.51_20070321_3501.tar.gz
090: this .url = null;
091: this .name = release;
092: if ((release == null) || (!release.endsWith(".tar.gz"))) {
093: throw new RuntimeException("release file name '" + release
094: + "' is not valid, no tar.gz");
095: }
096: // cut off tail
097: release = release.substring(0, release.length() - 7);
098: if (release.startsWith("yacy_pro_v")) {
099: release = release.substring(10);
100: } else if (release.startsWith("yacy_emb_v")) {
101: throw new RuntimeException("release file name '" + release
102: + "' is not valid, no support for emb");
103: } else if (release.startsWith("yacy_v")) {
104: release = release.substring(6);
105: } else {
106: throw new RuntimeException("release file name '" + release
107: + "' is not valid, wrong prefix");
108: }
109: // now all release names have the form
110: // ${releaseVersion}_${DSTAMP}_${releaseNr}
111: String[] comp = release.split("_"); // should be 3 parts
112: if (comp.length != 3) {
113: throw new RuntimeException("release file name '" + release
114: + "' is not valid, 3 information parts expected");
115: }
116: try {
117: this .releaseNr = Float.parseFloat(comp[0]);
118: } catch (NumberFormatException e) {
119: throw new RuntimeException("release file name '" + release
120: + "' is not valid, '" + comp[0]
121: + "' should be a float number");
122: }
123: this .mainRelease = ((int) (this .releaseNr * (float) 1000)) % 10 == 0;
124: //System.out.println("Release version " + this.releaseNr + " is " + ((this.mainRelease) ? "main" : "std"));
125: this .dateStamp = comp[1];
126: if (this .dateStamp.length() != 8) {
127: throw new RuntimeException("release file name '" + release
128: + "' is not valid, '" + comp[1]
129: + "' should be a 8-digit date string");
130: }
131: try {
132: this .svn = Integer.parseInt(comp[2]);
133: } catch (NumberFormatException e) {
134: throw new RuntimeException("release file name '" + release
135: + "' is not valid, '" + comp[2]
136: + "' should be a integer number");
137: }
138: // finished! we parsed a relase string
139: }
140:
141: public static final class DevMain {
142: public TreeSet<yacyVersion> dev, main;
143:
144: public DevMain(TreeSet<yacyVersion> dev,
145: TreeSet<yacyVersion> main) {
146: this .dev = dev;
147: this .main = main;
148: }
149: }
150:
151: public int compareTo(yacyVersion obj) {
152: // returns 0 if this object is equal to the obj, -1 if this is smaller than obj and 1 if this is greater than obj
153: return compare(this , obj);
154: }
155:
156: public int compare(yacyVersion v0, yacyVersion v1) {
157: // compare-operator for two yacyVersion objects
158: // must be implemented to make it possible to put this object into
159: // a ordered structure, like TreeSet or TreeMap
160: return (new Integer(v0.svn)).compareTo(new Integer(v1.svn));
161: }
162:
163: public boolean equals(Object obj) {
164: yacyVersion v = (yacyVersion) obj;
165: return (this .svn == v.svn)
166: && (this .url.toNormalform(true, true).equals(v.url
167: .toNormalform(true, true)));
168: }
169:
170: public int hashCode() {
171: return this .url.toNormalform(true, true).hashCode();
172: }
173:
174: public String toAnchor() {
175: // generates an anchor string that can be used to embed in an html for direct download
176: return "<a href=" + this .url.toNormalform(true, true)
177: + ">YaCy "
178: + ((this .mainRelease) ? "main release" : "dev release")
179: + " v" + this .releaseNr + ", SVN " + this .svn + "</a>";
180: }
181:
182: // static methods:
183:
184: public static final yacyVersion this Version() {
185: // construct a virtual release name for this release
186: if (this Version == null) {
187: plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
188: boolean full = new File(sb.getRootPath(), "libx").exists();
189: this Version = new yacyVersion("yacy"
190: + ((full) ? "" : "_emb") + "_v"
191: + sb.getConfig("version", "0.1") + "_"
192: + sb.getConfig("vdate", "19700101") + "_"
193: + sb.getConfig("svnRevision", "0") + ".tar.gz");
194: }
195: return this Version;
196: }
197:
198: public static final yacyVersion rulebasedUpdateInfo(boolean manual) {
199: // according to update properties, decide if we should retrieve update information
200: // if true, the release that can be obtained is returned.
201: // if false, null is returned
202: plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
203:
204: // check if update process allows update retrieve
205: String process = sb.getConfig("update.process", "manual");
206: if ((!manual) && (!process.equals("auto"))) {
207: yacyCore.log
208: .logInfo("rulebasedUpdateInfo: not an automatic update selected");
209: return null; // no, its a manual or guided process
210: }
211:
212: // check if the last retrieve time is a minimum time ago
213: long cycle = Math.max(1, sb.getConfigLong("update.cycle", 168)) * 60 * 60 * 1000; // update.cycle is hours
214: long timeLookup = sb.getConfigLong("update.time.lookup", System
215: .currentTimeMillis());
216: if ((!manual)
217: && (timeLookup + cycle > System.currentTimeMillis())) {
218: yacyCore.log
219: .logInfo("rulebasedUpdateInfo: too early for a lookup for a new release (timeLookup = "
220: + timeLookup
221: + ", cycle = "
222: + cycle
223: + ", now = "
224: + System.currentTimeMillis()
225: + ")");
226: return null; // no we have recently made a lookup
227: }
228:
229: // check if we know that there is a release that is more recent than that which we are using
230: DevMain releasess = yacyVersion.allReleases(true);
231: yacyVersion latestmain = (releasess.main.size() == 0) ? null
232: : releasess.main.last();
233: yacyVersion latestdev = (releasess.dev.size() == 0) ? null
234: : releasess.dev.last();
235: String concept = sb.getConfig("update.concept", "any");
236: String blacklist = sb.getConfig("update.blacklist",
237: ".\\...[123]");
238:
239: if ((manual) || (concept.equals("any"))) {
240: // return a dev-release or a main-release
241: if ((latestdev != null)
242: && ((latestmain == null) || (latestdev
243: .compareTo(latestmain) > 0))
244: && (!(Float.toString(latestdev.releaseNr)
245: .matches(blacklist)))) {
246: // consider a dev-release
247: if (latestdev.compareTo(this Version()) > 0) {
248: return latestdev;
249: } else {
250: yacyCore.log
251: .logInfo("rulebasedUpdateInfo: latest dev "
252: + latestdev.name
253: + " is not more recent than installed release "
254: + this Version().name);
255: return null;
256: }
257: }
258: if (latestmain != null) {
259: // consider a main release
260: if ((Float.toString(latestmain.releaseNr)
261: .matches(blacklist))) {
262: yacyCore.log
263: .logInfo("rulebasedUpdateInfo: latest dev "
264: + latestdev.name
265: + " matches with blacklist '"
266: + blacklist + "'");
267: return null;
268: }
269: if (latestmain.compareTo(this Version()) > 0)
270: return latestmain;
271: else {
272: yacyCore.log
273: .logInfo("rulebasedUpdateInfo: latest main "
274: + latestmain.name
275: + " is not more recent than installed release (1) "
276: + this Version().name);
277: return null;
278: }
279: }
280: }
281: if ((concept.equals("main")) && (latestmain != null)) {
282: // return a main-release
283: if ((Float.toString(latestmain.releaseNr)
284: .matches(blacklist))) {
285: yacyCore.log
286: .logInfo("rulebasedUpdateInfo: latest main "
287: + latestmain.name
288: + " matches with blacklist'"
289: + blacklist + "'");
290: return null;
291: }
292: if (latestmain.compareTo(this Version()) > 0)
293: return latestmain;
294: else {
295: yacyCore.log
296: .logInfo("rulebasedUpdateInfo: latest main "
297: + latestmain.name
298: + " is not more recent than installed release (2) "
299: + this Version().name);
300: return null;
301: }
302: }
303: yacyCore.log
304: .logInfo("rulebasedUpdateInfo: failed to find more recent release");
305: return null;
306: }
307:
308: public static DevMain allReleases(boolean force) {
309: // join the release infos
310: DevMain[] a = new DevMain[latestReleaseLocations.size()];
311: for (int j = 0; j < latestReleaseLocations.size(); j++) {
312: a[j] = getReleases(latestReleaseLocations.get(j), force);
313: }
314: TreeSet<yacyVersion> alldev = new TreeSet<yacyVersion>();
315: TreeSet<yacyVersion> allmain = new TreeSet<yacyVersion>();
316: for (int j = 0; j < a.length; j++)
317: if ((a[j] != null) && (a[j].dev != null))
318: alldev.addAll(a[j].dev);
319: for (int j = 0; j < a.length; j++)
320: if ((a[j] != null) && (a[j].main != null))
321: allmain.addAll(a[j].main);
322:
323: return new DevMain(alldev, allmain);
324: }
325:
326: private static DevMain getReleases(yacyURL location, boolean force) {
327: // get release info from a internet resource
328: DevMain latestRelease = latestReleases.get(location);
329: if (force || (latestRelease == null) /*||
330: ((latestRelease[0].size() == 0) &&
331: (latestRelease[1].size() == 0) &&
332: (latestRelease[2].size() == 0) &&
333: (latestRelease[3].size() == 0) )*/) {
334: latestRelease = allReleaseFrom(location);
335: latestReleases.put(location, latestRelease);
336: }
337: return latestRelease;
338: }
339:
340: private static DevMain allReleaseFrom(yacyURL url) {
341: // retrieves the latest info about releases
342: // this is done by contacting a release location,
343: // parsing the content and filtering+parsing links
344: // returns the version info if successful, null otherwise
345: htmlFilterContentScraper scraper;
346: try {
347: scraper = htmlFilterContentScraper.parseResource(url);
348: } catch (IOException e) {
349: return null;
350: }
351:
352: // analyse links in scraper resource, and find link to latest release in it
353: Map<yacyURL, String> anchors = scraper.getAnchors(); // a url (String) / name (String) relation
354: Iterator<yacyURL> i = anchors.keySet().iterator();
355: TreeSet<yacyVersion> devreleases = new TreeSet<yacyVersion>();
356: TreeSet<yacyVersion> mainreleases = new TreeSet<yacyVersion>();
357: yacyVersion release;
358: while (i.hasNext()) {
359: url = i.next();
360: try {
361: release = new yacyVersion(url);
362: //System.out.println("r " + release.toAnchor());
363: if (release.mainRelease)
364: mainreleases.add(release);
365: if (!release.mainRelease)
366: devreleases.add(release);
367: } catch (RuntimeException e) {
368: // the release string was not well-formed.
369: // that might have been another link
370: // just dont care
371: continue;
372: }
373: }
374: plasmaSwitchboard.getSwitchboard().setConfig(
375: "update.time.lookup", System.currentTimeMillis());
376: return new DevMain(devreleases, mainreleases);
377: }
378:
379: public static void downloadRelease(yacyVersion release)
380: throws IOException {
381: File storagePath = plasmaSwitchboard.getSwitchboard().releasePath;
382: // load file
383: File download = new File(storagePath, release.url.getFileName());
384: httpc.wget(release.url, release.url.getHost(), 300000, null,
385: null,
386: plasmaSwitchboard.getSwitchboard().remoteProxyConfig,
387: null, download);
388: if ((!download.exists()) || (download.length() == 0))
389: throw new IOException("wget of url " + release.url
390: + " failed");
391: plasmaSwitchboard.getSwitchboard().setConfig(
392: "update.time.download", System.currentTimeMillis());
393: }
394:
395: public static void restart() {
396: plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
397:
398: if (System.getProperty("os.name").toLowerCase().startsWith(
399: "win")) {
400: // create yacy.restart file which is used in Windows startscript
401: final File yacyRestart = new File(sb.getRootPath(),
402: "DATA/yacy.restart");
403: if (!yacyRestart.exists()) {
404: try {
405: yacyRestart.createNewFile();
406: plasmaSwitchboard.getSwitchboard().terminate(5000);
407: } catch (IOException e) {
408: serverLog
409: .logSevere("SHUTDOWN", "restart failed", e);
410: }
411: }
412:
413: }
414:
415: if (serverSystem.canExecUnix) {
416: // start a re-start daemon
417: try {
418: serverLog.logInfo("RESTART", "INITIATED");
419: String script = "#!/bin/sh" + serverCore.LF_STRING
420: + "cd " + sb.getRootPath() + "/DATA/RELEASE/"
421: + serverCore.LF_STRING
422: + "while [ -f ../yacy.running ]; do"
423: + serverCore.LF_STRING + "sleep 1"
424: + serverCore.LF_STRING + "done"
425: + serverCore.LF_STRING + "cd ../../"
426: + serverCore.LF_STRING
427: + "nohup ./startYACY.sh > /dev/null"
428: + serverCore.LF_STRING;
429: File scriptFile = new File(sb.getRootPath(),
430: "DATA/RELEASE/restart.sh");
431: serverSystem.deployScript(scriptFile, script);
432: serverLog.logInfo("RESTART", "wrote restart-script to "
433: + scriptFile.getAbsolutePath());
434: serverSystem.execAsynchronous(scriptFile);
435: serverLog.logInfo("RESTART", "script is running");
436: sb.terminate(5000);
437: } catch (IOException e) {
438: serverLog.logSevere("RESTART", "restart failed", e);
439: }
440: }
441: }
442:
443: public static void deployRelease(String release) {
444: //byte[] script = ("cd " + plasmaSwitchboard.getSwitchboard().getRootPath() + ";while [ -e ../yacy.running ]; do sleep 1;done;tar xfz " + release + ";cp -Rf yacy/* ../../;rm -Rf yacy;cd ../../;startYACY.sh").getBytes();
445: try {
446: plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
447: String apphome = sb.getRootPath().toString();
448: serverLog.logInfo("UPDATE", "INITIATED");
449: String script = "#!/bin/sh" + serverCore.LF_STRING + "cd "
450: + sb.getRootPath() + "/DATA/RELEASE/"
451: + serverCore.LF_STRING + "if gunzip -t " + release
452: + serverCore.LF_STRING + "then"
453: + serverCore.LF_STRING + "gunzip -c " + release
454: + " | tar xf -" + serverCore.LF_STRING
455: + "while [ -f ../yacy.running ]; do"
456: + serverCore.LF_STRING + "sleep 1"
457: + serverCore.LF_STRING + "done"
458: + serverCore.LF_STRING + "cp -Rf yacy/* " + apphome
459: + serverCore.LF_STRING + "rm -Rf yacy"
460: + serverCore.LF_STRING + "else"
461: + serverCore.LF_STRING
462: + "while [ -f ../yacy.running ]; do"
463: + serverCore.LF_STRING + "sleep 1"
464: + serverCore.LF_STRING + "done"
465: + serverCore.LF_STRING + "fi"
466: + serverCore.LF_STRING + "cd " + apphome
467: + serverCore.LF_STRING
468: + "nohup ./startYACY.sh > /dev/null"
469: + serverCore.LF_STRING;
470: File scriptFile = new File(sb.getRootPath(),
471: "DATA/RELEASE/update.sh");
472: serverSystem.deployScript(scriptFile, script);
473: serverLog.logInfo("UPDATE", "wrote update-script to "
474: + scriptFile.getAbsolutePath());
475: serverSystem.execAsynchronous(scriptFile);
476: serverLog.logInfo("UPDATE", "script is running");
477: sb.setConfig("update.time.deploy", System
478: .currentTimeMillis());
479: sb.terminate(5000);
480: } catch (IOException e) {
481: serverLog.logSevere("UPDATE", "update failed", e);
482: }
483: }
484:
485: /**
486: * Converts combined version-string to a pretty string, e.g. "0.435/01818" or "dev/01818" (development version) or "dev/00000" (in case of wrong input)
487: *
488: * @param ver Combined version string matching regular expression: "\A(\d+\.\d{3})(\d{4}|\d{5})\z" <br>
489: * (i.e.: start of input, 1 or more digits in front of decimal point, decimal point followed by 3 digits as major version, 4 or 5 digits for SVN-Version, end of input)
490: * @return If the major version is < 0.11 - major version is separated from SVN-version by '/', e.g. "0.435/01818" <br>
491: * If the major version is >= 0.11 - major version is replaced by "dev" and separated SVN-version by '/', e.g."dev/01818" <br>
492: * "dev/00000" - If the input does not matcht the regular expression above
493: */
494: public static String combined2prettyVersion(String ver) {
495: return combined2prettyVersion(ver, "");
496: }
497:
498: public static String combined2prettyVersion(String ver,
499: String computerName) {
500: final Matcher matcher = Pattern.compile(
501: "\\A(\\d+\\.\\d{1,3})(\\d{0,5})\\z").matcher(ver);
502:
503: if (!matcher.find()) {
504: serverLog.logWarning("STARTUP", "Peer '" + computerName
505: + "': wrong format of version-string: '" + ver
506: + "'. Using default string 'dev/00000' instead");
507: return "dev/00000";
508: }
509:
510: String mainversion = (Double.parseDouble(matcher.group(1)) < 0.11 ? "dev"
511: : matcher.group(1));
512: String revision = matcher.group(2);
513: for (int i = revision.length(); i < 5; ++i)
514: revision += "0";
515: return mainversion + "/" + revision;
516: }
517:
518: /**
519: * Combines the version of YaCy with the versionnumber from SVN to a
520: * combined version
521: *
522: * @param version Current given version.
523: * @param svn Current version given from SVN.
524: * @return String with the combined version.
525: */
526: public static double versvn2combinedVersion(double v, int svn) {
527: return (Math.rint((v * 100000000.0) + ((double) svn)) / 100000000);
528: }
529:
530: public static void main(String[] args) {
531: float base = (float) 0.53;
532: String blacklist = "....[123]";
533: String test;
534: for (int i = 0; i < 20; i++) {
535: test = Float.toString(base + (((float) i) / 1000));
536: System.out.println(test
537: + " is "
538: + ((test.matches(blacklist)) ? "blacklisted"
539: : " not blacklisted"));
540: }
541: }
542:
543: }
|