0001: // CacheStore.java
0002: // $Id: CacheStore.java,v 1.66 2007/02/09 22:16:38 ylafon Exp $
0003: // (c) COPYRIGHT MIT, INRIA and Keio, 1999.
0004: // Please first read the full copyright statement in file COPYRIGHT.html
0005:
0006: package org.w3c.www.protocol.http.cache;
0007:
0008: import java.util.Enumeration;
0009: import java.util.Hashtable;
0010: import java.util.Vector;
0011:
0012: import java.io.BufferedReader;
0013: import java.io.BufferedWriter;
0014: import java.io.File;
0015: import java.io.FileNotFoundException;
0016: import java.io.FileReader;
0017: import java.io.FileWriter;
0018: import java.io.FilenameFilter;
0019: import java.io.IOException;
0020: import java.io.PrintStream;
0021: import java.io.Reader;
0022: import java.io.Writer;
0023:
0024: import org.w3c.util.LRUAble;
0025: import org.w3c.util.ObservableProperties;
0026: import org.w3c.util.PropertyMonitoring;
0027: import org.w3c.util.SyncLRUList;
0028:
0029: import org.w3c.tools.sorter.Sorter;
0030:
0031: public class CacheStore implements PropertyMonitoring {
0032:
0033: /**
0034: * Name of the property giving the cache's directory.
0035: * This property defaults to the current directory.
0036: */
0037: public static final String CACHE_DIRECTORY_P = "org.w3c.www.protocol.http.cache.directory";
0038:
0039: /**
0040: * The name of the properties indicating the storesize of the cache
0041: * (in bytes).
0042: * This property will give the value of the disk-based cache size. This
0043: * value only takes into account the size of the entities saved, not
0044: * the size of the associated headers, and not the physical size on the
0045: * disc.
0046: * <p>This property defaults to <strong>20</strong> Mbytes.
0047: */
0048: public static final String STORE_SIZE_P = "org.w3c.www.protocol.http.cache.storesize";
0049:
0050: /**
0051: * Name of the property used to knkow the percentage of bytes to be
0052: * kept after a garbage collection
0053: * It defaults to 0.80 (80% of the cache size)
0054: */
0055: public static final String GARBAGE_COLLECTION_THRESHOLD_P = "org.w3c.www.protocol.http.cache.gc_threshold";
0056:
0057: /**
0058: * Name of the property indicating the max size for files to be cached.
0059: * This property gives the ratio (relative to the cache size) of
0060: * the number of bytes a single entry is able to occupy.
0061: * <p>The ratio should be given as a floating point value between
0062: * <strong>0</strong> and <strong>1</strong>. If set to <strong>0.1
0063: * </strong> and the cache size is <strong>5000000</strong>, files larger
0064: * then <strong>500000</strong> will not be cached (except if garbage
0065: * collection is disbabled).
0066: * <p>This property defaults to <strong>0.1</strong>.
0067: * Note that the generation size will be taken from this threshold
0068: */
0069: public static final String FILE_SIZE_RATIO_P = "org.w3c.www.protocol.http.cache.fileSizeRatio";
0070:
0071: /**
0072: * Name of the property enabling garbage collection of the cache.
0073: * This property defaults to <strong>true</strong>.
0074: */
0075: public static final String GARBAGE_COLLECTION_ENABLED_P = "org.w3c.www.protocol.http.cache.garbageCollectionEnabled";
0076:
0077: /**
0078: * Name of the property indicating the amount of time in second between
0079: * two synchronization of the database (aka dump on disk)
0080: * Milliseconds
0081: */
0082: public static final String SYNCHRONIZATION_DELAY_P = "org.w3c.www.protocol.http.cache.SynchronizationDelay";
0083:
0084: /**
0085: * Name of the property indicating the amount of time in second between
0086: * two attempts to compact generations. In milliseconds
0087: */
0088: public static final String GENERATION_COMPACT_DELAY_P = "org.w3c.www.protocol.http.cache.GenerationCompactDelay";
0089:
0090: /**
0091: * Name of the property indicating the maximal number of resources
0092: * the cache can load in memory (not the content of the resources)
0093: */
0094: public static final String MAX_CACHED_RESOURCES_P = "org.w3c.www.protocol.http.cache.MaxCachedResources";
0095:
0096: /**
0097: * Name of the property indicating the maximal number of generations
0098: * in this cache store
0099: */
0100: public static final String MAX_GENERATIONS_P = "org.w3c.www.protocol.http.cache.MaxGenerations";
0101:
0102: //Generation file name.
0103: public static final String GENERATION_FILENAME = "gen-";
0104:
0105: // the store state
0106: private StoreState state = null;
0107:
0108: // the state file
0109: private File statefile = null;
0110:
0111: private SyncLRUList generations = null;
0112:
0113: // the capacity of this cache
0114: private long bytelimit = 0;
0115:
0116: // the store capacity of this cache
0117: private long storelimit = 0;
0118:
0119: // the usual length of a generation
0120: private long generationlimit = 0;
0121:
0122: // the maximum number of CachedResource in memory
0123: private int cr_limit = 0;
0124:
0125: // the file ratio (threshold)
0126: private double threshold = 0.1;
0127:
0128: // the ratio kept after a garbage collection
0129: private double gc_kept_ratio = 0.80;
0130:
0131: // the number of directories used (default 128)
0132: private int nb_dir = 128;
0133:
0134: // The cache directory
0135: private File cache_dir = null;
0136:
0137: // the directories where bodies are stored
0138: private File dirs[] = null;
0139:
0140: // the garbage collection flag
0141: private boolean garbageCollectionEnabled = true;
0142:
0143: // our father
0144: private CacheFilter filter = null;
0145:
0146: // the traditional debug flag
0147: private boolean debug = false;
0148:
0149: // the maximal number of generations
0150: private int max_generation = 10;
0151:
0152: // the delay between two store sync
0153: private long sync_delay = 60000;
0154:
0155: // the delay between two attempts to compact the database
0156: private long gencomp_delay = 60000;
0157:
0158: /**
0159: * The properties we initialized ourself from.
0160: */
0161: protected ObservableProperties props = null;
0162:
0163: /**
0164: * Property monitoring for the CacheStore.
0165: * The CacheStore allows you to dynamically (typically through the property
0166: * setter) change the class of the sweeper, the validator, the size...
0167: * @param name The name of the property that has changed.
0168: * @return A boolean, <strong>true</strong> if the change was made,
0169: * <strong>false</strong> otherwise.
0170: */
0171: public boolean propertyChanged(String name) {
0172: double dval;
0173:
0174: if (name.equals(CacheFilter.CACHE_SIZE_P)) {
0175: bytelimit = props.getLong(name, bytelimit);
0176: return true;
0177: } else if (name.equals(STORE_SIZE_P)) {
0178: storelimit = props.getLong(name, storelimit);
0179: return true;
0180: } else if (name.equals(CacheFilter.DEBUG_P)) {
0181: debug = props.getBoolean(name, debug);
0182: return true;
0183: } else if (name.equals(GARBAGE_COLLECTION_ENABLED_P)) {
0184: garbageCollectionEnabled = props.getBoolean(name, true);
0185: return true;
0186: } else if (name.equals(FILE_SIZE_RATIO_P)) {
0187: dval = props.getDouble(name, threshold);
0188: if ((dval <= (double) 0.00001) || (dval >= (double) 1.0))
0189: return false;
0190: threshold = dval;
0191: return true;
0192: } else if (name.equals(GARBAGE_COLLECTION_THRESHOLD_P)) {
0193: dval = props.getDouble(name, gc_kept_ratio);
0194: if ((dval <= (double) 0.00001) || (dval >= (double) 1.0))
0195: return false;
0196: gc_kept_ratio = dval;
0197: return true;
0198: } else if (name.equals(MAX_GENERATIONS_P)) {
0199: int new_nb = props.getInteger(name, max_generation);
0200: if (new_nb <= 0) {
0201: return false;
0202: }
0203: // not that we won't do too much if it is reduced
0204: // the user will have to clean the cache...
0205: max_generation = new_nb;
0206: return true;
0207: } else if (name.equals(MAX_CACHED_RESOURCES_P)) {
0208: int new_nb = props.getInteger(name, cr_limit);
0209: if (new_nb <= 0) {
0210: return false;
0211: }
0212: // not that we won't do too much if it is reduced
0213: // the user will have to clean the cache...
0214: cr_limit = new_nb;
0215: return true;
0216: } else if (name.equals(SYNCHRONIZATION_DELAY_P)) {
0217: long new_nb = props.getLong(name, sync_delay);
0218: if (new_nb <= 0) {
0219: return false;
0220: }
0221: // not that we won't do too much if it is reduced
0222: // the user will have to clean the cache...
0223: sync_delay = new_nb;
0224: return true;
0225: } else if (name.equals(GENERATION_COMPACT_DELAY_P)) {
0226: long new_nb = props.getLong(name, gencomp_delay);
0227: if (new_nb <= 0) {
0228: return false;
0229: }
0230: // not that we won't do too much if it is reduced
0231: // the user will have to clean the cache...
0232: gencomp_delay = new_nb;
0233: return true;
0234: }
0235: // nothing changed, everything is ok!
0236: return true;
0237: }
0238:
0239: public StoreState getState() {
0240: return state;
0241: }
0242:
0243: /**
0244: * return the cache sweeper used by the cache
0245: * @return an instance of CacheSweeper
0246: */
0247: public CacheSweeper getSweeper() {
0248: return filter.sweeper;
0249: }
0250:
0251: /**
0252: * return the serializer used by the cache
0253: * @return an instance of Serializer
0254: */
0255: public CacheSerializer getSerializer() {
0256: return filter.serializer;
0257: }
0258:
0259: /**
0260: * return the cache validator used by the cache
0261: * @return an instance of CacheValidator
0262: */
0263: public CacheValidator getValidator() {
0264: return filter.validator;
0265: }
0266:
0267: /**
0268: * Get the next generation (in LRU order).
0269: * @param gen the current generation
0270: * @return a generation
0271: */
0272: public CacheGeneration getNextGeneration(CacheGeneration gen) {
0273: if (generations != null) {
0274: return (CacheGeneration) generations.getNext(gen);
0275: }
0276: return null;
0277: }
0278:
0279: /**
0280: * Get the previous generation (in LRU order).
0281: * @param gen the current generation
0282: * @return a generation
0283: */
0284: public CacheGeneration getPrevGeneration(CacheGeneration gen) {
0285: if (generations != null) {
0286: return (CacheGeneration) generations.getPrev(gen);
0287: }
0288: return null;
0289: }
0290:
0291: /**
0292: * Get the last generation, ie the Most recently used generation
0293: * @return a CacheGeneration, the most recently used one
0294: */
0295: public CacheGeneration getMRUGeneration() {
0296: return (CacheGeneration) generations.getHead();
0297: }
0298:
0299: /**
0300: * Get the fill ratio of the last generation (the most recently used)
0301: */
0302: public float getMRUGenerationRatio() {
0303: CacheGeneration last = (CacheGeneration) generations.getHead();
0304: return (last == null) ? (float) 0.0 : last.getFillRatio();
0305: }
0306:
0307: /**
0308: * Get the LRU generation, ie the Least recently used generation
0309: * @return a CacheGeneration, the least recently used one
0310: */
0311: public CacheGeneration getLRUGeneration() {
0312: return (CacheGeneration) generations.getTail();
0313: }
0314:
0315: /**
0316: * Get the oldest loaded generation
0317: * @return a generation
0318: */
0319: public CacheGeneration getLRULoadedGeneration() {
0320: CacheGeneration gen = (CacheGeneration) generations.getTail();
0321: while ((gen != null) && (!gen.isLoaded())) {
0322: gen = (CacheGeneration) generations.getPrev(gen);
0323: }
0324: return gen;
0325: }
0326:
0327: /**
0328: * Get the synchronization delay between to sync, in milliseconds
0329: * @return a long, the number of milliseconds
0330: */
0331: public long getSyncDelay() {
0332: return sync_delay;
0333: }
0334:
0335: /**
0336: * Get the delay between two attempts to compact the generations.
0337: * @return a long, the number of milliseconds
0338: */
0339: public long getCompactGenerationDelay() {
0340: return gencomp_delay;
0341: }
0342:
0343: /**
0344: * Create and add a new Generation. WARNING, this method is not
0345: * synchronized.
0346: * @return the newly created generation or null if the max number
0347: * of generations is reached or it the current size of the cache
0348: * (in memory) is too big to accept a new generation.
0349: * @exception InvalidCacheException if the cache is corrupted
0350: */
0351: protected CacheGeneration addNewGeneration()
0352: throws InvalidCacheException {
0353: long byteleft = bytelimit - state.getByteCount();
0354: if ((state.getNbGeneration() < max_generation)
0355: && (byteleft > generationlimit)) {
0356: CacheGeneration newgen = new CacheGeneration(this ,
0357: generationlimit);
0358: setGenerationFile(newgen);
0359: state.notifyGenerationCreated(newgen);
0360: generations.toHead(newgen);
0361: return newgen;
0362: } else {
0363: return null;
0364: }
0365: }
0366:
0367: /**
0368: * Load the given generation and unload the LRU loaded
0369: * generation if necessary.
0370: * @param cg the generation to load
0371: * @exception InvalidCacheException if the cache is corrupted
0372: */
0373: protected CacheGeneration loadGeneration(CacheGeneration cg)
0374: throws InvalidCacheException {
0375: // load this generation and unload the LRU loaded
0376: // generation
0377: CacheGeneration cglru = getLRULoadedGeneration();
0378: synchronized (this ) {
0379: // FIRST, unload the LRU loaded generation
0380: if (getCachedByteFree() < generationlimit) {
0381: if (cglru != null)
0382: unloadGeneration(cglru);
0383: }
0384: return _loadGeneration(cg);
0385: }
0386: }
0387:
0388: /**
0389: * Load a "unloaded" generation
0390: * @return the loaded generation
0391: * @exception InvalidCacheException if the cache is corrupted
0392: */
0393: private CacheGeneration _loadGeneration(CacheGeneration cg)
0394: throws InvalidCacheException {
0395: try {
0396: Reader reader = new BufferedReader(new FileReader(cg
0397: .getGenerationFile()));
0398: cg = getSerializer().readGeneration(cg, reader);
0399: state.notifyGenerationLoaded(cg);
0400: return cg;
0401: } catch (FileNotFoundException ex) {
0402: String msg = "Generation file does not exists: "
0403: + cg.getGenerationFile().getAbsolutePath();
0404: throw new InvalidCacheException(msg);
0405: } catch (IOException ex) {
0406: String msg = "IOError reading "
0407: + cg.getGenerationFile().getAbsolutePath();
0408: throw new InvalidCacheException(msg);
0409: }
0410: }
0411:
0412: /**
0413: * UnLoad a "loaded" generation. WARNING: not synchronized.
0414: * @param the loaded generation
0415: * @exception InvalidCacheException if the cache is corrupted
0416: */
0417: protected void unloadGeneration(CacheGeneration cg)
0418: throws InvalidCacheException {
0419: try {
0420: Writer writer = new BufferedWriter(new FileWriter(cg
0421: .getGenerationFile()));
0422: getSerializer().writeGeneration(cg, writer);
0423: state.notifyGenerationUnloaded(cg);
0424: } catch (FileNotFoundException ex) {
0425: String msg = "Generation file does not exists: "
0426: + cg.getGenerationFile().getAbsolutePath();
0427: throw new InvalidCacheException(msg);
0428: } catch (IOException ex) {
0429: String msg = "IOError writing on "
0430: + cg.getGenerationFile().getAbsolutePath();
0431: throw new InvalidCacheException(msg);
0432: }
0433: }
0434:
0435: /**
0436: * Save a generation.
0437: * @param the generation to be saved
0438: * @exception InvalidCacheException if the cache is corrupted
0439: */
0440: protected void saveGeneration(CacheGeneration cg)
0441: throws InvalidCacheException {
0442: if (cg.isLoaded() && (!cg.isSaved())) {
0443: try {
0444: Writer writer = new BufferedWriter(new FileWriter(cg
0445: .getGenerationFile()));
0446: getSerializer().writeGeneration(cg, writer);
0447: cg.setSaved(true);
0448: } catch (FileNotFoundException ex) {
0449: String msg = "Generation file does not exists: "
0450: + cg.getGenerationFile().getAbsolutePath();
0451: throw new InvalidCacheException(msg);
0452: } catch (IOException ex) {
0453: String msg = "IOError writing on "
0454: + cg.getGenerationFile().getAbsolutePath();
0455: throw new InvalidCacheException(msg);
0456: }
0457: }
0458: }
0459:
0460: /**
0461: * Compute the generation identifier from filename. Filenames
0462: * looks like gen-001
0463: * @param file the generation file
0464: * @return the generation number
0465: */
0466: private int getGenerationId(File file) {
0467: String name = file.getName();
0468: int idx = name.indexOf('-') + 1;
0469: String num = name.substring(idx);
0470: try {
0471: return Integer.parseInt(num);
0472: } catch (NumberFormatException ex) {
0473: return -1;
0474: }
0475: }
0476:
0477: /**
0478: * Load the generations. Only a subset of the generations will
0479: * actually be loaded, some generations will only have a description
0480: * (URLs, size) in memory.
0481: * This method load some generations (until cr_limit is reached) and the
0482: * remaining generations (if any) will only be a set of
0483: * CachedResourceDescription.
0484: */
0485: protected synchronized void loadGenerations()
0486: throws InvalidCacheException {
0487: generations = new SyncLRUList();
0488: File genfiles[] = getGenerationFiles();
0489: if (genfiles == null) {
0490: throw new InvalidCacheException("No generation files!");
0491: }
0492: int len = genfiles.length;
0493: Vector sorted = new Vector(len);
0494:
0495: //sort generation files
0496: for (int i = 0; i < len; i++) {
0497: Sorter.orderedFileInsert(genfiles[i], sorted);
0498: }
0499: sorted.copyInto(genfiles);
0500: //load generations
0501: for (int i = 0; i < len; i++) {
0502: int id = -1;
0503: try {
0504: File genfile = genfiles[i];
0505: id = getGenerationId(genfile);
0506: if (id != -1) { //valid generation filename
0507: CacheGeneration gen = new CacheGeneration(this ,
0508: generationlimit);
0509: Reader reader = new BufferedReader(new FileReader(
0510: genfile));
0511: if (state.getCrCount() < cr_limit) {
0512: getSerializer().readGeneration(gen, reader);
0513: } else {
0514: getSerializer().readDescription(gen, reader);
0515: }
0516: gen.setId(id);
0517: gen.setGenerationFile(genfile);
0518: generations.toHead(gen);
0519: } else if (debug) {
0520: System.err.println("Invalid generation filename : "
0521: + genfiles[i].getName());
0522: }
0523: } catch (FileNotFoundException ex) {
0524: if (debug) {
0525: String msg = "File not found generation[" + id
0526: + "]";
0527: System.err.println(msg + " " + ex.getMessage());
0528: }
0529: } catch (IOException ioex) {
0530: if (debug) {
0531: String msg = "Error loading generation [" + id
0532: + "]";
0533: System.err.println(msg + " " + ioex.getMessage());
0534: }
0535: }
0536: }
0537: }
0538:
0539: /**
0540: * Load the store state
0541: */
0542: protected void loadState() {
0543: try {
0544: Reader reader = new BufferedReader(
0545: new FileReader(statefile));
0546: state = (StoreState) getSerializer().read(reader);
0547: } catch (IOException ex) {
0548: if (debug) {
0549: System.err.println("Can't load StoreState : "
0550: + ex.getMessage());
0551: }
0552: state = new StoreState();
0553: }
0554: }
0555:
0556: /**
0557: * Save the current state
0558: */
0559: protected void saveState() {
0560: state.sync();
0561: try {
0562: Writer writer = new BufferedWriter(
0563: new FileWriter(statefile));
0564: getSerializer().write(state, writer);
0565: } catch (IOException ex) {
0566: if (debug) {
0567: System.err.println("Can't load StoreState : "
0568: + ex.getMessage());
0569: }
0570: }
0571: }
0572:
0573: /**
0574: * get the number of bytes the garbage collector needs to collect
0575: * to keep the cache in good state, it will only move the resource
0576: * to the delete list, another check has to be done to save physical space
0577: * @return a long, the number of bytes to save
0578: */
0579: public long getRequiredByteNumber() {
0580: return (long) (state.getByteCount() - (bytelimit * gc_kept_ratio));
0581: }
0582:
0583: /**
0584: * get the cached size of this cache
0585: * @return a long, the number of bytes cached
0586: */
0587: public synchronized long getCachedByteCount() {
0588: return state.getByteCount();
0589: }
0590:
0591: /**
0592: * get the number of bytes used phisycally by this cache
0593: * @return a long, the number of bytes cached
0594: */
0595: public synchronized long getStoredByteCount() {
0596: return state.getStoreCount();
0597: }
0598:
0599: /**
0600: * get the number of bytes available for the cache memory
0601: * @return a long, the number of bytes free
0602: */
0603: public synchronized long getCachedByteFree() {
0604: return (bytelimit - state.getByteCount());
0605: }
0606:
0607: /**
0608: * synchronize the internal database with the storage
0609: */
0610: public synchronized void sync() {
0611: CacheGeneration cg = (CacheGeneration) generations.getHead();
0612: if (cg == null) {
0613: // nothing to save
0614: return;
0615: }
0616: do {
0617: try {
0618: saveGeneration(cg);
0619: } catch (InvalidCacheException ex) {
0620: if (debug) {
0621: System.err.println("Unable to save generation ["
0622: + cg.getId() + "] " + ex.getMessage());
0623: }
0624: }
0625: cg = getNextGeneration(cg);
0626: } while (cg != null);
0627: saveState();
0628: }
0629:
0630: /**
0631: * Remove the given CachedResource from the given CacheGeneration.
0632: * WARNING: not synchronized
0633: * @exception NoSuchResourceException if this resource was not in this
0634: * generation
0635: */
0636: protected void removeResource(CacheGeneration cg, CachedResource cr)
0637: throws NoSuchResourceException {
0638: cg.removeResource(cr);
0639: state.notifyResourceRemoved(cr);
0640: }
0641:
0642: /**
0643: * Get a cached resource relative to the given URL. WARNING: the
0644: * CachedResource returned is no more in the CacheStore.
0645: * @param url the URL of the CachedResource
0646: * @return a CachedResource or null
0647: * @see #storeCachedResource
0648: * @exception InvalidCacheException if the cache is corrupted
0649: */
0650: public CachedResource getCachedResource(String url)
0651: throws InvalidCacheException {
0652: CacheGeneration cg = (CacheGeneration) generations.getHead();
0653: if (cg == null)
0654: return null;
0655: do {
0656: CachedResource cr = null;
0657: if (cg.isLoaded()) {
0658: // The generation is already loaded
0659: cr = cg.lookupResource(url);
0660: } else if (cg.containsResource(url)) {
0661: // load this generation and unload the LRU loaded
0662: // generation if necessary
0663: loadGeneration(cg);
0664: cr = cg.lookupResource(url);
0665: }
0666: // found something?
0667: if (cr != null) {
0668: try {
0669: synchronized (this ) {
0670: removeResource(cg, cr);
0671: }
0672: } catch (NoSuchResourceException ex) {
0673: //should not happen
0674: }
0675: return cr;
0676: }
0677: // try with next generation
0678: cg = getNextGeneration(cg);
0679: } while (cg != null);
0680: return null;
0681: }
0682:
0683: /**
0684: * extract a cached resource from the store
0685: * @param the cached resource to be extracted
0686: * @return the extracted cached resource
0687: * @exception InvalidCacheException if the cache is corrupted
0688: */
0689: public CachedResource getCachedResource(CachedResource cr)
0690: throws InvalidCacheException {
0691: CacheGeneration cg = cr.generation;
0692: if (cg != null) {
0693: try {
0694: synchronized (this ) {
0695: removeResource(cg, cr);
0696: }
0697: } catch (NoSuchResourceException ex) {
0698: //should not happen
0699: }
0700: return cr;
0701: }
0702: // FIXME do the lookup, but we shouldn't end up here anyway
0703: return cr;
0704: }
0705:
0706: /**
0707: * Resize the generation in order to be able to store the given
0708: * Resource.
0709: * @param cg the generation to resize
0710: * @param cr the CachedResource to store.
0711: */
0712: protected synchronized void resizeGeneration(CacheGeneration cg,
0713: CachedResource cr) {
0714: if (debug) {
0715: System.out.println("Resizing generation " + cg.getId());
0716: }
0717: long real_size = Math.max(cr.getContentLength(), cr
0718: .getCurrentLength());
0719: if (real_size > generationlimit) {
0720: cg.setByteLimit(real_size);
0721: generationlimit = (bytelimit - real_size)
0722: / (max_generation - 1);
0723: } else if (debug) {
0724: System.out.println("Asked for a not necessary resize!");
0725: }
0726: }
0727:
0728: /**
0729: * Get a cached resource relative to the given URL. WARNING: the
0730: * CachedResource returned is still in the cache store!
0731: * @param url the URL of the CachedResource
0732: * @return a CachedResource or null
0733: * @see #storeCachedResource
0734: * @exception InvalidCacheException if the cache is corrupted
0735: */
0736: public CachedResource getCachedResourceReference(String url)
0737: throws InvalidCacheException {
0738: CacheGeneration cg = (CacheGeneration) generations.getHead();
0739: if (cg == null)
0740: return null;
0741: do {
0742: CachedResource cr = null;
0743: if (cg.isLoaded()) {
0744: // The generation is already loaded
0745: cr = cg.lookupResource(url);
0746: } else if (cg.containsResource(url)) {
0747: // load this generation and unload the LRU loaded
0748: // generation if necessary
0749: loadGeneration(cg);
0750: cr = cg.lookupResource(url);
0751: }
0752: // found something?
0753: if (cr != null) {
0754: return cr;
0755: }
0756: // try with next generation
0757: cg = getNextGeneration(cg);
0758: } while (cg != null);
0759: return null;
0760: }
0761:
0762: /**
0763: * update this cached resource from generation x ot the latest
0764: * @param the cached resource to be updated
0765: * @return the updated
0766: * @exception InvalidCacheException if the cache is corrupted
0767: */
0768: public CachedResource updateResourceGeneration(CachedResource cr)
0769: throws InvalidCacheException {
0770: CacheGeneration cg = (CacheGeneration) generations.getHead();
0771: if (cg != cr.generation) {
0772: try {
0773: synchronized (this ) {
0774: removeResource(cr.generation, cr);
0775: }
0776: } catch (NoSuchResourceException ex) {
0777: //should not happen
0778: }
0779: storeCachedResource(cr, cr.getCurrentLength());
0780: }
0781: return cr;
0782: }
0783:
0784: /**
0785: * Store a newly created (or updated) CachedResource.
0786: * @param cr the CachedResource to store
0787: * @exception InvalidCacheException if the cache is corrupted
0788: * @return <code>true</code> if the resource has been cached
0789: */
0790: public boolean storeCachedResource(CachedResource cr)
0791: throws InvalidCacheException {
0792: return storeCachedResource(cr, 0);
0793: }
0794:
0795: /**
0796: * Store a newly created (or updated) CachedResource.
0797: * @param cr the CachedResource to store
0798: * @return <code>true</code> if the resource has been cached
0799: * @exception InvalidCacheException if the cache is corrupted
0800: */
0801: public boolean storeCachedResource(CachedResource cr, long oldsize)
0802: throws InvalidCacheException {
0803: CacheGeneration cg = (CacheGeneration) generations.getHead();
0804: long size = cr.getCurrentLength();
0805: long maxsize = (long) (((double) bytelimit) * threshold);
0806: // size > threshold, can't cache it...
0807: if (size > maxsize) {
0808: return false;
0809: }
0810: if (cg != null) {
0811: // firt try to add in last generation
0812: if (cg.addResource(cr, size, oldsize)) {
0813: // success
0814: return true;
0815: }
0816: // create a new generation
0817: CacheGeneration new_cg = addNewGeneration();
0818: if (new_cg != null) {
0819: // generation created
0820: if (!new_cg.addResource(cr, size, oldsize)) {
0821: // resize generation
0822: resizeGeneration(new_cg, cr);
0823: // try again
0824: if (!new_cg.addResource(cr, size, oldsize)) {
0825: String msg = "Unable to add a cachedResource in a "
0826: + "resized generation!!";
0827: throw new InvalidCacheException(msg);
0828: }
0829: }
0830: } else {
0831: // failed! unload a generation on disk and create a new
0832: // generation if the storelimit and the max number of
0833: // generations have not been reached.
0834: if ((state.getStoreCount() < (storelimit - generationlimit))
0835: && (state.getNbGeneration() < max_generation)) {
0836: // unload the oldest loaded generation until
0837: // we can create a new generation
0838: while (new_cg == null) {
0839: CacheGeneration older_cg = getLRULoadedGeneration();
0840: if (older_cg == null) {
0841: String msg = "No Generation Loaded but store limit reached";
0842: throw new InvalidCacheException(msg);
0843: }
0844: synchronized (this ) {
0845: unloadGeneration(older_cg);
0846: // Create a new generation
0847: new_cg = addNewGeneration();
0848: }
0849: }
0850: } else {
0851: // we can't create a new generation so empty the oldest
0852: // generation
0853: CacheGeneration older_cg = null;
0854: synchronized (this ) {
0855: older_cg = getLRUGeneration();
0856: if (older_cg == null) {
0857: String msg = "No Generation Loaded"
0858: + " but store limit reached";
0859: throw new InvalidCacheException(msg);
0860: }
0861: older_cg.emptyGeneration();
0862: long cg_limit = older_cg.getByteLimit();
0863: generationlimit = ((generationlimit * (max_generation - 1)) + cg_limit)
0864: / max_generation;
0865: generations.toHead(older_cg);
0866: setGenerationFile(older_cg);
0867: }
0868: getSweeper().collectStored(older_cg);
0869: new_cg = older_cg;
0870: }
0871: // finally add the resource in the new generation
0872: if (!new_cg.addResource(cr, size, oldsize)) {
0873: // resize generation
0874: resizeGeneration(new_cg, cr);
0875: // try again
0876: if (!new_cg.addResource(cr, size, oldsize)) {
0877: String msg = "Unable to add a cachedResource in a "
0878: + "resized generation!!";
0879: throw new InvalidCacheException(msg);
0880: }
0881: return true;
0882: }
0883: }
0884: } else {
0885: CacheGeneration new_cg = null;
0886: synchronized (this ) {
0887: new_cg = addNewGeneration();
0888: }
0889: if (new_cg == null) {
0890: String msg = "Unable create the first generation!!";
0891: throw new InvalidCacheException(msg);
0892: }
0893: if (!new_cg.addResource(cr, size, oldsize)) {
0894: // resize generation
0895: resizeGeneration(new_cg, cr);
0896: // try again
0897: if (!new_cg.addResource(cr, size, oldsize)) {
0898: String msg = "Unable to add a cachedResource in a "
0899: + "resized generation!!";
0900: throw new InvalidCacheException(msg);
0901: }
0902: return true;
0903: }
0904: }
0905: return false;
0906: }
0907:
0908: /**
0909: * Get a beautiful file number, eg :
0910: * <ul>
0911: * <li>2 => 002 if size == 3
0912: * <li>50 => 0050 if size == 4
0913: * <li>5000 => 5000 if size < 4
0914: * @param nb the file number
0915: * @param size the 'size' of the number
0916: * @return a String
0917: */
0918: private String getFileNumber(int nb, int size) {
0919: //compute the number of '0' to add
0920: int cpt = 0;
0921: int nb2 = nb;
0922: if (nb2 == 0) {
0923: cpt = 1;
0924: } else {
0925: while (nb2 > 0) {
0926: nb2 = nb2 / 10;
0927: cpt++;
0928: }
0929: }
0930: //number of '0' to add = size - cpt;
0931: int zero2add = size - cpt;
0932: StringBuffer buffer = new StringBuffer();
0933: for (int i = 0; i < zero2add; i++) {
0934: buffer.append("0");
0935: }
0936: buffer.append(Integer.toString(nb));
0937: return buffer.toString();
0938:
0939: }
0940:
0941: private File[] getGenerationFiles() throws InvalidCacheException {
0942: FilenameFilter filter = new FilenameFilter() {
0943: /**
0944: * Accept file like gen-xxxx (xxxx is a number)
0945: */
0946: public boolean accept(File dir, String name) {
0947: return (name.startsWith(GENERATION_FILENAME));
0948: }
0949: };
0950: if (cache_dir == null) {
0951: throw new InvalidCacheException("No Cache Directory!!");
0952: } else if (!cache_dir.exists()) {
0953: cache_dir.mkdirs();
0954: }
0955: return cache_dir.listFiles(filter);
0956: }
0957:
0958: /**
0959: * allocate a new name for the next generation file.
0960: * @return a File, used to dump the generation
0961: * @exception InvalidCacheException if the cache is corrupted
0962: */
0963: private synchronized void setGenerationFile(CacheGeneration gen)
0964: throws InvalidCacheException {
0965: int current_generation = state.incrCurrentGeneration();
0966: File file = new File(cache_dir, GENERATION_FILENAME
0967: + getFileNumber(current_generation, 4));
0968: if (debug) {
0969: System.err.println(file);
0970: }
0971: gen.setGenerationFile(file);
0972: gen.setId(current_generation);
0973: }
0974:
0975: /**
0976: * allocate a new name for the next cached resource. This method
0977: * create some directories if needed.
0978: * @return a File, used to dump the entry
0979: */
0980: protected File getNewEntryFile() {
0981: int curnum;
0982: synchronized (this ) {
0983: curnum = state.incrEntryNum();
0984: }
0985: int filenum = curnum / nb_dir;
0986: int dirnum = curnum % nb_dir;
0987: File dir = dirs[dirnum];
0988: return new File(dir, getFileNumber(filenum, 4));
0989: }
0990:
0991: public String toString() {
0992: StringBuffer buffer = new StringBuffer();
0993: buffer.append(">>> CacheStore [").append(
0994: cache_dir.getAbsolutePath()).append("] <<<");
0995: buffer.append("\n Store limit : ").append(storelimit);
0996: buffer.append("\n Byte limit : ").append(bytelimit);
0997: buffer.append("\n CR limit : ").append(cr_limit);
0998: buffer.append(state);
0999: return buffer.toString();
1000: }
1001:
1002: /**
1003: * Check the subdirectories. Create them if necessary.
1004: */
1005: protected void checkDirs() {
1006: dirs = new File[nb_dir];
1007: for (int i = 0; i < nb_dir; i++) {
1008: dirs[i] = new File(cache_dir, getFileNumber(i, 3));
1009: if (!dirs[i].exists()) {
1010: dirs[i].mkdirs();
1011: }
1012: }
1013: }
1014:
1015: /**
1016: * Clean the Cache directory, remove unused files.
1017: * @return the number of files deleted.
1018: */
1019: protected int cleanCacheDir() {
1020: // first, store all the known files
1021: Hashtable entryfiles = new Hashtable();
1022: CacheGeneration cg = (CacheGeneration) generations.getHead();
1023: while (cg != null) {
1024: Enumeration fenum = cg.getFiles();
1025: while (fenum.hasMoreElements()) {
1026: entryfiles.put(fenum.nextElement(), Boolean.TRUE);
1027: }
1028: cg = getNextGeneration(cg);
1029: }
1030: // check all files found in dirs
1031: int cpt = 0;
1032: for (int i = 0; i < dirs.length; i++) {
1033: File files[] = dirs[i].listFiles();
1034: if (files != null) {
1035: for (int j = 0; j < files.length; j++) {
1036: if (entryfiles.get(files[j]) == null) {
1037: if (files[j].delete()) {
1038: cpt++;
1039: if (debug) {
1040: System.out.println(files[j]
1041: + " not used, removed");
1042: }
1043: }
1044: }
1045: }
1046: }
1047: }
1048: return cpt;
1049: }
1050:
1051: /**
1052: * update the mode of the sweeper according to the state of the
1053: * cache store,
1054: */
1055: protected void updateSweeperPriority() {
1056: long byteCount = state.getByteCount();
1057: long storeCount = state.getStoreCount();
1058: long crCount = state.getCrCount();
1059: CacheSweeper sweeper = getSweeper();
1060: // over the limit of cached resource usage, stop and clean
1061: if (byteCount > bytelimit) {
1062: sweeper
1063: .setState(CacheSweeper.STATE_FORCE_CLEAN_GENERATIONS);
1064: } else if (byteCount > (long) ((float) bytelimit * gc_kept_ratio)) {
1065: // near the limit, start loosely to remove resources
1066: sweeper.setState(CacheSweeper.STATE_CLEAN_GENERATIONS);
1067: } else if (storeCount > storelimit) {
1068: // to many on the disk, remove then asap
1069: sweeper.setState(CacheSweeper.STATE_FORCE_CLEAN_STORED);
1070: } else {
1071: // normal operation, remove in a smooth way the deleted resources
1072: sweeper.setState(CacheSweeper.STATE_CLEAN_STORED);
1073: }
1074: }
1075:
1076: /**
1077: * Compact our generations
1078: * The algorithm is the following,
1079: * If the number of generation is the maximum number allowed,
1080: * then a check is done from the the generation after the MRU one
1081: * and if the sum of two generation can fit into one, it is done, and
1082: * the generation is removed from the list
1083: */
1084: protected void compactGenerations() {
1085: if (debug)
1086: System.out.println("*** trying to compact generations");
1087: // limit not reached? exit ASAP
1088: if (state.getNbGeneration() < max_generation)
1089: return;
1090: if (debug)
1091: System.out
1092: .println("*** compact: Max reached, compacting...");
1093: CacheGeneration gen = getMRUGeneration();
1094: CacheGeneration nextGen;
1095: gen = getNextGeneration(gen);
1096: while (gen != null) {
1097: if (debug) {
1098: System.out
1099: .println("*** compact: working on generation "
1100: + gen.getId());
1101: }
1102: nextGen = getNextGeneration(gen);
1103: // last one, exit
1104: if (nextGen == null) {
1105: break;
1106: }
1107: if ((gen.getCachedByteCount() + nextGen
1108: .getCachedByteCount()) < gen.getByteLimit()) {
1109: // do the dirty work now...
1110: synchronized (generations) {
1111: if (debug) {
1112: System.out.println("*** compact: merging ("
1113: + gen.getId() + ") and ("
1114: + nextGen.getId() + ")");
1115: }
1116: generations.remove(nextGen);
1117: gen.copyInto(nextGen);
1118: nextGen.deleteGenerationFile();
1119: state.decrGenerationNum();
1120: }
1121: }
1122: gen = getNextGeneration(gen);
1123: }
1124: }
1125:
1126: /**
1127: * used for debugging, display some internal information about the state
1128: * of the cache
1129: */
1130: protected synchronized void checkState() {
1131: long byteCount = state.getByteCount();
1132: long storeCount = state.getStoreCount();
1133: long crCount = state.getCrCount();
1134: int entryNum = state.getEntryNum();
1135: int nbGeneration = state.getNbGeneration();
1136: int currentGeneration = state.getCurrentGeneration();
1137: double ratio = (((double) byteCount / bytelimit) * 100);
1138: String rt = String.valueOf(ratio);
1139: if (rt.length() > 5)
1140: rt = rt.substring(0, 5);
1141: System.out.println(" Ratio (BC/BL)*100 : " + rt + " %");
1142: System.out.println(">>> Generations <<<");
1143: CacheGeneration cg = getMRUGeneration();
1144: System.out.println(" Id | Loaded | CR cnt | BT lim | "
1145: + "BT cnt | ST cnt | ratio");
1146: System.out
1147: .println(" ---------------------------------------------"
1148: + "---------------------------");
1149: long bc = 0;
1150: long sc = 0;
1151: while (cg != null) {
1152: long gbc = cg.getCachedByteCount();
1153: long gsc = cg.getStoredByteCount();
1154: long gbl = cg.getByteLimit();
1155: bc += gbc;
1156: sc += gsc;
1157: double percent = (((double) gbc / gbl) * 100);
1158: String pc = String.valueOf(percent);
1159: if (pc.length() > 5)
1160: pc = pc.substring(0, 5);
1161: System.out.print(" " + getFileNumber(cg.getId(), 2));
1162: System.out.print(" | " + cg.isLoaded() + " ");
1163: System.out.print(" | "
1164: + getFileNumber((int) cg.getCRCount(), 7));
1165: System.out.print(" | " + getFileNumber((int) gbl, 7));
1166: System.out.print(" | " + getFileNumber((int) gbc, 7));
1167: System.out.print(" | " + getFileNumber((int) gsc, 7));
1168: System.out.println(" | " + pc + " %");
1169: cg = getNextGeneration(cg);
1170: }
1171: System.out.println(">>> Check State <<<");
1172: System.out.println(" Byte Count <= Byte Limit : "
1173: + (byteCount <= bytelimit));
1174: System.out.println(" Store Count <= Store Limit : "
1175: + (storeCount <= storelimit));
1176: System.out.println(" CR Count <= CR Limit : "
1177: + (crCount <= cr_limit));
1178: System.out.println(" Byte Count <= Store Count : "
1179: + (byteCount <= storeCount));
1180: System.out.println(" CR Count <= Entry Num : "
1181: + (crCount <= entryNum));
1182: System.out.println(" Current gen >= Number of gen : "
1183: + (currentGeneration >= nbGeneration));
1184: System.out.println(" Generations SC == Store SC : "
1185: + (sc == storeCount));
1186: System.out.println(" Generations BC == Store BC : "
1187: + (bc == byteCount));
1188: }
1189:
1190: /**
1191: * initialize this CacheStore, and get some infos from the parent,
1192: * aka the cache filter
1193: * @param filter a CacheFilter, our parent
1194: * @exception InvalidCacheException if the cache not initialized
1195: */
1196: public void initialize(CacheFilter filter)
1197: throws InvalidCacheException {
1198: this .filter = filter;
1199: props = filter.props;
1200: cache_dir = props.getFile(CACHE_DIRECTORY_P, null);
1201: if (cache_dir == null) {
1202: cache_dir = new File(System.getProperty("user.dir"));
1203: cache_dir = new File(cache_dir, ".web-cache");
1204: }
1205: // the capacity of the cache, defaulting to 20Mo
1206: bytelimit = props.getLong(CacheFilter.CACHE_SIZE_P, 20971520);
1207: // the store capacity of the cache, defaulting to 20Mo + 10%
1208: storelimit = props.getLong(STORE_SIZE_P, 23068672);
1209: // the delay between two store sync
1210: sync_delay = props.getLong(SYNCHRONIZATION_DELAY_P, 60000);
1211: // the delay between two attempts to compact the database
1212: gencomp_delay = props
1213: .getLong(GENERATION_COMPACT_DELAY_P, 60000);
1214: // the garbage collection flag
1215: garbageCollectionEnabled = props.getBoolean(
1216: GARBAGE_COLLECTION_ENABLED_P, true);
1217: // the threshold (as in an LRU-threshold cache policy
1218: threshold = props.getDouble(FILE_SIZE_RATIO_P, 0.1);
1219: // the gc limit
1220: gc_kept_ratio = props.getDouble(GARBAGE_COLLECTION_THRESHOLD_P,
1221: 0.80);
1222: // the maximal number of generations
1223: max_generation = props.getInteger(MAX_GENERATIONS_P, 10);
1224: // the generation limit size
1225: generationlimit = bytelimit / max_generation;
1226: // debug flag
1227: debug = props.getBoolean(CacheFilter.DEBUG_P, false);
1228: //load
1229: cr_limit = props.getInteger(MAX_CACHED_RESOURCES_P, 50000);
1230: // load the store state
1231: this .statefile = new File(cache_dir, "state");
1232: loadState();
1233: // check the subdirectories
1234: checkDirs();
1235: // load the generations...
1236: System.out.println("Loading generations...");
1237: loadGenerations();
1238: // clean
1239: System.out.println("Cleaning cache directories...");
1240: int cleaned = cleanCacheDir();
1241: if (cleaned > 0) {
1242: System.out.println(cleaned + " unused files deleted.");
1243: }
1244: }
1245: }
|