0001: /*-
0002: * See the file LICENSE for redistribution information.
0003: *
0004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
0005: *
0006: * $Id: UtilizationProfile.java,v 1.52.2.10 2008/01/07 15:14:08 cwl Exp $
0007: */
0008:
0009: package com.sleepycat.je.cleaner;
0010:
0011: import java.io.File;
0012: import java.util.ArrayList;
0013: import java.util.Arrays;
0014: import java.util.Iterator;
0015: import java.util.List;
0016: import java.util.Set;
0017: import java.util.SortedMap;
0018: import java.util.SortedSet;
0019: import java.util.StringTokenizer;
0020: import java.util.TreeMap;
0021: import java.util.logging.Level;
0022:
0023: import com.sleepycat.je.DatabaseConfig;
0024: import com.sleepycat.je.DatabaseEntry;
0025: import com.sleepycat.je.DatabaseException;
0026: import com.sleepycat.je.DbInternal;
0027: import com.sleepycat.je.OperationStatus;
0028: import com.sleepycat.je.TransactionConfig;
0029: import com.sleepycat.je.config.EnvironmentParams;
0030: import com.sleepycat.je.dbi.CursorImpl;
0031: import com.sleepycat.je.dbi.DatabaseId;
0032: import com.sleepycat.je.dbi.DatabaseImpl;
0033: import com.sleepycat.je.dbi.DbConfigManager;
0034: import com.sleepycat.je.dbi.DbTree;
0035: import com.sleepycat.je.dbi.EnvConfigObserver;
0036: import com.sleepycat.je.dbi.EnvironmentImpl;
0037: import com.sleepycat.je.dbi.MemoryBudget;
0038: import com.sleepycat.je.dbi.CursorImpl.SearchMode;
0039: import com.sleepycat.je.log.FileManager;
0040: import com.sleepycat.je.log.entry.LNLogEntry;
0041: import com.sleepycat.je.tree.BIN;
0042: import com.sleepycat.je.tree.FileSummaryLN;
0043: import com.sleepycat.je.tree.Tree;
0044: import com.sleepycat.je.tree.TreeLocation;
0045: import com.sleepycat.je.txn.AutoTxn;
0046: import com.sleepycat.je.txn.BasicLocker;
0047: import com.sleepycat.je.txn.LockType;
0048: import com.sleepycat.je.txn.Locker;
0049: import com.sleepycat.je.utilint.DbLsn;
0050:
0051: /**
0052: * The UP tracks utilization summary information for all log files.
0053: *
0054: * <p>Unlike the UtilizationTracker, the UP is not accessed under the log write
0055: * latch and is instead synchronized on itself for protecting the cache. It is
0056: * not accessed during the primary data access path, except for when flushing
0057: * (writing) file summary LNs. This occurs in the following cases:
0058: * <ol>
0059: * <li>The summary information is flushed at the end of a checkpoint. This
0060: * allows tracking to occur in memory in between checkpoints, and replayed
0061: * during recovery.</li>
0062: * <li>When committing the truncateDatabase and removeDatabase operations, the
0063: * summary information is flushed because detail tracking for those operations
0064: * is not replayed during recovery</li>
0065: * <li>The evictor will ask the UtilizationTracker to flush the largest summary
0066: * if the memory taken by the tracker exeeds its budget.</li>
0067: * </ol>
0068: *
0069: * <p>The cache is populated by the RecoveryManager just before performing the
0070: * initial checkpoint. The UP must be open and populated in order to respond
0071: * to requests to flush summaries and to evict tracked detail, even if the
0072: * cleaner is disabled.</p>
0073: *
0074: * <p>WARNING: While synchronized on this object, eviction is not permitted.
0075: * If it were, this could cause deadlocks because the order of locking would be
0076: * the UP object and then the evictor. During normal eviction the order is to
0077: * first lock the evictor and then the UP, when evicting tracked detail.</p>
0078: *
0079: * <p>The methods in this class synchronize to protect the cached summary
0080: * information. Some methods also access the UP database. However, because
0081: * eviction must not occur while synchronized, UP database access is not
0082: * performed while synchronized except in one case: when inserting a new
0083: * summary record. In that case we disallow eviction during the database
0084: * operation.</p>
0085: */
0086: public class UtilizationProfile implements EnvConfigObserver {
0087:
0088: /*
0089: * Note that age is a distance between files not a number of files, that
0090: * is, deleted files are counted in the age.
0091: */
0092: private EnvironmentImpl env;
0093: private UtilizationTracker tracker;
0094: private DatabaseImpl fileSummaryDb; // stored fileNum -> FileSummary
0095: private SortedMap fileSummaryMap; // cached fileNum -> FileSummary
0096: private boolean cachePopulated;
0097: private boolean rmwFixEnabled;
0098: private long totalLogSize;
0099:
0100: /**
0101: * Minimum overall utilization threshold that triggers cleaning. Is
0102: * non-private for unit tests.
0103: */
0104: int minUtilization;
0105:
0106: /**
0107: * Minimum utilization threshold for an individual log file that triggers
0108: * cleaning. Is non-private for unit tests.
0109: */
0110: int minFileUtilization;
0111:
0112: /**
0113: * Minumum age to qualify for cleaning. If the first active LSN file is 5
0114: * and the mininum age is 2, file 4 won't qualify but file 3 will. Must be
0115: * greater than zero because we never clean the first active LSN file. Is
0116: * non-private for unit tests.
0117: */
0118: int minAge;
0119:
0120: /**
0121: * An array of pairs of file numbers, where each pair is a range of files
0122: * to be force cleaned. Index i is the from value and i+1 is the to value,
0123: * both inclusive.
0124: */
0125: private long[] forceCleanFiles;
0126:
0127: /**
0128: * Creates an empty UP.
0129: */
0130: public UtilizationProfile(EnvironmentImpl env,
0131: UtilizationTracker tracker) throws DatabaseException {
0132:
0133: this .env = env;
0134: this .tracker = tracker;
0135: fileSummaryMap = new TreeMap();
0136:
0137: rmwFixEnabled = env.getConfigManager().getBoolean(
0138: EnvironmentParams.CLEANER_RMW_FIX);
0139: parseForceCleanFiles(env.getConfigManager().get(
0140: EnvironmentParams.CLEANER_FORCE_CLEAN_FILES));
0141:
0142: /* Initialize mutable properties and register for notifications. */
0143: envConfigUpdate(env.getConfigManager());
0144: env.addConfigObserver(this );
0145: }
0146:
0147: /**
0148: * Process notifications of mutable property changes.
0149: */
0150: public void envConfigUpdate(DbConfigManager cm)
0151: throws DatabaseException {
0152:
0153: minAge = cm.getInt(EnvironmentParams.CLEANER_MIN_AGE);
0154: minUtilization = cm
0155: .getInt(EnvironmentParams.CLEANER_MIN_UTILIZATION);
0156: minFileUtilization = cm
0157: .getInt(EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION);
0158: }
0159:
0160: /**
0161: * @see EnvironmentParams#CLEANER_RMW_FIX
0162: * @see FileSummaryLN#postFetchInit
0163: */
0164: public boolean isRMWFixEnabled() {
0165: return rmwFixEnabled;
0166: }
0167:
0168: /**
0169: * Returns the number of files in the profile.
0170: */
0171: synchronized int getNumberOfFiles() throws DatabaseException {
0172:
0173: assert cachePopulated;
0174:
0175: return fileSummaryMap.size();
0176: }
0177:
0178: /**
0179: * Returns an approximation of the total log size. Used for stats.
0180: */
0181: long getTotalLogSize() {
0182:
0183: /* Start with the size known to the profile. */
0184: long size = totalLogSize;
0185:
0186: /*
0187: * Add sizes that are known to the tracker but are not yet in the
0188: * profile. The FileSummary.totalSize field is the delta for new
0189: * log entries added. Typically the last log file is only one that
0190: * will have a delta, but previous files may also not have been added
0191: * to the profile yet.
0192: */
0193: TrackedFileSummary[] trackedFiles = tracker.getTrackedFiles();
0194: for (int i = 0; i < trackedFiles.length; i += 1) {
0195: size += trackedFiles[i].totalSize;
0196: }
0197:
0198: return size;
0199: }
0200:
0201: /**
0202: * Returns the cheapest file to clean from the given list of files. This
0203: * method is used to select the first file to be cleaned in the batch of
0204: * to-be-cleaned files.
0205: */
0206: synchronized Long getCheapestFileToClean(SortedSet files)
0207: throws DatabaseException {
0208:
0209: if (files.size() == 1) {
0210: return (Long) files.first();
0211: }
0212:
0213: assert cachePopulated;
0214:
0215: Long bestFile = null;
0216: int bestCost = Integer.MAX_VALUE;
0217:
0218: for (Iterator iter = files.iterator(); iter.hasNext();) {
0219: Long file = (Long) iter.next();
0220:
0221: /*
0222: * Ignore files in the given set that are not in the profile. This
0223: * can occur with multiple cleaner threads because we don't hold a
0224: * synchronized lock during the entire execution of the
0225: * FileSelector.selectFileForCleaning method. [#14431]
0226: */
0227: if (!fileSummaryMap.containsKey(file)) {
0228: continue;
0229: }
0230:
0231: /* Calculate this file's cost to clean. */
0232: FileSummary summary = getFileSummary(file);
0233: int this Cost = summary.getNonObsoleteCount();
0234:
0235: /* Select this file if it has the lowest cost so far. */
0236: if (bestFile == null || this Cost < bestCost) {
0237: bestFile = file;
0238: bestCost = this Cost;
0239: }
0240: }
0241:
0242: return bestFile;
0243: }
0244:
0245: /**
0246: * Returns the best file that qualifies for cleaning, or null if no file
0247: * qualifies.
0248: *
0249: * @param fileSelector is used to determine valid cleaning candidates.
0250: *
0251: * @param forceCleaning is true to always select a file, even if its
0252: * utilization is above the minimum utilization threshold.
0253: *
0254: * @param lowUtilizationFiles is a returned set of files that are below the
0255: * minimum utilization threshold.
0256: */
0257: synchronized Long getBestFileForCleaning(FileSelector fileSelector,
0258: boolean forceCleaning, Set lowUtilizationFiles)
0259: throws DatabaseException {
0260:
0261: /* Start with an empty set.*/
0262: if (lowUtilizationFiles != null) {
0263: lowUtilizationFiles.clear();
0264: }
0265:
0266: assert cachePopulated;
0267:
0268: /* Paranoia. There should always be 1 file. */
0269: if (fileSummaryMap.size() == 0) {
0270: return null;
0271: }
0272:
0273: /* Use local variables for mutable properties. */
0274: final int useMinUtilization = minUtilization;
0275: final int useMinFileUtilization = minFileUtilization;
0276: final int useMinAge = minAge;
0277:
0278: /* There must have been at least one checkpoint previously. */
0279: long firstActiveLsn = env.getCheckpointer().getFirstActiveLsn();
0280: if (firstActiveLsn == DbLsn.NULL_LSN) {
0281: return null;
0282: }
0283:
0284: /* Calculate totals and find the best file. */
0285: Iterator iter = fileSummaryMap.keySet().iterator();
0286: Long bestFile = null;
0287: int bestUtilization = 101;
0288: long totalSize = 0;
0289: long totalObsoleteSize = 0;
0290:
0291: while (iter.hasNext()) {
0292: Long file = (Long) iter.next();
0293: long fileNum = file.longValue();
0294:
0295: /* Calculate this file's utilization. */
0296: FileSummary summary = getFileSummary(file);
0297: int obsoleteSize = summary.getObsoleteSize();
0298:
0299: /*
0300: * If the file is already being cleaned, only total the
0301: * non-obsolete amount. This is an optimistic prediction of the
0302: * results of cleaning, and is used to prevent over-cleaning.
0303: * Update the total obsolete size to include the utilization DB
0304: * records that will be deleted when the log file is deleted.
0305: */
0306: if (fileSelector.isFileCleaningInProgress(file)) {
0307: totalSize += summary.totalSize - obsoleteSize;
0308: totalObsoleteSize += estimateUPObsoleteSize(summary);
0309: continue;
0310: }
0311:
0312: /* Add this file's value to the totals. */
0313: totalSize += summary.totalSize;
0314: totalObsoleteSize += obsoleteSize;
0315:
0316: /* If the file is too young to be cleaned, skip it. */
0317: if (DbLsn.getFileNumber(firstActiveLsn) - fileNum < useMinAge) {
0318: continue;
0319: }
0320:
0321: /* Select this file if it has the lowest utilization so far. */
0322: int this Utilization = utilization(obsoleteSize,
0323: summary.totalSize);
0324: if (bestFile == null || this Utilization < bestUtilization) {
0325: bestFile = file;
0326: bestUtilization = this Utilization;
0327: }
0328:
0329: /* Return all low utilization files. */
0330: if (lowUtilizationFiles != null
0331: && this Utilization < useMinUtilization) {
0332: lowUtilizationFiles.add(file);
0333: }
0334: }
0335:
0336: /*
0337: * Return the best file if we are under the minimum utilization or
0338: * we're cleaning aggressively.
0339: */
0340: int totalUtilization = utilization(totalObsoleteSize, totalSize);
0341: if (forceCleaning || totalUtilization < useMinUtilization
0342: || bestUtilization < useMinFileUtilization) {
0343: return bestFile;
0344: } else {
0345: return null;
0346: }
0347: }
0348:
0349: /**
0350: * Calculate the utilization percentage.
0351: */
0352: public static int utilization(long obsoleteSize, long totalSize) {
0353: if (totalSize != 0) {
0354: return (int) (((totalSize - obsoleteSize) * 100) / totalSize);
0355: } else {
0356: return 0;
0357: }
0358: }
0359:
0360: /**
0361: * Estimate the log size that will be made obsolete when a log file is
0362: * deleted and we delete its UP records.
0363: *
0364: * Note that we do not count the space taken by the deleted FileSummaryLN
0365: * records written during log file deletion. These add the same amount to
0366: * the total log size and the obsolete log size, and therefore have a small
0367: * impact on total utilization.
0368: */
0369: private int estimateUPObsoleteSize(FileSummary summary) {
0370:
0371: /* Disabled for now; needs more testing. */
0372: if (true)
0373: return 0;
0374:
0375: /*
0376: * FileSummaryLN overhead:
0377: * 14 Header
0378: * 8 Node
0379: * 1 Deleted
0380: * 4 Data Length (0)
0381: * 32 Base Summary (8 X 4)
0382: * 8 PackedOffsets size and length (2 * 4)
0383: * 8 PackedOffsets first offset
0384: */
0385: final int OVERHEAD = 75;
0386:
0387: /*
0388: * Make an arbitrary estimate of the number of offsets per
0389: * FileSummaryLN. Then estimate the total byte size, assuming all
0390: * short (2 byte) offsets. Round up the total number of log entries.
0391: */
0392: int OFFSETS_PER_LN = 1000;
0393: int BYTES_PER_LN = OVERHEAD
0394: + (OFFSETS_PER_LN * 2 /* Size of short */);
0395: int totalNodes = summary.totalLNCount + summary.totalINCount;
0396: int logEntries = (totalNodes / OFFSETS_PER_LN) + 1 /* Round up */;
0397: return logEntries * BYTES_PER_LN;
0398: }
0399:
0400: /**
0401: * Gets the base summary from the cached map. Add the tracked summary, if
0402: * one exists, to the base summary. Sets all entries obsolete, if the file
0403: * is in the forceCleanFiles set.
0404: */
0405: private synchronized FileSummary getFileSummary(Long file) {
0406:
0407: /* Get base summary. */
0408: FileSummary summary = (FileSummary) fileSummaryMap.get(file);
0409: long fileNum = file.longValue();
0410:
0411: /* Add tracked summary */
0412: TrackedFileSummary trackedSummary = tracker
0413: .getTrackedFile(fileNum);
0414: if (trackedSummary != null) {
0415: FileSummary totals = new FileSummary();
0416: totals.add(summary);
0417: totals.add(trackedSummary);
0418: summary = totals;
0419: }
0420:
0421: /* Set everything obsolete for a file in the forceCleanFiles set. */
0422: if (isForceCleanFile(fileNum)) {
0423: FileSummary allObsolete = new FileSummary();
0424: allObsolete.add(summary);
0425: allObsolete.obsoleteLNCount = allObsolete.totalLNCount;
0426: allObsolete.obsoleteINCount = allObsolete.totalINCount;
0427: summary = allObsolete;
0428: }
0429:
0430: return summary;
0431: }
0432:
0433: /**
0434: * Returns whether the given file is in the forceCleanFiles set.
0435: */
0436: private boolean isForceCleanFile(long file) {
0437:
0438: if (forceCleanFiles != null) {
0439: for (int i = 0; i < forceCleanFiles.length; i += 2) {
0440: long from = forceCleanFiles[i];
0441: long to = forceCleanFiles[i + 1];
0442: if (file >= from && file <= to) {
0443: return true;
0444: }
0445: }
0446: }
0447: return false;
0448: }
0449:
0450: /**
0451: * Parses the je.cleaner.forceCleanFiles property value.
0452: */
0453: private void parseForceCleanFiles(String propValue) {
0454:
0455: if (propValue == null || propValue.length() == 0) {
0456: forceCleanFiles = null;
0457: } else {
0458:
0459: String errPrefix = "Error in "
0460: + EnvironmentParams.CLEANER_FORCE_CLEAN_FILES
0461: .getName() + "=" + propValue + ": ";
0462:
0463: StringTokenizer tokens = new StringTokenizer(propValue,
0464: ",-", true /*returnDelims*/);
0465:
0466: /* Resulting list of Long file numbers. */
0467: List list = new ArrayList();
0468:
0469: while (tokens.hasMoreTokens()) {
0470:
0471: /* Get "from" file number. */
0472: String fromStr = tokens.nextToken();
0473: long fromNum;
0474: try {
0475: fromNum = Long.parseLong(fromStr, 16);
0476: } catch (NumberFormatException e) {
0477: throw new IllegalArgumentException(errPrefix
0478: + "Invalid hex file number: " + fromStr);
0479: }
0480:
0481: long toNum = -1;
0482: if (tokens.hasMoreTokens()) {
0483:
0484: /* Get delimiter. */
0485: String delim = tokens.nextToken();
0486: if (",".equals(delim)) {
0487: toNum = fromNum;
0488: } else if ("-".equals(delim)) {
0489:
0490: /* Get "to" file number." */
0491: if (tokens.hasMoreTokens()) {
0492: String toStr = tokens.nextToken();
0493: try {
0494: toNum = Long.parseLong(toStr, 16);
0495: } catch (NumberFormatException e) {
0496: throw new IllegalArgumentException(
0497: errPrefix
0498: + "Invalid hex file number: "
0499: + toStr);
0500: }
0501: } else {
0502: throw new IllegalArgumentException(
0503: errPrefix
0504: + "Expected file number: "
0505: + delim);
0506: }
0507: } else {
0508: throw new IllegalArgumentException(errPrefix
0509: + "Expected '-' or ',': " + delim);
0510: }
0511: } else {
0512: toNum = fromNum;
0513: }
0514:
0515: assert toNum != -1;
0516: list.add(new Long(fromNum));
0517: list.add(new Long(toNum));
0518: }
0519:
0520: forceCleanFiles = new long[list.size()];
0521: for (int i = 0; i < forceCleanFiles.length; i += 1) {
0522: forceCleanFiles[i] = ((Long) list.get(i)).longValue();
0523: }
0524: }
0525: }
0526:
0527: /**
0528: * Count the given tracked info as obsolete and then log the summaries.
0529: */
0530: public void countAndLogSummaries(TrackedFileSummary[] summaries)
0531: throws DatabaseException {
0532:
0533: /* Count tracked info under the log write latch. */
0534: env.getLogManager().countObsoleteNodes(summaries);
0535:
0536: /* Utilization flushing may be disabled for unittests. */
0537: if (!DbInternal.getCheckpointUP(env.getConfigManager()
0538: .getEnvironmentConfig())) {
0539: return;
0540: }
0541:
0542: /* Write out the modified file summaries. */
0543: for (int i = 0; i < summaries.length; i += 1) {
0544: long fileNum = summaries[i].getFileNumber();
0545: TrackedFileSummary tfs = tracker.getTrackedFile(fileNum);
0546: if (tfs != null) {
0547: flushFileSummary(tfs);
0548: }
0549: }
0550: }
0551:
0552: /**
0553: * Returns a copy of the current file summary map, optionally including
0554: * tracked summary information, for use by the DbSpace utility and by unit
0555: * tests. The returned map's key is a Long file number and its value is a
0556: * FileSummary.
0557: */
0558: public synchronized SortedMap getFileSummaryMap(
0559: boolean includeTrackedFiles) throws DatabaseException {
0560:
0561: assert cachePopulated;
0562:
0563: if (includeTrackedFiles) {
0564: TreeMap map = new TreeMap();
0565: Iterator iter = fileSummaryMap.keySet().iterator();
0566: while (iter.hasNext()) {
0567: Long file = (Long) iter.next();
0568: FileSummary summary = getFileSummary(file);
0569: map.put(file, summary);
0570: }
0571: TrackedFileSummary[] trackedFiles = tracker
0572: .getTrackedFiles();
0573: for (int i = 0; i < trackedFiles.length; i += 1) {
0574: TrackedFileSummary summary = trackedFiles[i];
0575: long fileNum = summary.getFileNumber();
0576: Long file = new Long(fileNum);
0577: if (!map.containsKey(file)) {
0578: map.put(file, summary);
0579: }
0580: }
0581: return map;
0582: } else {
0583: return new TreeMap(fileSummaryMap);
0584: }
0585: }
0586:
0587: /**
0588: * Clears the cache of file summary info. The cache starts out unpopulated
0589: * and is populated on the first call to getBestFileForCleaning.
0590: */
0591: public synchronized void clearCache() {
0592:
0593: int memorySize = fileSummaryMap.size()
0594: * MemoryBudget.UTILIZATION_PROFILE_ENTRY;
0595: MemoryBudget mb = env.getMemoryBudget();
0596: mb.updateMiscMemoryUsage(0 - memorySize);
0597:
0598: fileSummaryMap = new TreeMap();
0599: cachePopulated = false;
0600: }
0601:
0602: /**
0603: * Removes a file from the utilization database and the profile, after it
0604: * has been deleted by the cleaner.
0605: */
0606: void removeFile(Long fileNum) throws DatabaseException {
0607:
0608: /* Synchronize to update the cache. */
0609: synchronized (this ) {
0610: assert cachePopulated;
0611:
0612: /* Remove from the cache. */
0613: FileSummary oldSummary = (FileSummary) fileSummaryMap
0614: .remove(fileNum);
0615: if (oldSummary != null) {
0616: MemoryBudget mb = env.getMemoryBudget();
0617: mb
0618: .updateMiscMemoryUsage(0 - MemoryBudget.UTILIZATION_PROFILE_ENTRY);
0619: totalLogSize -= oldSummary.totalSize;
0620: }
0621: }
0622:
0623: /* Do not synchronize during LN deletion, to permit eviction. */
0624: deleteFileSummary(fileNum);
0625: }
0626:
0627: /**
0628: * For the LN at the cursor position deletes all LNs for the file. This
0629: * method performs eviction and is not synchronized.
0630: */
0631: private void deleteFileSummary(Long fileNum)
0632: throws DatabaseException {
0633:
0634: Locker locker = null;
0635: CursorImpl cursor = null;
0636: try {
0637: locker = new BasicLocker(env);
0638: cursor = new CursorImpl(fileSummaryDb, locker);
0639: /* Perform eviction in unsynchronized methods. */
0640: cursor.setAllowEviction(true);
0641:
0642: DatabaseEntry keyEntry = new DatabaseEntry();
0643: DatabaseEntry dataEntry = new DatabaseEntry();
0644: long fileNumVal = fileNum.longValue();
0645:
0646: /* Search by file number. */
0647: if (!getFirstFSLN(cursor, fileNumVal, keyEntry, dataEntry,
0648: LockType.WRITE)) {
0649: return;
0650: }
0651:
0652: /* Delete all LNs for this file number. */
0653: OperationStatus status = OperationStatus.SUCCESS;
0654: while (status == OperationStatus.SUCCESS) {
0655:
0656: /* Perform eviction once per operation. */
0657: env.getEvictor().doCriticalEviction(true); // backgroundIO
0658:
0659: FileSummaryLN ln = (FileSummaryLN) cursor
0660: .getCurrentLN(LockType.NONE);
0661:
0662: if (ln != null) {
0663: /* Stop if the file number changes. */
0664: if (fileNumVal != ln.getFileNumber(keyEntry
0665: .getData())) {
0666: break;
0667: }
0668:
0669: TrackedFileSummary tfs = tracker
0670: .getTrackedFile(fileNumVal);
0671: /* Associate the tracked summary so it will be cleared. */
0672: if (tfs != null) {
0673: ln.setTrackedSummary(tfs);
0674: }
0675:
0676: /*
0677: * Do not evict after deleting since the compressor would
0678: * have to fetch it again.
0679: */
0680: cursor.latchBIN();
0681: try {
0682: cursor.delete();
0683: } finally {
0684: cursor.releaseBIN();
0685: }
0686: }
0687:
0688: status = cursor.getNext(keyEntry, dataEntry,
0689: LockType.WRITE, true, // forward
0690: false); // alreadyLatched
0691: }
0692: } finally {
0693: if (cursor != null) {
0694: cursor.releaseBINs();
0695: cursor.close();
0696: }
0697: if (locker != null) {
0698: locker.operationEnd();
0699: }
0700: }
0701: }
0702:
0703: /**
0704: * Updates and stores the FileSummary for a given tracked file, if flushing
0705: * of the summary is allowed.
0706: */
0707: public void flushFileSummary(TrackedFileSummary tfs)
0708: throws DatabaseException {
0709:
0710: if (tfs.getAllowFlush()) {
0711: putFileSummary(tfs);
0712: }
0713: }
0714:
0715: /**
0716: * Updates and stores the FileSummary for a given tracked file. This
0717: * method is synchronized and may not perform eviction.
0718: */
0719: private synchronized PackedOffsets putFileSummary(
0720: TrackedFileSummary tfs) throws DatabaseException {
0721:
0722: if (env.isReadOnly()) {
0723: throw new DatabaseException(
0724: "Cannot write file summary in a read-only environment");
0725: }
0726:
0727: if (tfs.isEmpty()) {
0728: return null; // no delta
0729: }
0730:
0731: if (!cachePopulated) {
0732: /* Db does not exist and this is a read-only environment. */
0733: return null;
0734: }
0735:
0736: long fileNum = tfs.getFileNumber();
0737: Long fileNumLong = new Long(fileNum);
0738:
0739: /* Get existing file summary or create an empty one. */
0740: FileSummary summary = (FileSummary) fileSummaryMap
0741: .get(fileNumLong);
0742: if (summary == null) {
0743:
0744: /*
0745: * An obsolete node may have been counted after its file was
0746: * deleted, for example, when compressing a BIN. Do not insert a
0747: * new profile record if no corresponding log file exists. But if
0748: * the file number is greater than the last known file, this is a
0749: * new file that has been buffered but not yet flushed to disk; in
0750: * that case we should insert a new profile record.
0751: */
0752: if (!fileSummaryMap.isEmpty()
0753: && fileNum < ((Long) fileSummaryMap.lastKey())
0754: .longValue()) {
0755: File file = new File(
0756: env.getFileManager().getFullFileName(fileNum,
0757: FileManager.JE_SUFFIX));
0758: if (!file.exists()) {
0759:
0760: /*
0761: * File was deleted by the cleaner. Remove it from the
0762: * UtilizationTracker and return. Note that a file is
0763: * normally removed from the tracker by
0764: * FileSummaryLN.writeToLog method when it is called via
0765: * insertFileSummary below. [#15512]
0766: */
0767: env.getLogManager().removeTrackedFile(tfs);
0768: return null;
0769: }
0770: }
0771:
0772: summary = new FileSummary();
0773: }
0774:
0775: /*
0776: * The key discriminator is a sequence that must be increasing over the
0777: * life of the file. We use the sum of all entries counted. We must
0778: * add the tracked and current summaries here to calculate the key.
0779: */
0780: FileSummary tmp = new FileSummary();
0781: tmp.add(summary);
0782: tmp.add(tfs);
0783: totalLogSize += tfs.totalSize;
0784: int sequence = tmp.getEntriesCounted();
0785:
0786: /* Insert an LN with the existing and tracked summary info. */
0787: FileSummaryLN ln = new FileSummaryLN(summary);
0788: ln.setTrackedSummary(tfs);
0789: insertFileSummary(ln, fileNum, sequence);
0790:
0791: /* Cache the updated summary object. */
0792: summary = ln.getBaseSummary();
0793: if (fileSummaryMap.put(fileNumLong, summary) == null) {
0794: MemoryBudget mb = env.getMemoryBudget();
0795: mb
0796: .updateMiscMemoryUsage(MemoryBudget.UTILIZATION_PROFILE_ENTRY);
0797: }
0798:
0799: return ln.getObsoleteOffsets();
0800: }
0801:
0802: /**
0803: * Returns the stored/packed obsolete offsets and the tracked obsolete
0804: * offsets for the given file. The tracked summary object returned can be
0805: * used to test for obsolete offsets that are being added during cleaning
0806: * by other threads participating in lazy migration. The caller must call
0807: * TrackedFileSummary.setAllowFlush(true) when cleaning is complete.
0808: * This method performs eviction and is not synchronized.
0809: * @param logUpdate if true, log any updates to the utilization profile. If
0810: * false, only retrieve the new information.
0811: */
0812: TrackedFileSummary getObsoleteDetail(Long fileNum,
0813: PackedOffsets packedOffsets, boolean logUpdate)
0814: throws DatabaseException {
0815:
0816: /* Return if no detail is being tracked. */
0817: if (!env.getCleaner().trackDetail) {
0818: return null;
0819: }
0820:
0821: assert cachePopulated;
0822:
0823: long fileNumVal = fileNum.longValue();
0824: List list = new ArrayList();
0825:
0826: /*
0827: * Get an unflushable summary that will remain valid for the duration
0828: * of file cleaning.
0829: */
0830: TrackedFileSummary tfs = env.getLogManager()
0831: .getUnflushableTrackedSummary(fileNumVal);
0832:
0833: /* Read the summary db. */
0834: Locker locker = null;
0835: CursorImpl cursor = null;
0836: try {
0837: locker = new BasicLocker(env);
0838: cursor = new CursorImpl(fileSummaryDb, locker);
0839: /* Perform eviction in unsynchronized methods. */
0840: cursor.setAllowEviction(true);
0841:
0842: DatabaseEntry keyEntry = new DatabaseEntry();
0843: DatabaseEntry dataEntry = new DatabaseEntry();
0844:
0845: /* Search by file number. */
0846: OperationStatus status = OperationStatus.SUCCESS;
0847: if (!getFirstFSLN(cursor, fileNumVal, keyEntry, dataEntry,
0848: LockType.NONE)) {
0849: status = OperationStatus.NOTFOUND;
0850: }
0851:
0852: /* Read all LNs for this file number. */
0853: while (status == OperationStatus.SUCCESS) {
0854:
0855: /* Perform eviction once per operation. */
0856: env.getEvictor().doCriticalEviction(true); // backgroundIO
0857:
0858: FileSummaryLN ln = (FileSummaryLN) cursor
0859: .getCurrentLN(LockType.NONE);
0860: if (ln != null) {
0861: /* Stop if the file number changes. */
0862: if (fileNumVal != ln.getFileNumber(keyEntry
0863: .getData())) {
0864: break;
0865: }
0866:
0867: PackedOffsets offsets = ln.getObsoleteOffsets();
0868: if (offsets != null) {
0869: list.add(offsets.toArray());
0870: }
0871:
0872: /* Always evict after using a file summary LN. */
0873: cursor.evict();
0874: }
0875:
0876: status = cursor.getNext(keyEntry, dataEntry,
0877: LockType.NONE, true, // forward
0878: false); // alreadyLatched
0879: }
0880: } finally {
0881: if (cursor != null) {
0882: cursor.releaseBINs();
0883: cursor.close();
0884: }
0885: if (locker != null) {
0886: locker.operationEnd();
0887: }
0888: }
0889:
0890: /*
0891: * Write out tracked detail, if any, and add its offsets to the list.
0892: */
0893: if (!tfs.isEmpty()) {
0894: PackedOffsets offsets = null;
0895: if (logUpdate) {
0896: offsets = putFileSummary(tfs);
0897: if (offsets != null) {
0898: list.add(offsets.toArray());
0899: }
0900: } else {
0901: long[] offsetList = tfs.getObsoleteOffsets();
0902: if (offsetList != null) {
0903: list.add(offsetList);
0904: }
0905: }
0906: }
0907:
0908: /* Merge all offsets into a single array and pack the result. */
0909: int size = 0;
0910: for (int i = 0; i < list.size(); i += 1) {
0911: long[] a = (long[]) list.get(i);
0912: size += a.length;
0913: }
0914: long[] offsets = new long[size];
0915: int index = 0;
0916: for (int i = 0; i < list.size(); i += 1) {
0917: long[] a = (long[]) list.get(i);
0918: System.arraycopy(a, 0, offsets, index, a.length);
0919: index += a.length;
0920: }
0921: assert index == offsets.length;
0922:
0923: packedOffsets.pack(offsets);
0924:
0925: return tfs;
0926: }
0927:
0928: /**
0929: * Populate the profile for file selection. This method performs eviction
0930: * and is not synchronized. It must be called before recovery is complete
0931: * so that synchronization is unnecessary. It must be called before the
0932: * recovery checkpoint so that the checkpoint can flush file summary
0933: * information.
0934: */
0935: public boolean populateCache() throws DatabaseException {
0936:
0937: assert !cachePopulated;
0938:
0939: /* Open the file summary db on first use. */
0940: if (!openFileSummaryDatabase()) {
0941: /* Db does not exist and this is a read-only environment. */
0942: return false;
0943: }
0944:
0945: int oldMemorySize = fileSummaryMap.size()
0946: * MemoryBudget.UTILIZATION_PROFILE_ENTRY;
0947:
0948: totalLogSize = 0;
0949:
0950: /*
0951: * It is possible to have an undeleted FileSummaryLN in the database
0952: * for a deleted log file if we crash after deleting a file but before
0953: * deleting the FileSummaryLN. Iterate through all FileSummaryLNs and
0954: * add them to the cache if their corresponding log file exists. But
0955: * delete those records that have no corresponding log file.
0956: */
0957: Long[] existingFiles = env.getFileManager().getAllFileNumbers();
0958: Locker locker = null;
0959: CursorImpl cursor = null;
0960: try {
0961: locker = new BasicLocker(env);
0962: cursor = new CursorImpl(fileSummaryDb, locker);
0963: /* Perform eviction in unsynchronized methods. */
0964: cursor.setAllowEviction(true);
0965:
0966: DatabaseEntry keyEntry = new DatabaseEntry();
0967: DatabaseEntry dataEntry = new DatabaseEntry();
0968:
0969: if (cursor.positionFirstOrLast(true, null)) {
0970:
0971: /* Retrieve the first record. */
0972: OperationStatus status = cursor
0973: .getCurrentAlreadyLatched(keyEntry, dataEntry,
0974: LockType.NONE, true);
0975: if (status != OperationStatus.SUCCESS) {
0976: /* The record we're pointing at may be deleted. */
0977: status = cursor.getNext(keyEntry, dataEntry,
0978: LockType.NONE, true, // go forward
0979: false); // do need to latch
0980: }
0981:
0982: while (status == OperationStatus.SUCCESS) {
0983:
0984: /*
0985: * Perform eviction once per operation. Pass false for
0986: * backgroundIO because this is done during recovery and
0987: * there is no reason to sleep.
0988: */
0989: env.getEvictor().doCriticalEviction(false); // backgroundIO
0990:
0991: FileSummaryLN ln = (FileSummaryLN) cursor
0992: .getCurrentLN(LockType.NONE);
0993:
0994: if (ln == null) {
0995: /* Advance past a cleaned record. */
0996: status = cursor.getNext(keyEntry, dataEntry,
0997: LockType.NONE, true, // go forward
0998: false); // do need to latch
0999: continue;
1000: }
1001:
1002: byte[] keyBytes = keyEntry.getData();
1003: boolean isOldVersion = ln.hasStringKey(keyBytes);
1004: long fileNum = ln.getFileNumber(keyBytes);
1005: Long fileNumLong = new Long(fileNum);
1006:
1007: if (Arrays.binarySearch(existingFiles, fileNumLong) >= 0) {
1008:
1009: /* File exists, cache the FileSummaryLN. */
1010: FileSummary summary = ln.getBaseSummary();
1011: fileSummaryMap.put(fileNumLong, summary);
1012: totalLogSize += summary.totalSize;
1013:
1014: /*
1015: * Update old version records to the new version. A
1016: * zero sequence number is used to distinguish the
1017: * converted records and to ensure that later records
1018: * will have a greater sequence number.
1019: */
1020: if (isOldVersion && !env.isReadOnly()) {
1021: insertFileSummary(ln, fileNum, 0);
1022: cursor.latchBIN();
1023: cursor.delete();
1024: cursor.releaseBIN();
1025: } else {
1026: /* Always evict after using a file summary LN. */
1027: cursor.evict();
1028: }
1029: } else {
1030:
1031: /*
1032: * File does not exist, remove the summary from the map
1033: * and delete all FileSummaryLN records.
1034: */
1035: fileSummaryMap.remove(fileNumLong);
1036:
1037: if (!env.isReadOnly()) {
1038: if (isOldVersion) {
1039: cursor.latchBIN();
1040: cursor.delete();
1041: cursor.releaseBIN();
1042: } else {
1043: deleteFileSummary(fileNumLong);
1044: }
1045: }
1046:
1047: /*
1048: * Do not evict after deleting since the compressor
1049: * would have to fetch it again.
1050: */
1051: }
1052:
1053: /* Go on to the next entry. */
1054: if (isOldVersion) {
1055:
1056: /* Advance past the single old version record. */
1057: status = cursor.getNext(keyEntry, dataEntry,
1058: LockType.NONE, true, // go forward
1059: false); // do need to latch
1060: } else {
1061:
1062: /*
1063: * Skip over other records for this file by adding one
1064: * to the file number and doing a range search.
1065: */
1066: if (!getFirstFSLN(cursor, fileNum + 1,
1067: keyEntry, dataEntry, LockType.NONE)) {
1068: status = OperationStatus.NOTFOUND;
1069: }
1070: }
1071: }
1072: }
1073: } finally {
1074: if (cursor != null) {
1075: cursor.releaseBINs();
1076: cursor.close();
1077: }
1078: if (locker != null) {
1079: locker.operationEnd();
1080: }
1081:
1082: int newMemorySize = fileSummaryMap.size()
1083: * MemoryBudget.UTILIZATION_PROFILE_ENTRY;
1084: MemoryBudget mb = env.getMemoryBudget();
1085: mb.updateMiscMemoryUsage(newMemorySize - oldMemorySize);
1086: }
1087:
1088: cachePopulated = true;
1089: return true;
1090: }
1091:
1092: /**
1093: * Positions at the most recent LN for the given file number.
1094: */
1095: private boolean getFirstFSLN(CursorImpl cursor, long fileNum,
1096: DatabaseEntry keyEntry, DatabaseEntry dataEntry,
1097: LockType lockType) throws DatabaseException {
1098:
1099: byte[] keyBytes = FileSummaryLN.makePartialKey(fileNum);
1100: keyEntry.setData(keyBytes);
1101:
1102: int result = cursor.searchAndPosition(keyEntry, dataEntry,
1103: SearchMode.SET_RANGE, lockType);
1104: if ((result & CursorImpl.FOUND) == 0) {
1105: return false;
1106: }
1107:
1108: boolean exactKeyMatch = ((result & CursorImpl.EXACT_KEY) != 0);
1109:
1110: if (exactKeyMatch
1111: && cursor.getCurrentAlreadyLatched(keyEntry, dataEntry,
1112: lockType, true) != OperationStatus.KEYEMPTY) {
1113: return true;
1114: }
1115:
1116: /* Always evict after using a file summary LN. */
1117: cursor.evict(!exactKeyMatch); // alreadyLatched
1118:
1119: OperationStatus status = cursor.getNext(keyEntry, dataEntry,
1120: lockType, true, // forward
1121: !exactKeyMatch); // alreadyLatched
1122:
1123: return status == OperationStatus.SUCCESS;
1124: }
1125:
1126: /**
1127: * If the file summary db is already open, return, otherwise attempt to
1128: * open it. If the environment is read-only and the database doesn't
1129: * exist, return false. If the environment is read-write the database will
1130: * be created if it doesn't exist.
1131: */
1132: private boolean openFileSummaryDatabase() throws DatabaseException {
1133:
1134: if (fileSummaryDb != null) {
1135: return true;
1136: }
1137: DbTree dbTree = env.getDbMapTree();
1138: Locker autoTxn = null;
1139: boolean operationOk = false;
1140: try {
1141: autoTxn = new AutoTxn(env, new TransactionConfig());
1142:
1143: /*
1144: * releaseDb is not called after this getDb or createDb because we
1145: * want to prohibit eviction of this database until the environment
1146: * is closed.
1147: */
1148: DatabaseImpl db = dbTree.getDb(autoTxn,
1149: DbTree.UTILIZATION_DB_NAME, null);
1150: if (db == null) {
1151: if (env.isReadOnly()) {
1152: return false;
1153: }
1154: db = dbTree.createDb(autoTxn,
1155: DbTree.UTILIZATION_DB_NAME,
1156: new DatabaseConfig(), null);
1157: }
1158: fileSummaryDb = db;
1159: operationOk = true;
1160: return true;
1161: } finally {
1162: if (autoTxn != null) {
1163: autoTxn.operationEnd(operationOk);
1164: }
1165: }
1166: }
1167:
1168: /**
1169: * For unit testing.
1170: */
1171: DatabaseImpl getFileSummaryDb() {
1172: return fileSummaryDb;
1173: }
1174:
1175: /**
1176: * Insert the given LN with the given key values. This method is
1177: * synchronized and may not perform eviction.
1178: */
1179: private synchronized void insertFileSummary(FileSummaryLN ln,
1180: long fileNum, int sequence) throws DatabaseException {
1181:
1182: byte[] keyBytes = FileSummaryLN.makeFullKey(fileNum, sequence);
1183:
1184: Locker locker = null;
1185: CursorImpl cursor = null;
1186: try {
1187: locker = new BasicLocker(env);
1188: cursor = new CursorImpl(fileSummaryDb, locker);
1189:
1190: /* Insert the LN. */
1191: OperationStatus status = cursor.putLN(keyBytes, ln, false);
1192: if (status == OperationStatus.KEYEXIST) {
1193: env.getLogger().log(
1194: Level.SEVERE,
1195: "Cleaner duplicate key sequence file=0x"
1196: + Long.toHexString(fileNum)
1197: + " sequence=0x"
1198: + Long.toHexString(sequence));
1199: }
1200:
1201: /* Always evict after using a file summary LN. */
1202: cursor.evict();
1203: } finally {
1204: if (cursor != null) {
1205: cursor.close();
1206: }
1207: if (locker != null) {
1208: locker.operationEnd();
1209: }
1210: }
1211: }
1212:
1213: /**
1214: * Checks that all FSLN offsets are indeed obsolete. Assumes that the
1215: * system is quiesent (does not lock LNs). This method is not synchronized
1216: * (because it doesn't access fileSummaryMap) and eviction is allowed.
1217: *
1218: * @return true if no verification failures.
1219: */
1220: public boolean verifyFileSummaryDatabase() throws DatabaseException {
1221:
1222: DatabaseEntry key = new DatabaseEntry();
1223: DatabaseEntry data = new DatabaseEntry();
1224:
1225: openFileSummaryDatabase();
1226: Locker locker = null;
1227: CursorImpl cursor = null;
1228: boolean ok = true;
1229:
1230: try {
1231: locker = new BasicLocker(env);
1232: cursor = new CursorImpl(fileSummaryDb, locker);
1233: cursor.setAllowEviction(true);
1234:
1235: if (cursor.positionFirstOrLast(true, null)) {
1236:
1237: OperationStatus status = cursor
1238: .getCurrentAlreadyLatched(key, data,
1239: LockType.NONE, true);
1240:
1241: /* Iterate over all file summary lns. */
1242: while (status == OperationStatus.SUCCESS) {
1243:
1244: /* Perform eviction once per operation. */
1245: env.getEvictor().doCriticalEviction(true); // backgroundIO
1246:
1247: FileSummaryLN ln = (FileSummaryLN) cursor
1248: .getCurrentLN(LockType.NONE);
1249:
1250: if (ln != null) {
1251: long fileNumVal = ln.getFileNumber(key
1252: .getData());
1253: PackedOffsets offsets = ln.getObsoleteOffsets();
1254:
1255: /*
1256: * Check every offset in the fsln to make sure it's
1257: * truely obsolete.
1258: */
1259: if (offsets != null) {
1260: long[] vals = offsets.toArray();
1261: for (int i = 0; i < vals.length; i++) {
1262: long lsn = DbLsn.makeLsn(fileNumVal,
1263: vals[i]);
1264: if (!verifyLsnIsObsolete(lsn)) {
1265: ok = false;
1266: }
1267: }
1268: }
1269:
1270: cursor.evict();
1271: status = cursor.getNext(key, data,
1272: LockType.NONE, true, // forward
1273: false); // already latched
1274: }
1275: }
1276: }
1277: } finally {
1278: if (cursor != null) {
1279: cursor.close();
1280: }
1281: if (locker != null) {
1282: locker.operationEnd();
1283: }
1284: }
1285:
1286: return ok;
1287: }
1288:
1289: /*
1290: * Return true if the LN at this lsn is obsolete.
1291: */
1292: private boolean verifyLsnIsObsolete(long lsn)
1293: throws DatabaseException {
1294:
1295: /* Read the whole entry out of the log. */
1296: Object o = env.getLogManager().getLogEntry(lsn);
1297: if (!(o instanceof LNLogEntry)) {
1298: return true;
1299: }
1300: LNLogEntry entry = (LNLogEntry) o;
1301:
1302: /* All deleted LNs are obsolete. */
1303: if (entry.getLN().isDeleted()) {
1304: return true;
1305: }
1306:
1307: /* Find the owning database. */
1308: DatabaseId dbId = entry.getDbId();
1309: DatabaseImpl db = env.getDbMapTree().getDb(dbId);
1310:
1311: /*
1312: * Search down to the bottom most level for the parent of this LN.
1313: */
1314: BIN bin = null;
1315: try {
1316:
1317: /*
1318: * The whole database is gone, so this LN is obsolete. No need
1319: * to worry about delete cleanup; this is just verification and
1320: * no cleaning is done.
1321: */
1322: if (db == null || db.isDeleted()) {
1323: return true;
1324: }
1325:
1326: Tree tree = db.getTree();
1327: TreeLocation location = new TreeLocation();
1328: boolean parentFound = tree.getParentBINForChildLN(location,
1329: entry.getKey(), entry.getDupKey(), entry.getLN(),
1330: false, // splitsAllowed
1331: true, // findDeletedEntries
1332: false, // searchDupTree ???
1333: false); // updateGeneration
1334: bin = location.bin;
1335: int index = location.index;
1336:
1337: /* Is bin latched ? */
1338: if (!parentFound) {
1339: return true;
1340: }
1341:
1342: /*
1343: * Now we're at the parent for this LN, whether BIN, DBIN or DIN.
1344: * If knownDeleted, LN is deleted and can be purged.
1345: */
1346: if (bin.isEntryKnownDeleted(index)) {
1347: return true;
1348: }
1349:
1350: if (bin.getLsn(index) != lsn) {
1351: return true;
1352: }
1353:
1354: /* Oh no -- this lsn is in the tree. */
1355: /* should print, or trace? */
1356: System.err.println("lsn " + DbLsn.getNoFormatString(lsn)
1357: + " was found in tree.");
1358: return false;
1359: } finally {
1360: env.getDbMapTree().releaseDb(db);
1361: if (bin != null) {
1362: bin.releaseLatch();
1363: }
1364: }
1365: }
1366: }
|