0001: /*-
0002: * See the file LICENSE for redistribution information.
0003: *
0004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
0005: *
0006: * $Id: CleanerTest.java,v 1.87.2.7 2008/01/07 15:14:25 cwl Exp $
0007: */
0008:
0009: package com.sleepycat.je.cleaner;
0010:
0011: import java.io.File;
0012: import java.io.IOException;
0013: import java.util.HashMap;
0014: import java.util.HashSet;
0015: import java.util.Iterator;
0016: import java.util.Map;
0017: import java.util.Set;
0018:
0019: import junit.framework.TestCase;
0020:
0021: import com.sleepycat.bind.tuple.IntegerBinding;
0022: import com.sleepycat.je.CheckpointConfig;
0023: import com.sleepycat.je.Cursor;
0024: import com.sleepycat.je.Database;
0025: import com.sleepycat.je.DatabaseConfig;
0026: import com.sleepycat.je.DatabaseEntry;
0027: import com.sleepycat.je.DatabaseException;
0028: import com.sleepycat.je.DbInternal;
0029: import com.sleepycat.je.Environment;
0030: import com.sleepycat.je.EnvironmentConfig;
0031: import com.sleepycat.je.EnvironmentMutableConfig;
0032: import com.sleepycat.je.EnvironmentStats;
0033: import com.sleepycat.je.LockMode;
0034: import com.sleepycat.je.OperationStatus;
0035: import com.sleepycat.je.Transaction;
0036: import com.sleepycat.je.cleaner.Cleaner;
0037: import com.sleepycat.je.cleaner.FileSummary;
0038: import com.sleepycat.je.cleaner.TrackedFileSummary;
0039: import com.sleepycat.je.cleaner.UtilizationProfile;
0040: import com.sleepycat.je.config.EnvironmentParams;
0041: import com.sleepycat.je.dbi.CursorImpl;
0042: import com.sleepycat.je.dbi.DatabaseImpl;
0043: import com.sleepycat.je.dbi.EnvironmentImpl;
0044: import com.sleepycat.je.dbi.MemoryBudget;
0045: import com.sleepycat.je.log.FileManager;
0046: import com.sleepycat.je.tree.BIN;
0047: import com.sleepycat.je.tree.FileSummaryLN;
0048: import com.sleepycat.je.tree.IN;
0049: import com.sleepycat.je.txn.BasicLocker;
0050: import com.sleepycat.je.txn.LockType;
0051: import com.sleepycat.je.util.StringDbt;
0052: import com.sleepycat.je.util.TestUtils;
0053:
0054: public class CleanerTest extends TestCase {
0055:
0056: private static final int N_KEYS = 300;
0057: private static final int N_KEY_BYTES = 10;
0058:
0059: /*
0060: * Make the log file size small enough to allow cleaning, but large enough
0061: * not to generate a lot of fsyncing at the log file boundaries.
0062: */
0063: private static final int FILE_SIZE = 10000;
0064: protected File envHome = null;
0065: protected Database db = null;
0066: private Environment exampleEnv;
0067: private Database exampleDb;
0068: private CheckpointConfig forceConfig;
0069:
0070: public CleanerTest() {
0071: envHome = new File(System.getProperty(TestUtils.DEST_DIR));
0072: forceConfig = new CheckpointConfig();
0073: forceConfig.setForce(true);
0074: }
0075:
0076: public void setUp() throws IOException, DatabaseException {
0077:
0078: TestUtils.removeLogFiles("Setup", envHome, false);
0079: TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX);
0080: }
0081:
0082: private void initEnv(boolean createDb, boolean allowDups)
0083: throws DatabaseException {
0084:
0085: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
0086: DbInternal.disableParameterValidation(envConfig);
0087: envConfig.setTransactional(true);
0088: envConfig.setAllowCreate(true);
0089: envConfig.setTxnNoSync(Boolean.getBoolean(TestUtils.NO_SYNC));
0090: envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX
0091: .getName(), Integer.toString(FILE_SIZE));
0092: envConfig.setConfigParam(EnvironmentParams.ENV_CHECK_LEAKS
0093: .getName(), "false");
0094: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0095: .getName(), "false");
0096: envConfig.setConfigParam(EnvironmentParams.CLEANER_REMOVE
0097: .getName(), "false");
0098: envConfig.setConfigParam(
0099: EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(),
0100: "80");
0101: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CHECKPOINTER
0102: .getName(), "false");
0103: envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(),
0104: "6");
0105: envConfig.setConfigParam(EnvironmentParams.BIN_DELTA_PERCENT
0106: .getName(), "75");
0107:
0108: /* Don't use detail tracking in this test. */
0109: envConfig.setConfigParam(EnvironmentParams.CLEANER_TRACK_DETAIL
0110: .getName(), "false");
0111:
0112: exampleEnv = new Environment(envHome, envConfig);
0113:
0114: String databaseName = "cleanerDb";
0115: DatabaseConfig dbConfig = new DatabaseConfig();
0116: dbConfig.setTransactional(true);
0117: dbConfig.setAllowCreate(createDb);
0118: dbConfig.setSortedDuplicates(allowDups);
0119: exampleDb = exampleEnv.openDatabase(null, databaseName,
0120: dbConfig);
0121: }
0122:
0123: public void tearDown() throws IOException, DatabaseException {
0124:
0125: if (exampleEnv != null) {
0126: try {
0127: exampleEnv.close();
0128: } catch (DatabaseException e) {
0129: System.out.println("tearDown: " + e);
0130: }
0131: }
0132: exampleDb = null;
0133: exampleEnv = null;
0134:
0135: //*
0136: TestUtils.removeLogFiles("TearDown", envHome, true);
0137: TestUtils.removeFiles("TearDown", envHome,
0138: FileManager.DEL_SUFFIX);
0139: //*/
0140: }
0141:
0142: private void closeEnv() throws DatabaseException {
0143:
0144: if (exampleDb != null) {
0145: exampleDb.close();
0146: exampleDb = null;
0147: }
0148:
0149: if (exampleEnv != null) {
0150: exampleEnv.close();
0151: exampleEnv = null;
0152: }
0153: }
0154:
0155: public void testCleanerNoDupes() throws Throwable {
0156:
0157: initEnv(true, false);
0158: try {
0159: doCleanerTest(N_KEYS, 1);
0160: } catch (Throwable t) {
0161: t.printStackTrace();
0162: throw t;
0163: }
0164: }
0165:
0166: public void testCleanerWithDupes() throws Throwable {
0167:
0168: initEnv(true, true);
0169: try {
0170: doCleanerTest(2, 500);
0171: } catch (Throwable t) {
0172: t.printStackTrace();
0173: throw t;
0174: }
0175: }
0176:
0177: private void doCleanerTest(int nKeys, int nDupsPerKey)
0178: throws DatabaseException {
0179:
0180: EnvironmentImpl environment = DbInternal
0181: .envGetEnvironmentImpl(exampleEnv);
0182: FileManager fileManager = environment.getFileManager();
0183: Map expectedMap = new HashMap();
0184: doLargePut(expectedMap, nKeys, nDupsPerKey, true);
0185: Long lastNum = fileManager.getLastFileNum();
0186:
0187: /* Read the data back. */
0188: StringDbt foundKey = new StringDbt();
0189: StringDbt foundData = new StringDbt();
0190:
0191: Cursor cursor = exampleDb.openCursor(null, null);
0192:
0193: while (cursor.getNext(foundKey, foundData, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
0194: }
0195:
0196: exampleEnv.checkpoint(forceConfig);
0197:
0198: for (int i = 0; i < (int) lastNum.longValue(); i++) {
0199:
0200: /*
0201: * Force clean one file. Utilization-based cleaning won't
0202: * work here, since utilization is over 90%.
0203: */
0204: DbInternal.envGetEnvironmentImpl(exampleEnv).getCleaner()
0205: .doClean(false, // cleanMultipleFiles
0206: true); // forceCleaning
0207: }
0208:
0209: EnvironmentStats stats = exampleEnv
0210: .getStats(TestUtils.FAST_STATS);
0211: assertTrue(stats.getNINsCleaned() > 0);
0212:
0213: cursor.close();
0214: closeEnv();
0215:
0216: initEnv(false, (nDupsPerKey > 1));
0217:
0218: checkData(expectedMap);
0219: assertTrue(fileManager.getLastFileNum().longValue() > lastNum
0220: .longValue());
0221:
0222: closeEnv();
0223: }
0224:
0225: /**
0226: * Ensure that INs are cleaned.
0227: */
0228: public void testCleanInternalNodes() throws DatabaseException {
0229:
0230: initEnv(true, true);
0231: int nKeys = 200;
0232:
0233: EnvironmentImpl environment = DbInternal
0234: .envGetEnvironmentImpl(exampleEnv);
0235: FileManager fileManager = environment.getFileManager();
0236: /* Insert a lot of keys. ExpectedMap holds the expected data */
0237: Map expectedMap = new HashMap();
0238: doLargePut(expectedMap, nKeys, 1, true);
0239:
0240: /* Modify every other piece of data. */
0241: modifyData(expectedMap, 10, true);
0242: checkData(expectedMap);
0243:
0244: /* Checkpoint */
0245: exampleEnv.checkpoint(forceConfig);
0246: checkData(expectedMap);
0247:
0248: /* Modify every other piece of data. */
0249: modifyData(expectedMap, 10, true);
0250: checkData(expectedMap);
0251:
0252: /* Checkpoint -- this should obsolete INs. */
0253: exampleEnv.checkpoint(forceConfig);
0254: checkData(expectedMap);
0255:
0256: /* Clean */
0257: Long lastNum = fileManager.getLastFileNum();
0258: exampleEnv.cleanLog();
0259:
0260: /* Validate after cleaning. */
0261: checkData(expectedMap);
0262: EnvironmentStats stats = exampleEnv
0263: .getStats(TestUtils.FAST_STATS);
0264:
0265: /* Make sure we really cleaned something.*/
0266: assertTrue(stats.getNINsCleaned() > 0);
0267: assertTrue(stats.getNLNsCleaned() > 0);
0268:
0269: closeEnv();
0270: initEnv(false, true);
0271: checkData(expectedMap);
0272: assertTrue(fileManager.getLastFileNum().longValue() > lastNum
0273: .longValue());
0274:
0275: closeEnv();
0276: }
0277:
0278: /**
0279: * See if we can clean in the middle of the file set.
0280: */
0281: public void testCleanFileHole() throws Throwable {
0282:
0283: initEnv(true, true);
0284:
0285: int nKeys = 20; // test ends up inserting 2*nKeys
0286: int nDupsPerKey = 30;
0287:
0288: EnvironmentImpl environment = DbInternal
0289: .envGetEnvironmentImpl(exampleEnv);
0290: FileManager fileManager = environment.getFileManager();
0291:
0292: /* Insert some non dup data, modify, insert dup data. */
0293: Map expectedMap = new HashMap();
0294: doLargePut(expectedMap, nKeys, 1, true);
0295: modifyData(expectedMap, 10, true);
0296: doLargePut(expectedMap, nKeys, nDupsPerKey, true);
0297: checkData(expectedMap);
0298:
0299: /*
0300: * Delete all the data, but abort. (Try to fill up the log
0301: * with entries we don't need.
0302: */
0303: deleteData(expectedMap, false, false);
0304: checkData(expectedMap);
0305:
0306: /* Do some more insertions, but abort them. */
0307: doLargePut(expectedMap, nKeys, nDupsPerKey, false);
0308: checkData(expectedMap);
0309:
0310: /* Do some more insertions and commit them. */
0311: doLargePut(expectedMap, nKeys, nDupsPerKey, true);
0312: checkData(expectedMap);
0313:
0314: /* Checkpoint */
0315: exampleEnv.checkpoint(forceConfig);
0316: checkData(expectedMap);
0317:
0318: /* Clean */
0319: Long lastNum = fileManager.getLastFileNum();
0320: exampleEnv.cleanLog();
0321:
0322: /* Validate after cleaning. */
0323: checkData(expectedMap);
0324: EnvironmentStats stats = exampleEnv
0325: .getStats(TestUtils.FAST_STATS);
0326:
0327: /* Make sure we really cleaned something.*/
0328: assertTrue(stats.getNINsCleaned() > 0);
0329: assertTrue(stats.getNLNsCleaned() > 0);
0330:
0331: closeEnv();
0332: initEnv(false, true);
0333: checkData(expectedMap);
0334: assertTrue(fileManager.getLastFileNum().longValue() > lastNum
0335: .longValue());
0336:
0337: closeEnv();
0338: }
0339:
0340: /**
0341: * Test for SR13191. This SR shows a problem where a MapLN is initialized
0342: * with a DatabaseImpl that has a null EnvironmentImpl. When the Database
0343: * gets used, a NullPointerException occurs in the Cursor code which
0344: * expects there to be an EnvironmentImpl present. The MapLN gets init'd
0345: * by the Cleaner reading through a log file and encountering a MapLN which
0346: * is not presently in the DbTree. As an efficiency, the Cleaner calls
0347: * updateEntry on the BIN to try to insert the MapLN into the BIN so that
0348: * it won't have to fetch it when it migrates the BIN. But this is bad
0349: * since the MapLN has not been init'd properly. The fix was to ensure
0350: * that the MapLN is init'd correctly by calling postFetchInit on it just
0351: * prior to inserting it into the BIN.
0352: *
0353: * This test first creates an environment and two databases. The first
0354: * database it just adds to the tree with no data. This will be the MapLN
0355: * that eventually gets instantiated by the cleaner. The second database
0356: * is used just to create a bunch of data that will get deleted so as to
0357: * create a low utilization for one of the log files. Once the data for
0358: * db2 is created, the log is flipped (so file 0 is the one with the MapLN
0359: * for db1 in it), and the environment is closed and reopened. We insert
0360: * more data into db2 until we have enough .jdb files that file 0 is
0361: * attractive to the cleaner. Call the cleaner to have it instantiate the
0362: * MapLN and then use the MapLN in a Database.get() call.
0363: */
0364: public void testSR13191() throws Throwable {
0365:
0366: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
0367: envConfig.setAllowCreate(true);
0368: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0369: .getName(), "false");
0370: Environment env = new Environment(envHome, envConfig);
0371: EnvironmentImpl envImpl = DbInternal.envGetEnvironmentImpl(env);
0372: FileManager fileManager = DbInternal.envGetEnvironmentImpl(env)
0373: .getFileManager();
0374:
0375: DatabaseConfig dbConfig = new DatabaseConfig();
0376: dbConfig.setAllowCreate(true);
0377: Database db1 = env.openDatabase(null, "db1", dbConfig);
0378:
0379: Database db2 = env.openDatabase(null, "db2", dbConfig);
0380:
0381: DatabaseEntry key = new DatabaseEntry();
0382: DatabaseEntry data = new DatabaseEntry();
0383: IntegerBinding.intToEntry(1, key);
0384: data.setData(new byte[100000]);
0385: for (int i = 0; i < 50; i++) {
0386: assertEquals(OperationStatus.SUCCESS, db2.put(null, key,
0387: data));
0388: }
0389: db1.close();
0390: db2.close();
0391: assertEquals("Should have 0 as current file", 0L, fileManager
0392: .getCurrentFileNum());
0393: envImpl.forceLogFileFlip();
0394: env.close();
0395:
0396: env = new Environment(envHome, envConfig);
0397: fileManager = DbInternal.envGetEnvironmentImpl(env)
0398: .getFileManager();
0399: assertEquals("Should have 1 as current file", 1L, fileManager
0400: .getCurrentFileNum());
0401:
0402: db2 = env.openDatabase(null, "db2", dbConfig);
0403:
0404: for (int i = 0; i < 250; i++) {
0405: assertEquals(OperationStatus.SUCCESS, db2.put(null, key,
0406: data));
0407: }
0408:
0409: db2.close();
0410: env.cleanLog();
0411: db1 = env.openDatabase(null, "db1", dbConfig);
0412: db1.get(null, key, data, null);
0413: db1.close();
0414: env.close();
0415: }
0416:
0417: /**
0418: * Tests that setting je.env.runCleaner=false stops the cleaner from
0419: * processing more files even if the target minUtilization is not met
0420: * [#15158].
0421: */
0422: public void testCleanerStop() throws Throwable {
0423:
0424: final int fileSize = 1000000;
0425: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
0426: envConfig.setAllowCreate(true);
0427: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0428: .getName(), "false");
0429: envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX
0430: .getName(), Integer.toString(fileSize));
0431: envConfig.setConfigParam(
0432: EnvironmentParams.CLEANER_MIN_UTILIZATION.getName(),
0433: "80");
0434: Environment env = new Environment(envHome, envConfig);
0435:
0436: DatabaseConfig dbConfig = new DatabaseConfig();
0437: dbConfig.setAllowCreate(true);
0438: Database db = env.openDatabase(null, "CleanerStop", dbConfig);
0439:
0440: DatabaseEntry key = new DatabaseEntry(new byte[1]);
0441: DatabaseEntry data = new DatabaseEntry(new byte[fileSize]);
0442: for (int i = 0; i <= 10; i += 1) {
0443: db.put(null, key, data);
0444: }
0445: env.checkpoint(forceConfig);
0446:
0447: EnvironmentStats stats = env.getStats(null);
0448: assertEquals(0, stats.getNCleanerRuns());
0449:
0450: envConfig = env.getConfig();
0451: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0452: .getName(), "true");
0453: env.setMutableConfig(envConfig);
0454:
0455: int iter = 0;
0456: while (stats.getNCleanerRuns() == 0) {
0457: iter += 1;
0458: if (iter == 20) {
0459: fail("Cleaner did not run after " + iter + " tries");
0460: }
0461: Thread.yield();
0462: Thread.sleep(1);
0463: stats = env.getStats(null);
0464: }
0465:
0466: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0467: .getName(), "false");
0468: env.setMutableConfig(envConfig);
0469:
0470: int prevNFiles = stats.getNCleanerRuns();
0471: stats = env.getStats(null);
0472: int currNFiles = stats.getNCleanerRuns();
0473: if (currNFiles - prevNFiles > 5) {
0474: fail("Expected less than 5 files cleaned," + " prevNFiles="
0475: + prevNFiles + " currNFiles=" + currNFiles);
0476: }
0477:
0478: //System.out.println("Num runs: " + stats.getNCleanerRuns());
0479:
0480: db.close();
0481: env.close();
0482: }
0483:
0484: /**
0485: * Tests that when a file being cleaned is deleted, we ignore the error and
0486: * don't repeatedly try to clean it. This is happening when we mistakedly
0487: * clean a file after it has been queued for deletion. The workaround is
0488: * to catch LogFileNotFoundException in the cleaner and ignore the error.
0489: * We're testing the workaround here by forcing cleaning of deleted files.
0490: * [#15528]
0491: */
0492: public void testUnexpectedFileDeletion() throws DatabaseException,
0493: IOException {
0494:
0495: initEnv(true, false);
0496: EnvironmentMutableConfig config = exampleEnv.getMutableConfig();
0497: config.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0498: .getName(), "true");
0499: config.setConfigParam(EnvironmentParams.CLEANER_MIN_UTILIZATION
0500: .getName(), "80");
0501: exampleEnv.setMutableConfig(config);
0502:
0503: final EnvironmentImpl envImpl = DbInternal
0504: .envGetEnvironmentImpl(exampleEnv);
0505: final Cleaner cleaner = envImpl.getCleaner();
0506:
0507: Map expectedMap = new HashMap();
0508: doLargePut(expectedMap, 1000, 1, true);
0509: checkData(expectedMap);
0510:
0511: for (int i = 0; i < 100; i += 1) {
0512: modifyData(expectedMap, 1, true);
0513: checkData(expectedMap);
0514: cleaner.injectFileForCleaning(new Long(0));
0515: exampleEnv.cleanLog();
0516: exampleEnv.checkpoint(forceConfig);
0517: }
0518: checkData(expectedMap);
0519:
0520: closeEnv();
0521: }
0522:
0523: /**
0524: * Helper routine. Generates keys with random alpha values while data
0525: * is numbered numerically.
0526: */
0527: private void doLargePut(Map expectedMap, int nKeys,
0528: int nDupsPerKey, boolean commit) throws DatabaseException {
0529:
0530: Transaction txn = exampleEnv.beginTransaction(null, null);
0531: for (int i = 0; i < nKeys; i++) {
0532: byte[] key = new byte[N_KEY_BYTES];
0533: TestUtils.generateRandomAlphaBytes(key);
0534: String keyString = new String(key);
0535:
0536: /*
0537: * The data map is keyed by key value, and holds a hash
0538: * map of all data values.
0539: */
0540: Set dataVals = new HashSet();
0541: if (commit) {
0542: expectedMap.put(keyString, dataVals);
0543: }
0544: for (int j = 0; j < nDupsPerKey; j++) {
0545: String dataString = Integer.toString(j);
0546: exampleDb.put(txn, new StringDbt(keyString),
0547: new StringDbt(dataString));
0548: dataVals.add(dataString);
0549: }
0550: }
0551: if (commit) {
0552: txn.commit();
0553: } else {
0554: txn.abort();
0555: }
0556: }
0557:
0558: /**
0559: * Increment each data value.
0560: */
0561: private void modifyData(Map expectedMap, int increment,
0562: boolean commit) throws DatabaseException {
0563:
0564: Transaction txn = exampleEnv.beginTransaction(null, null);
0565:
0566: StringDbt foundKey = new StringDbt();
0567: StringDbt foundData = new StringDbt();
0568:
0569: Cursor cursor = exampleDb.openCursor(txn, null);
0570: OperationStatus status = cursor.getFirst(foundKey, foundData,
0571: LockMode.DEFAULT);
0572:
0573: boolean toggle = true;
0574: while (status == OperationStatus.SUCCESS) {
0575: if (toggle) {
0576:
0577: String foundKeyString = foundKey.getString();
0578: String foundDataString = foundData.getString();
0579: int newValue = Integer.parseInt(foundDataString)
0580: + increment;
0581: String newDataString = Integer.toString(newValue);
0582:
0583: /* If committing, adjust the expected map. */
0584: if (commit) {
0585:
0586: Set dataVals = (Set) expectedMap
0587: .get(foundKeyString);
0588: if (dataVals == null) {
0589: fail("Couldn't find " + foundKeyString + "/"
0590: + foundDataString);
0591: } else if (dataVals.contains(foundDataString)) {
0592: dataVals.remove(foundDataString);
0593: dataVals.add(newDataString);
0594: } else {
0595: fail("Couldn't find " + foundKeyString + "/"
0596: + foundDataString);
0597: }
0598: }
0599:
0600: assertEquals(OperationStatus.SUCCESS, cursor.delete());
0601: assertEquals(OperationStatus.SUCCESS, cursor.put(
0602: foundKey, new StringDbt(newDataString)));
0603: toggle = false;
0604: } else {
0605: toggle = true;
0606: }
0607:
0608: status = cursor.getNext(foundKey, foundData,
0609: LockMode.DEFAULT);
0610: }
0611:
0612: cursor.close();
0613: if (commit) {
0614: txn.commit();
0615: } else {
0616: txn.abort();
0617: }
0618: }
0619:
0620: /**
0621: * Delete data.
0622: */
0623: private void deleteData(Map expectedMap, boolean everyOther,
0624: boolean commit) throws DatabaseException {
0625:
0626: Transaction txn = exampleEnv.beginTransaction(null, null);
0627:
0628: StringDbt foundKey = new StringDbt();
0629: StringDbt foundData = new StringDbt();
0630:
0631: Cursor cursor = exampleDb.openCursor(txn, null);
0632: OperationStatus status = cursor.getFirst(foundKey, foundData,
0633: LockMode.DEFAULT);
0634:
0635: boolean toggle = true;
0636: while (status == OperationStatus.SUCCESS) {
0637: if (toggle) {
0638:
0639: String foundKeyString = foundKey.getString();
0640: String foundDataString = foundData.getString();
0641:
0642: /* If committing, adjust the expected map */
0643: if (commit) {
0644:
0645: Set dataVals = (Set) expectedMap
0646: .get(foundKeyString);
0647: if (dataVals == null) {
0648: fail("Couldn't find " + foundKeyString + "/"
0649: + foundDataString);
0650: } else if (dataVals.contains(foundDataString)) {
0651: dataVals.remove(foundDataString);
0652: if (dataVals.size() == 0) {
0653: expectedMap.remove(foundKeyString);
0654: }
0655: } else {
0656: fail("Couldn't find " + foundKeyString + "/"
0657: + foundDataString);
0658: }
0659: }
0660:
0661: assertEquals(OperationStatus.SUCCESS, cursor.delete());
0662: }
0663:
0664: if (everyOther) {
0665: toggle = toggle ? false : true;
0666: }
0667:
0668: status = cursor.getNext(foundKey, foundData,
0669: LockMode.DEFAULT);
0670: }
0671:
0672: cursor.close();
0673: if (commit) {
0674: txn.commit();
0675: } else {
0676: txn.abort();
0677: }
0678: }
0679:
0680: /**
0681: * Check what's in the database against what's in the expected map.
0682: */
0683: private void checkData(Map expectedMap) throws DatabaseException {
0684:
0685: StringDbt foundKey = new StringDbt();
0686: StringDbt foundData = new StringDbt();
0687: Cursor cursor = exampleDb.openCursor(null, null);
0688: OperationStatus status = cursor.getFirst(foundKey, foundData,
0689: LockMode.DEFAULT);
0690:
0691: /*
0692: * Make a copy of expectedMap so that we're free to delete out
0693: * of the set of expected results when we verify.
0694: * Also make a set of counts for each key value, to test count.
0695: */
0696:
0697: Map checkMap = new HashMap();
0698: Map countMap = new HashMap();
0699: Iterator iter = expectedMap.entrySet().iterator();
0700: while (iter.hasNext()) {
0701: Map.Entry entry = (Map.Entry) iter.next();
0702: Set copySet = new HashSet();
0703: copySet.addAll((Set) entry.getValue());
0704: checkMap.put(entry.getKey(), copySet);
0705: countMap.put(entry.getKey(), new Integer(copySet.size()));
0706: }
0707:
0708: while (status == OperationStatus.SUCCESS) {
0709: String foundKeyString = foundKey.getString();
0710: String foundDataString = foundData.getString();
0711:
0712: /* Check that the current value is in the check values map */
0713: Set dataVals = (Set) checkMap.get(foundKeyString);
0714: if (dataVals == null) {
0715: fail("Couldn't find " + foundKeyString + "/"
0716: + foundDataString);
0717: } else if (dataVals.contains(foundDataString)) {
0718: dataVals.remove(foundDataString);
0719: if (dataVals.size() == 0) {
0720: checkMap.remove(foundKeyString);
0721: }
0722: } else {
0723: fail("Couldn't find " + foundKeyString + "/"
0724: + foundDataString + " in data vals");
0725: }
0726:
0727: /* Check that the count is right. */
0728: int count = cursor.count();
0729: assertEquals(((Integer) countMap.get(foundKeyString))
0730: .intValue(), count);
0731:
0732: status = cursor.getNext(foundKey, foundData,
0733: LockMode.DEFAULT);
0734: }
0735:
0736: cursor.close();
0737:
0738: if (checkMap.size() != 0) {
0739: dumpExpected(checkMap);
0740: fail("checkMapSize = " + checkMap.size());
0741:
0742: }
0743: assertEquals(0, checkMap.size());
0744: }
0745:
0746: private void dumpExpected(Map expectedMap) {
0747: Iterator iter = expectedMap.entrySet().iterator();
0748: while (iter.hasNext()) {
0749: Map.Entry entry = (Map.Entry) iter.next();
0750: String key = (String) entry.getKey();
0751: Iterator dataIter = ((Set) entry.getValue()).iterator();
0752: while (dataIter.hasNext()) {
0753: System.out.println("key=" + key + " data="
0754: + (String) dataIter.next());
0755: }
0756: }
0757: }
0758:
0759: /**
0760: * Tests that cleaner mutable configuration parameters can be changed and
0761: * that the changes actually take effect.
0762: */
0763: public void testMutableConfig() throws DatabaseException {
0764:
0765: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
0766: envConfig.setAllowCreate(true);
0767: exampleEnv = new Environment(envHome, envConfig);
0768: envConfig = exampleEnv.getConfig();
0769: EnvironmentImpl envImpl = DbInternal
0770: .envGetEnvironmentImpl(exampleEnv);
0771: Cleaner cleaner = envImpl.getCleaner();
0772: UtilizationProfile profile = envImpl.getUtilizationProfile();
0773: MemoryBudget budget = envImpl.getMemoryBudget();
0774: String name;
0775: String val;
0776:
0777: /* je.cleaner.minUtilization */
0778: name = EnvironmentParams.CLEANER_MIN_UTILIZATION.getName();
0779: setParam(name, "33");
0780: assertEquals(33, profile.minUtilization);
0781:
0782: /* je.cleaner.minFileUtilization */
0783: name = EnvironmentParams.CLEANER_MIN_FILE_UTILIZATION.getName();
0784: setParam(name, "7");
0785: assertEquals(7, profile.minFileUtilization);
0786:
0787: /* je.cleaner.bytesInterval */
0788: name = EnvironmentParams.CLEANER_BYTES_INTERVAL.getName();
0789: setParam(name, "1000");
0790: assertEquals(1000, cleaner.cleanerBytesInterval);
0791:
0792: /* je.cleaner.deadlockRetry */
0793: name = EnvironmentParams.CLEANER_DEADLOCK_RETRY.getName();
0794: setParam(name, "7");
0795: assertEquals(7, cleaner.nDeadlockRetries);
0796:
0797: /* je.cleaner.lockTimeout */
0798: name = EnvironmentParams.CLEANER_LOCK_TIMEOUT.getName();
0799: setParam(name, "7000");
0800: assertEquals(7, cleaner.lockTimeout);
0801:
0802: /* je.cleaner.expunge */
0803: name = EnvironmentParams.CLEANER_REMOVE.getName();
0804: val = "false".equals(envConfig.getConfigParam(name)) ? "true"
0805: : "false";
0806: setParam(name, val);
0807: assertEquals(val.equals("true"), cleaner.expunge);
0808:
0809: /* je.cleaner.minAge */
0810: name = EnvironmentParams.CLEANER_MIN_AGE.getName();
0811: setParam(name, "7");
0812: assertEquals(7, profile.minAge);
0813:
0814: /* je.cleaner.cluster */
0815: name = EnvironmentParams.CLEANER_CLUSTER.getName();
0816: val = "false".equals(envConfig.getConfigParam(name)) ? "true"
0817: : "false";
0818: setParam(name, val);
0819: assertEquals(val.equals("true"), cleaner.clusterResident);
0820: /* Cannot set both cluster and clusterAll to true. */
0821: setParam(name, "false");
0822:
0823: /* je.cleaner.clusterAll */
0824: name = EnvironmentParams.CLEANER_CLUSTER_ALL.getName();
0825: val = "false".equals(envConfig.getConfigParam(name)) ? "true"
0826: : "false";
0827: setParam(name, val);
0828: assertEquals(val.equals("true"), cleaner.clusterAll);
0829:
0830: /* je.cleaner.maxBatchFiles */
0831: name = EnvironmentParams.CLEANER_MAX_BATCH_FILES.getName();
0832: setParam(name, "7");
0833: assertEquals(7, cleaner.maxBatchFiles);
0834:
0835: /* je.cleaner.readSize */
0836: name = EnvironmentParams.CLEANER_READ_SIZE.getName();
0837: setParam(name, "7777");
0838: assertEquals(7777, cleaner.readBufferSize);
0839:
0840: /* je.cleaner.detailMaxMemoryPercentage */
0841: name = EnvironmentParams.CLEANER_DETAIL_MAX_MEMORY_PERCENTAGE
0842: .getName();
0843: setParam(name, "7");
0844: assertEquals((budget.getMaxMemory() * 7) / 100, budget
0845: .getTrackerBudget());
0846:
0847: /* je.cleaner.threads */
0848: name = EnvironmentParams.CLEANER_THREADS.getName();
0849: setParam(name, "7");
0850: assertEquals((envImpl.isNoLocking() ? 0 : 7),
0851: countCleanerThreads());
0852:
0853: exampleEnv.close();
0854: exampleEnv = null;
0855: }
0856:
0857: /**
0858: * Sets a mutable config param, checking that the given value is not
0859: * already set and that it actually changes.
0860: */
0861: private void setParam(String name, String val)
0862: throws DatabaseException {
0863:
0864: EnvironmentMutableConfig config = exampleEnv.getMutableConfig();
0865: String myVal = config.getConfigParam(name);
0866: assertTrue(!val.equals(myVal));
0867:
0868: config.setConfigParam(name, val);
0869: exampleEnv.setMutableConfig(config);
0870:
0871: config = exampleEnv.getMutableConfig();
0872: myVal = config.getConfigParam(name);
0873: assertTrue(val.equals(myVal));
0874: }
0875:
0876: /**
0877: * Count the number of threads with the name "Cleaner#".
0878: */
0879: private int countCleanerThreads() {
0880:
0881: Thread[] threads = new Thread[Thread.activeCount()];
0882: Thread.enumerate(threads);
0883:
0884: int count = 0;
0885: for (int i = 0; i < threads.length; i += 1) {
0886: if (threads[i] != null
0887: && threads[i].getName().startsWith("Cleaner")) {
0888: count += 1;
0889: }
0890: }
0891:
0892: return count;
0893: }
0894:
0895: /**
0896: * Checks that the memory budget is updated properly by the
0897: * UtilizationTracker. Prior to a bug fix [#15505] amounts were added to
0898: * the budget but not subtracted when two TrackedFileSummary objects were
0899: * merged. Merging occurs when a local tracker is added to the global
0900: * tracker. Local trackers are used during recovery, checkpoints, lazy
0901: * compression, and reverse splits.
0902: */
0903: public void testTrackerMemoryBudget() throws DatabaseException {
0904:
0905: /* Open environment. */
0906: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
0907: envConfig.setAllowCreate(true);
0908: envConfig.setTransactional(true);
0909: envConfig.setConfigParam(EnvironmentParams.ENV_CHECK_LEAKS
0910: .getName(), "false");
0911: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0912: .getName(), "false");
0913: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_INCOMPRESSOR
0914: .getName(), "false");
0915: exampleEnv = new Environment(envHome, envConfig);
0916: EnvironmentImpl envImpl = DbInternal
0917: .envGetEnvironmentImpl(exampleEnv);
0918: MemoryBudget budget = envImpl.getMemoryBudget();
0919:
0920: /* Open database. */
0921: DatabaseConfig dbConfig = new DatabaseConfig();
0922: dbConfig.setTransactional(true);
0923: dbConfig.setAllowCreate(true);
0924: exampleDb = exampleEnv.openDatabase(null, "foo", dbConfig);
0925:
0926: /* Insert data. */
0927: DatabaseEntry key = new DatabaseEntry();
0928: DatabaseEntry data = new DatabaseEntry();
0929: for (int i = 1; i <= 200; i += 1) {
0930: IntegerBinding.intToEntry(i, key);
0931: IntegerBinding.intToEntry(i, data);
0932: exampleDb.put(null, key, data);
0933: }
0934:
0935: /* Sav the misc budget baseline. */
0936: flushTrackedFiles();
0937: long misc = budget.getMiscMemoryUsage();
0938:
0939: /*
0940: * Nothing becomes obsolete when inserting and no INs are logged, so
0941: * the budget does not increase.
0942: */
0943: IntegerBinding.intToEntry(201, key);
0944: exampleDb.put(null, key, data);
0945: assertEquals(misc, budget.getMiscMemoryUsage());
0946: flushTrackedFiles();
0947: assertEquals(misc, budget.getMiscMemoryUsage());
0948:
0949: /*
0950: * Update a record and expect the budget to increase because the old
0951: * LN becomes obsolete.
0952: */
0953: exampleDb.put(null, key, data);
0954: assertTrue(misc < budget.getMiscMemoryUsage());
0955: flushTrackedFiles();
0956: assertEquals(misc, budget.getMiscMemoryUsage());
0957:
0958: /*
0959: * Delete all records and expect the budget to increase because LNs
0960: * become obsolete.
0961: */
0962: for (int i = 1; i <= 201; i += 1) {
0963: IntegerBinding.intToEntry(i, key);
0964: exampleDb.delete(null, key);
0965: }
0966: assertTrue(misc < budget.getMiscMemoryUsage());
0967: flushTrackedFiles();
0968: assertEquals(misc, budget.getMiscMemoryUsage());
0969:
0970: /*
0971: * Compress and expect no change to the budget. Prior to the fix for
0972: * [#15505] the assertion below failed because the baseline misc budget
0973: * was not restored.
0974: */
0975: exampleEnv.compress();
0976: flushTrackedFiles();
0977: assertEquals(misc, budget.getMiscMemoryUsage());
0978:
0979: closeEnv();
0980: }
0981:
0982: /**
0983: * Flushes all tracked files to subtract tracked info from the misc memory
0984: * budget.
0985: */
0986: private void flushTrackedFiles() throws DatabaseException {
0987:
0988: EnvironmentImpl envImpl = DbInternal
0989: .envGetEnvironmentImpl(exampleEnv);
0990: UtilizationTracker tracker = envImpl.getUtilizationTracker();
0991: UtilizationProfile profile = envImpl.getUtilizationProfile();
0992:
0993: TrackedFileSummary[] files = tracker.getTrackedFiles();
0994: for (int i = 0; i < files.length; i += 1) {
0995: profile.flushFileSummary(files[i]);
0996: }
0997: }
0998:
0999: /**
1000: * Tests that memory is budgeted correctly for FileSummaryLNs that are
1001: * inserted and deleted after calling setTrackedSummary. The size of the
1002: * FileSummaryLN changes during logging when setTrackedSummary is called,
1003: * and this is accounted for specially in Tree.logLNAfterInsert. [#15831]
1004: */
1005: public void testFileSummaryLNMemoryUsage() throws DatabaseException {
1006:
1007: /* Open environment, prevent concurrent access by daemons. */
1008: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
1009: envConfig.setAllowCreate(true);
1010: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
1011: .getName(), "false");
1012: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CHECKPOINTER
1013: .getName(), "false");
1014: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_INCOMPRESSOR
1015: .getName(), "false");
1016: exampleEnv = new Environment(envHome, envConfig);
1017:
1018: EnvironmentImpl envImpl = DbInternal
1019: .envGetEnvironmentImpl(exampleEnv);
1020: DatabaseImpl fileSummaryDb = envImpl.getUtilizationProfile()
1021: .getFileSummaryDb();
1022: MemoryBudget memBudget = envImpl.getMemoryBudget();
1023:
1024: BasicLocker locker = null;
1025: CursorImpl cursor = null;
1026: try {
1027: locker = new BasicLocker(envImpl);
1028: cursor = new CursorImpl(fileSummaryDb, locker);
1029:
1030: /* Get parent BIN. There should be only one BIN in the tree. */
1031: IN root = fileSummaryDb.getTree()
1032: .getRootIN(true /*updateGeneration*/);
1033: root.releaseLatch();
1034: assertEquals(1, root.getNEntries());
1035: BIN parent = (BIN) root.getTarget(0);
1036:
1037: /* Use an artificial FileSummaryLN with a tracked summary. */
1038: FileSummaryLN ln = new FileSummaryLN(new FileSummary());
1039: TrackedFileSummary tfs = new TrackedFileSummary(envImpl
1040: .getUtilizationTracker(), 0 /*fileNum*/, true /*trackDetail*/);
1041: tfs.trackObsolete(0);
1042: byte[] keyBytes = FileSummaryLN.makeFullKey(0 /*fileNum*/,
1043: 123 /*sequence*/);
1044: int keySize = MemoryBudget.byteArraySize(keyBytes.length);
1045:
1046: /* Perform insert after calling setTrackedSummary. */
1047: long oldSize = ln.getMemorySizeIncludedByParent();
1048: long oldParentSize = parent.getInMemorySize();
1049: ln.setTrackedSummary(tfs);
1050: OperationStatus status = cursor
1051: .putLN(keyBytes, ln, false /*allowDuplicates*/);
1052: assertSame(status, OperationStatus.SUCCESS);
1053: long newSize = ln.getMemorySizeIncludedByParent();
1054: long newParentSize = parent.getInMemorySize();
1055:
1056: /* The size of the LN increases during logging. */
1057: assertEquals(newSize, oldSize
1058: + ln.getObsoleteOffsets().getExtraMemorySize());
1059:
1060: /* The correct size is accounted for by the parent BIN. */
1061: assertEquals(newSize + keySize, newParentSize
1062: - oldParentSize);
1063:
1064: /* Correct size is subtracted during eviction. */
1065: oldParentSize = newParentSize;
1066: cursor.evict();
1067: newParentSize = parent.getInMemorySize();
1068: assertEquals(oldParentSize - newSize, newParentSize);
1069:
1070: /* Fetch a fresh FileSummaryLN before deleting it. */
1071: oldParentSize = newParentSize;
1072: ln = (FileSummaryLN) cursor.getCurrentLN(LockType.READ);
1073: newSize = ln.getMemorySizeIncludedByParent();
1074: newParentSize = parent.getInMemorySize();
1075: assertEquals(newSize, newParentSize - oldParentSize);
1076:
1077: /* Perform delete after calling setTrackedSummary. */
1078: oldSize = newSize;
1079: oldParentSize = newParentSize;
1080: ln.setTrackedSummary(tfs);
1081: status = cursor.delete();
1082: assertSame(status, OperationStatus.SUCCESS);
1083: newSize = ln.getMemorySizeIncludedByParent();
1084: newParentSize = parent.getInMemorySize();
1085:
1086: /* Size changes during delete also. */
1087: assertTrue(newSize < oldSize);
1088: assertTrue(oldSize - newSize > ln.getObsoleteOffsets()
1089: .getExtraMemorySize());
1090: assertEquals(newSize - oldSize, newParentSize
1091: - oldParentSize);
1092: } finally {
1093: if (cursor != null) {
1094: cursor.releaseBINs();
1095: cursor.close();
1096: }
1097: if (locker != null) {
1098: locker.operationEnd();
1099: }
1100: }
1101:
1102: closeEnv();
1103: }
1104: }
|