0001: /*-
0002: * See the file LICENSE for redistribution information.
0003: *
0004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
0005: *
0006: * $Id: TruncateAndRemoveTest.java,v 1.18.2.4 2008/01/07 15:14:25 cwl Exp $
0007: */
0008:
0009: package com.sleepycat.je.cleaner;
0010:
0011: import java.io.File;
0012: import java.io.IOException;
0013: import java.util.HashSet;
0014: import java.util.Iterator;
0015: import java.util.Set;
0016: import java.nio.ByteBuffer;
0017:
0018: import junit.framework.TestCase;
0019:
0020: import com.sleepycat.je.CheckpointConfig;
0021: import com.sleepycat.je.Cursor;
0022: import com.sleepycat.je.Database;
0023: import com.sleepycat.je.DatabaseConfig;
0024: import com.sleepycat.je.DatabaseEntry;
0025: import com.sleepycat.je.DatabaseException;
0026: import com.sleepycat.je.DbInternal;
0027: import com.sleepycat.je.Environment;
0028: import com.sleepycat.je.EnvironmentConfig;
0029: import com.sleepycat.je.LockMode;
0030: import com.sleepycat.je.OperationStatus;
0031: import com.sleepycat.je.Transaction;
0032: import com.sleepycat.je.config.EnvironmentParams;
0033: import com.sleepycat.je.dbi.DatabaseId;
0034: import com.sleepycat.je.dbi.DatabaseImpl;
0035: import com.sleepycat.je.junit.JUnitThread;
0036: import com.sleepycat.je.log.DumpFileReader;
0037: import com.sleepycat.je.log.FileManager;
0038: import com.sleepycat.je.log.LogEntryType;
0039: import com.sleepycat.je.log.entry.INLogEntry;
0040: import com.sleepycat.je.log.entry.LNLogEntry;
0041: import com.sleepycat.je.log.entry.LogEntry;
0042: import com.sleepycat.je.util.TestUtils;
0043: import com.sleepycat.je.utilint.DbLsn;
0044: import com.sleepycat.je.utilint.TestHook;
0045:
0046: public class TruncateAndRemoveTest extends TestCase {
0047:
0048: private static final String DB_NAME1 = "foo";
0049: private static final String DB_NAME2 = "bar";
0050: private static final long RECORD_COUNT = 100;
0051:
0052: private static final CheckpointConfig FORCE_CHECKPOINT = new CheckpointConfig();
0053: static {
0054: FORCE_CHECKPOINT.setForce(true);
0055: }
0056:
0057: private static final boolean DEBUG = false;
0058:
0059: private File envHome;
0060: private Environment env;
0061: private Database db;
0062: private DatabaseImpl dbImpl;
0063: private JUnitThread junitThread;
0064: private boolean fetchObsoleteSize;
0065: private boolean dbEviction;
0066:
0067: public TruncateAndRemoveTest() {
0068: envHome = new File(System.getProperty(TestUtils.DEST_DIR));
0069: }
0070:
0071: public void setUp() throws IOException, DatabaseException {
0072:
0073: TestUtils.removeLogFiles("Setup", envHome, false);
0074: TestUtils.removeFiles("Setup", envHome, FileManager.DEL_SUFFIX);
0075: }
0076:
0077: public void tearDown() throws IOException, DatabaseException {
0078:
0079: if (junitThread != null) {
0080: while (junitThread.isAlive()) {
0081: junitThread.interrupt();
0082: Thread.yield();
0083: }
0084: junitThread = null;
0085: }
0086:
0087: try {
0088: if (env != null) {
0089: env.close();
0090: }
0091: } catch (Throwable e) {
0092: System.out.println("tearDown: " + e);
0093: }
0094:
0095: try {
0096: //*
0097: TestUtils.removeLogFiles("tearDown", envHome, true);
0098: TestUtils.removeFiles("tearDown", envHome,
0099: FileManager.DEL_SUFFIX);
0100: //*/
0101: } catch (Throwable e) {
0102: System.out.println("tearDown: " + e);
0103: }
0104:
0105: db = null;
0106: dbImpl = null;
0107: env = null;
0108: envHome = null;
0109: }
0110:
0111: /**
0112: * Opens the environment.
0113: */
0114: private void openEnv(boolean transactional)
0115: throws DatabaseException {
0116:
0117: EnvironmentConfig config = TestUtils.initEnvConfig();
0118: config.setTransactional(transactional);
0119: config.setAllowCreate(true);
0120: /* Do not run the daemons since they interfere with LN counting. */
0121: config.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
0122: .getName(), "false");
0123: config.setConfigParam(EnvironmentParams.ENV_RUN_EVICTOR
0124: .getName(), "false");
0125: config.setConfigParam(EnvironmentParams.ENV_RUN_CHECKPOINTER
0126: .getName(), "false");
0127: config.setConfigParam(EnvironmentParams.ENV_RUN_INCOMPRESSOR
0128: .getName(), "false");
0129:
0130: /* Use small nodes to test the post-txn scanning. */
0131: config.setConfigParam(EnvironmentParams.NODE_MAX.getName(),
0132: "10");
0133: config.setConfigParam(EnvironmentParams.NODE_MAX_DUPTREE
0134: .getName(), "10");
0135:
0136: /* Use small files to ensure that there is cleaning. */
0137: config.setConfigParam("je.cleaner.minUtilization", "90");
0138: DbInternal.disableParameterValidation(config);
0139: config.setConfigParam("je.log.fileMax", "4000");
0140:
0141: /* Obsolete LN size counting is optional per test. */
0142: if (fetchObsoleteSize) {
0143: config.setConfigParam(
0144: EnvironmentParams.CLEANER_FETCH_OBSOLETE_SIZE
0145: .getName(), "true");
0146: }
0147:
0148: env = new Environment(envHome, config);
0149:
0150: config = env.getConfig();
0151: dbEviction = config.getConfigParam(
0152: EnvironmentParams.ENV_DB_EVICTION.getName()).equals(
0153: "true");
0154: }
0155:
0156: /**
0157: * Opens that database.
0158: */
0159: private void openDb(Transaction useTxn, String dbName)
0160: throws DatabaseException {
0161:
0162: DatabaseConfig dbConfig = new DatabaseConfig();
0163: EnvironmentConfig envConfig = env.getConfig();
0164: dbConfig.setTransactional(envConfig.getTransactional());
0165: dbConfig.setAllowCreate(true);
0166: db = env.openDatabase(useTxn, dbName, dbConfig);
0167: dbImpl = DbInternal.dbGetDatabaseImpl(db);
0168: }
0169:
0170: /**
0171: * Closes the database.
0172: */
0173: private void closeDb() throws DatabaseException {
0174:
0175: if (db != null) {
0176: db.close();
0177: db = null;
0178: dbImpl = null;
0179: }
0180: }
0181:
0182: /**
0183: * Closes the environment and database.
0184: */
0185: private void closeEnv() throws DatabaseException {
0186:
0187: closeDb();
0188:
0189: if (env != null) {
0190: env.close();
0191: env = null;
0192: }
0193: }
0194:
0195: /**
0196: * Test that truncate generates the right number of obsolete LNs.
0197: */
0198: public void testTruncate() throws Exception {
0199:
0200: openEnv(true);
0201: openDb(null, DB_NAME1);
0202: writeAndCountRecords(null, RECORD_COUNT);
0203: DatabaseImpl saveDb = dbImpl;
0204: DatabaseId saveId = dbImpl.getId();
0205: closeDb();
0206:
0207: Transaction txn = env.beginTransaction(null, null);
0208: truncate(txn, true);
0209: ObsoleteCounts beforeCommit = getObsoleteCounts();
0210: txn.commit();
0211:
0212: /* Make sure use count is decremented when we commit. */
0213: assertDbInUse(saveDb, false);
0214: openDb(null, DB_NAME1);
0215: saveDb = dbImpl;
0216: closeDb();
0217: assertDbInUse(saveDb, false);
0218:
0219: verifyUtilization(beforeCommit, RECORD_COUNT + // LNs
0220: 3, // prev MapLN + deleted MapLN + prev NameLN
0221: 15); // 1 root, 2 INs, 12 BINs
0222:
0223: closeEnv();
0224: batchCleanAndVerify(saveId);
0225: }
0226:
0227: /**
0228: * Test that aborting truncate generates the right number of obsolete LNs.
0229: */
0230: public void testTruncateAbort() throws Exception {
0231:
0232: openEnv(true);
0233: openDb(null, DB_NAME1);
0234: writeAndCountRecords(null, RECORD_COUNT);
0235: DatabaseImpl saveDb = dbImpl;
0236: closeDb();
0237:
0238: Transaction txn = env.beginTransaction(null, null);
0239: truncate(txn, true);
0240: ObsoleteCounts beforeAbort = getObsoleteCounts();
0241: txn.abort();
0242:
0243: /* Make sure use count is decremented when we abort. */
0244: assertDbInUse(saveDb, false);
0245: openDb(null, DB_NAME1);
0246: saveDb = dbImpl;
0247: closeDb();
0248: assertDbInUse(saveDb, false);
0249:
0250: /*
0251: * The obsolete count should include the records inserted after
0252: * the truncate.
0253: */
0254: verifyUtilization(beforeAbort,
0255: /* 1 new nameLN, 2 copies of MapLN for new db */
0256: 3, 0);
0257:
0258: /* Reopen, db should be populated. */
0259: openDb(null, DB_NAME1);
0260: assertEquals(RECORD_COUNT, countRecords(null));
0261: closeEnv();
0262: }
0263:
0264: /**
0265: * Test that aborting truncate generates the right number of obsolete LNs.
0266: */
0267: public void testTruncateRepopulateAbort() throws Exception {
0268:
0269: openEnv(true);
0270: openDb(null, DB_NAME1);
0271: writeAndCountRecords(null, RECORD_COUNT);
0272: closeDb();
0273:
0274: Transaction txn = env.beginTransaction(null, null);
0275: truncate(txn, true);
0276:
0277: /* populate the database with some more records. */
0278: openDb(txn, DB_NAME1);
0279: writeAndCountRecords(txn, RECORD_COUNT / 4);
0280: DatabaseImpl saveDb = dbImpl;
0281: DatabaseId saveId = dbImpl.getId();
0282: closeDb();
0283: ObsoleteCounts beforeAbort = getObsoleteCounts();
0284: txn.abort();
0285:
0286: /* Make sure use count is decremented when we abort. */
0287: assertDbInUse(saveDb, false);
0288: openDb(null, DB_NAME1);
0289: saveDb = dbImpl;
0290: closeDb();
0291: assertDbInUse(saveDb, false);
0292:
0293: /*
0294: * The obsolete count should include the records inserted after
0295: * the truncate.
0296: */
0297: verifyUtilization(beforeAbort,
0298: /* newly inserted LNs, 1 new nameLN,
0299: * 2 copies of MapLN for new db */
0300: (RECORD_COUNT / 4) + 3, 5);
0301:
0302: /* Reopen, db should be populated. */
0303: openDb(null, DB_NAME1);
0304: assertEquals(RECORD_COUNT, countRecords(null));
0305:
0306: closeEnv();
0307: batchCleanAndVerify(saveId);
0308: }
0309:
0310: /**
0311: * Test that remove generates the right number of obsolete LNs.
0312: */
0313: public void testRemove() throws Exception {
0314:
0315: openEnv(true);
0316: openDb(null, DB_NAME1);
0317: writeAndCountRecords(null, RECORD_COUNT);
0318: DatabaseImpl saveDb = dbImpl;
0319: DatabaseId saveId = dbImpl.getId();
0320: closeDb();
0321:
0322: Transaction txn = env.beginTransaction(null, null);
0323: env.removeDatabase(txn, DB_NAME1);
0324: ObsoleteCounts beforeCommit = getObsoleteCounts();
0325: txn.commit();
0326:
0327: /* Make sure use count is decremented when we commit. */
0328: assertDbInUse(saveDb, false);
0329:
0330: verifyUtilization(beforeCommit,
0331: /* LNs + old NameLN, old MapLN, delete MapLN */
0332: RECORD_COUNT + 3, 15);
0333:
0334: openDb(null, DB_NAME1);
0335: assertEquals(0, countRecords(null));
0336:
0337: closeEnv();
0338: batchCleanAndVerify(saveId);
0339: }
0340:
0341: /**
0342: * Test that remove generates the right number of obsolete LNs.
0343: */
0344: public void testNonTxnalRemove() throws Exception {
0345:
0346: openEnv(false);
0347: openDb(null, DB_NAME1);
0348: writeAndCountRecords(null, RECORD_COUNT);
0349: DatabaseImpl saveDb = dbImpl;
0350: DatabaseId saveId = dbImpl.getId();
0351: closeDb();
0352: ObsoleteCounts beforeOperation = getObsoleteCounts();
0353: env.removeDatabase(null, DB_NAME1);
0354:
0355: /* Make sure use count is decremented. */
0356: assertDbInUse(saveDb, false);
0357:
0358: verifyUtilization(beforeOperation,
0359: /* LNs + new NameLN, old NameLN, old MapLN, delete
0360: MapLN */
0361: RECORD_COUNT + 4, 15);
0362:
0363: openDb(null, DB_NAME1);
0364: assertEquals(0, countRecords(null));
0365:
0366: closeEnv();
0367: batchCleanAndVerify(saveId);
0368: }
0369:
0370: /**
0371: * Test that aborting remove generates the right number of obsolete LNs.
0372: */
0373: public void testRemoveAbort() throws Exception {
0374:
0375: /* Create database, populate, remove, abort the remove. */
0376: openEnv(true);
0377: openDb(null, DB_NAME1);
0378: writeAndCountRecords(null, RECORD_COUNT);
0379: DatabaseImpl saveDb = dbImpl;
0380: closeDb();
0381: Transaction txn = env.beginTransaction(null, null);
0382: env.removeDatabase(txn, DB_NAME1);
0383: ObsoleteCounts beforeAbort = getObsoleteCounts();
0384: txn.abort();
0385:
0386: /* Make sure use count is decremented when we abort. */
0387: assertDbInUse(saveDb, false);
0388:
0389: verifyUtilization(beforeAbort, 0, 0);
0390:
0391: /* All records should be there. */
0392: openDb(null, DB_NAME1);
0393: assertEquals(RECORD_COUNT, countRecords(null));
0394:
0395: closeEnv();
0396:
0397: /*
0398: * Batch clean and then check the record count again, just to make sure
0399: * we don't lose any valid data.
0400: */
0401: openEnv(true);
0402: while (env.cleanLog() > 0) {
0403: }
0404: CheckpointConfig force = new CheckpointConfig();
0405: force.setForce(true);
0406: env.checkpoint(force);
0407: closeEnv();
0408:
0409: openEnv(true);
0410: openDb(null, DB_NAME1);
0411: assertEquals(RECORD_COUNT, countRecords(null));
0412: closeEnv();
0413: }
0414:
0415: /**
0416: * The same as testRemoveNotResident but forces fetching of obsolets LNs
0417: * in order to count their sizes accurately.
0418: */
0419: public void testRemoveNotResidentFetchObsoleteSize()
0420: throws Exception {
0421:
0422: fetchObsoleteSize = true;
0423: testRemoveNotResident();
0424: }
0425:
0426: /**
0427: * Test that we can properly account for a non-resident database.
0428: */
0429: public void testRemoveNotResident() throws Exception {
0430:
0431: /* Create a database, populate. */
0432: openEnv(true);
0433: openDb(null, DB_NAME1);
0434: writeAndCountRecords(null, RECORD_COUNT);
0435: DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
0436: closeEnv();
0437:
0438: /*
0439: * Open the environment and remove the database. The
0440: * database is not resident at all.
0441: */
0442: openEnv(true);
0443: Transaction txn = env.beginTransaction(null, null);
0444: env.removeDatabase(txn, DB_NAME1);
0445: ObsoleteCounts beforeCommit = getObsoleteCounts();
0446: txn.commit();
0447:
0448: verifyUtilization(beforeCommit,
0449: /* LNs + old NameLN, old MapLN, delete MapLN */
0450: RECORD_COUNT + 3,
0451: /* 15 IN for data tree,
0452: 2 for re-logged INs */
0453: 15 + 2, true);
0454:
0455: /* check record count. */
0456: openDb(null, DB_NAME1);
0457: assertEquals(0, countRecords(null));
0458:
0459: closeEnv();
0460: batchCleanAndVerify(saveId);
0461: }
0462:
0463: /**
0464: * The same as testRemovePartialResident but forces fetching of obsolets
0465: * LNs in order to count their sizes accurately.
0466: */
0467: public void testRemovePartialResidentFetchObsoleteSize()
0468: throws Exception {
0469:
0470: fetchObsoleteSize = true;
0471: testRemovePartialResident();
0472: }
0473:
0474: /**
0475: * Test that we can properly account for partially resident tree.
0476: */
0477: public void testRemovePartialResident() throws Exception {
0478:
0479: /* Create a database, populate. */
0480: openEnv(true);
0481: openDb(null, DB_NAME1);
0482: writeAndCountRecords(null, RECORD_COUNT);
0483: DatabaseId saveId = DbInternal.dbGetDatabaseImpl(db).getId();
0484: closeEnv();
0485:
0486: /*
0487: * Open the environment and remove the database. Pull 1 BIN in.
0488: */
0489: openEnv(true);
0490: openDb(null, DB_NAME1);
0491: Cursor c = db.openCursor(null, null);
0492: assertEquals(OperationStatus.SUCCESS, c.getFirst(
0493: new DatabaseEntry(), new DatabaseEntry(),
0494: LockMode.DEFAULT));
0495: c.close();
0496: DatabaseImpl saveDb = dbImpl;
0497: closeDb();
0498:
0499: Transaction txn = env.beginTransaction(null, null);
0500: env.removeDatabase(txn, DB_NAME1);
0501: ObsoleteCounts beforeCommit = getObsoleteCounts();
0502: txn.commit();
0503:
0504: /* Make sure use count is decremented when we commit. */
0505: assertDbInUse(saveDb, false);
0506:
0507: verifyUtilization(beforeCommit,
0508: /* LNs + old NameLN, old MapLN, delete MapLN */
0509: RECORD_COUNT + 3,
0510: /* 15 IN for data tree, 2 for file summary db */
0511: 15 + 2, true);
0512:
0513: /* check record count. */
0514: openDb(null, DB_NAME1);
0515: assertEquals(0, countRecords(null));
0516:
0517: closeEnv();
0518: batchCleanAndVerify(saveId);
0519: }
0520:
0521: /**
0522: * Tests that a log file is not deleted by the cleaner when it contains
0523: * entries in a database that is pending deletion.
0524: */
0525: public void testDBPendingDeletion() throws DatabaseException,
0526: InterruptedException {
0527:
0528: doDBPendingTest(RECORD_COUNT, false /*deleteAll*/, 7);
0529: }
0530:
0531: /**
0532: * Like testDBPendingDeletion but creates a scenario where only a single
0533: * log file is cleaned, and that log file contains only known obsolete
0534: * log entries. This reproduced a bug where we neglected to adding
0535: * pending deleted DBs to the cleaner's pending DB set if all entries in
0536: * the log file were known obsoleted. [#13333]
0537: */
0538: public void testObsoleteLogFile() throws DatabaseException,
0539: InterruptedException {
0540:
0541: doDBPendingTest(40, true /*deleteAll*/, 1);
0542: }
0543:
0544: private void doDBPendingTest(long recordCount, boolean deleteAll,
0545: int expectFilesCleaned) throws DatabaseException,
0546: InterruptedException {
0547:
0548: /* Create a database, populate, close. */
0549: Set logFiles = new HashSet();
0550: openEnv(true);
0551: openDb(null, DB_NAME1);
0552: writeAndMakeWaste(recordCount, logFiles, deleteAll);
0553: long remainingRecordCount = deleteAll ? 0 : recordCount;
0554: env.checkpoint(FORCE_CHECKPOINT);
0555: ObsoleteCounts obsoleteCounts = getObsoleteCounts();
0556: DatabaseImpl saveDb = dbImpl;
0557: closeDb();
0558: assertTrue(!saveDb.isDeleteFinished());
0559: assertTrue(!saveDb.isDeleted());
0560: assertDbInUse(saveDb, false);
0561:
0562: /* Make sure that we wrote a full file's worth of LNs. */
0563: assertTrue(logFiles.size() >= 3);
0564: assertTrue(logFilesExist(logFiles));
0565:
0566: /* Remove the database but do not commit yet. */
0567: final Transaction txn = env.beginTransaction(null, null);
0568: env.removeDatabase(txn, DB_NAME1);
0569:
0570: /* The obsolete count should be <= 1 (for the NameLN). */
0571: obsoleteCounts = verifyUtilization(obsoleteCounts, 1, 0);
0572:
0573: junitThread = new JUnitThread("Committer") {
0574: public void testBody() throws DatabaseException {
0575: try {
0576: txn.commit();
0577: } catch (Throwable e) {
0578: e.printStackTrace();
0579: }
0580: }
0581: };
0582:
0583: /*
0584: * Set a hook to cause the commit to block. The commit is done in a
0585: * separate thread. The commit will set the DB state to pendingDeleted
0586: * and will then wait for the hook to return.
0587: */
0588: final Object lock = new Object();
0589:
0590: saveDb.setPendingDeletedHook(new TestHook() {
0591: public void doIOHook() throws IOException {
0592: throw new UnsupportedOperationException();
0593: }
0594:
0595: public void doHook() {
0596: synchronized (lock) {
0597: try {
0598: lock.notify();
0599: lock.wait();
0600: } catch (InterruptedException e) {
0601: e.printStackTrace();
0602: throw new RuntimeException(e.toString());
0603: }
0604: }
0605: }
0606:
0607: public Object getHookValue() {
0608: return null;
0609: }
0610: });
0611:
0612: /* Start the committer thread; expect the pending deleted state. */
0613: synchronized (lock) {
0614: junitThread.start();
0615: lock.wait();
0616: }
0617: assertTrue(!saveDb.isDeleteFinished());
0618: assertTrue(saveDb.isDeleted());
0619: assertDbInUse(saveDb, true);
0620:
0621: /* Expect obsolete LNs: NameLN */
0622: obsoleteCounts = verifyUtilization(obsoleteCounts, 1, 0);
0623:
0624: /* The DB deletion is pending; the log file should still exist. */
0625: int filesCleaned = env.cleanLog();
0626: assertEquals(expectFilesCleaned, filesCleaned);
0627: assertTrue(filesCleaned > 0);
0628: env.checkpoint(FORCE_CHECKPOINT);
0629: env.checkpoint(FORCE_CHECKPOINT);
0630: assertTrue(logFilesExist(logFiles));
0631:
0632: /*
0633: * When the commiter thread finishes, the DB deletion will be
0634: * complete and the DB state will change to deleted.
0635: */
0636: synchronized (lock) {
0637: lock.notify();
0638: }
0639: try {
0640: junitThread.finishTest();
0641: junitThread = null;
0642: } catch (Throwable e) {
0643: e.printStackTrace();
0644: fail(e.toString());
0645: }
0646: assertTrue(saveDb.isDeleteFinished());
0647: assertTrue(saveDb.isDeleted());
0648: assertDbInUse(saveDb, false);
0649:
0650: /* Expect obsolete LNs: recordCount + MapLN + FSLNs (apprx). */
0651: verifyUtilization(obsoleteCounts, remainingRecordCount + 6, 0);
0652:
0653: /* The DB deletion is complete; the log file should be deleted. */
0654: env.checkpoint(FORCE_CHECKPOINT);
0655: env.checkpoint(FORCE_CHECKPOINT);
0656: assertTrue(!logFilesExist(logFiles));
0657: }
0658:
0659: private void writeAndCountRecords(Transaction txn, long count)
0660: throws DatabaseException {
0661:
0662: for (int i = 1; i <= count; i += 1) {
0663: DatabaseEntry entry = new DatabaseEntry(TestUtils
0664: .getTestArray(i));
0665:
0666: db.put(txn, entry, entry);
0667: }
0668:
0669: /* Insert and delete some records, insert and abort some records. */
0670: DatabaseEntry entry = new DatabaseEntry(TestUtils
0671: .getTestArray((int) count + 1));
0672: db.put(txn, entry, entry);
0673: db.delete(txn, entry);
0674:
0675: EnvironmentConfig envConfig = env.getConfig();
0676: if (envConfig.getTransactional()) {
0677: entry = new DatabaseEntry(TestUtils.getTestArray(0));
0678: Transaction txn2 = env.beginTransaction(null, null);
0679: db.put(txn2, entry, entry);
0680: txn2.abort();
0681: txn2 = null;
0682: }
0683:
0684: assertEquals(count, countRecords(txn));
0685: }
0686:
0687: /**
0688: * Writes the specified number of records to db. Check the number of
0689: * records, and return the number of obsolete records. Returns a set of
0690: * the file numbers that are written to.
0691: *
0692: * Makes waste (obsolete records): If doDelete=true, deletes records as
0693: * they are added; otherwise does updates to produce obsolete records
0694: * interleaved with non-obsolete records.
0695: */
0696: private void writeAndMakeWaste(long count, Set logFilesWritten,
0697: boolean doDelete) throws DatabaseException {
0698:
0699: Transaction txn = env.beginTransaction(null, null);
0700: Cursor cursor = db.openCursor(txn, null);
0701: for (int i = 0; i < count; i += 1) {
0702: DatabaseEntry entry = new DatabaseEntry(TestUtils
0703: .getTestArray(i));
0704: cursor.put(entry, entry);
0705: /* Add log file written. */
0706: long file = CleanerTestUtils.getLogFile(this , cursor);
0707: logFilesWritten.add(new Long(file));
0708: /* Make waste. */
0709: if (!doDelete) {
0710: cursor.put(entry, entry);
0711: cursor.put(entry, entry);
0712: }
0713: }
0714: if (doDelete) {
0715: DatabaseEntry key = new DatabaseEntry();
0716: DatabaseEntry data = new DatabaseEntry();
0717: OperationStatus status;
0718: for (status = cursor.getFirst(key, data, null); status == OperationStatus.SUCCESS; status = cursor
0719: .getNext(key, data, null)) {
0720: /* Make waste. */
0721: cursor.delete();
0722: /* Add log file written. */
0723: long file = CleanerTestUtils.getLogFile(this , cursor);
0724: logFilesWritten.add(new Long(file));
0725: }
0726: }
0727: cursor.close();
0728: txn.commit();
0729: assertEquals(doDelete ? 0 : count, countRecords(null));
0730: }
0731:
0732: /* Truncate database and check the count. */
0733: private void truncate(Transaction useTxn, boolean getCount)
0734: throws DatabaseException {
0735:
0736: long nTruncated = env.truncateDatabase(useTxn, DB_NAME1,
0737: getCount);
0738:
0739: if (getCount) {
0740: assertEquals(RECORD_COUNT, nTruncated);
0741: }
0742:
0743: assertEquals(0, countRecords(useTxn));
0744: }
0745:
0746: /**
0747: * Returns how many records are in the database.
0748: */
0749: private int countRecords(Transaction useTxn)
0750: throws DatabaseException {
0751:
0752: DatabaseEntry key = new DatabaseEntry();
0753: DatabaseEntry data = new DatabaseEntry();
0754: boolean opened = false;
0755: if (db == null) {
0756: openDb(useTxn, DB_NAME1);
0757: opened = true;
0758: }
0759: Cursor cursor = db.openCursor(useTxn, null);
0760: int count = 0;
0761: try {
0762: OperationStatus status = cursor.getFirst(key, data, null);
0763: while (status == OperationStatus.SUCCESS) {
0764: count += 1;
0765: status = cursor.getNext(key, data, null);
0766: }
0767: } finally {
0768: cursor.close();
0769: }
0770: if (opened) {
0771: closeDb();
0772: }
0773: return count;
0774: }
0775:
0776: /**
0777: * Return the total number of obsolete node counts according to the
0778: * UtilizationProfile and UtilizationTracker.
0779: */
0780: private ObsoleteCounts getObsoleteCounts() throws DatabaseException {
0781:
0782: FileSummary[] files = (FileSummary[]) DbInternal
0783: .envGetEnvironmentImpl(env).getUtilizationProfile()
0784: .getFileSummaryMap(true).values().toArray(
0785: new FileSummary[0]);
0786: int lnCount = 0;
0787: int inCount = 0;
0788: int lnSize = 0;
0789: int lnSizeCounted = 0;
0790: for (int i = 0; i < files.length; i += 1) {
0791: lnCount += files[i].obsoleteLNCount;
0792: inCount += files[i].obsoleteINCount;
0793: lnSize += files[i].obsoleteLNSize;
0794: lnSizeCounted += files[i].obsoleteLNSizeCounted;
0795: }
0796:
0797: return new ObsoleteCounts(lnCount, inCount, lnSize,
0798: lnSizeCounted);
0799: }
0800:
0801: private class ObsoleteCounts {
0802: int obsoleteLNs;
0803: int obsoleteINs;
0804: int obsoleteLNSize;
0805: int obsoleteLNSizeCounted;
0806:
0807: ObsoleteCounts(int obsoleteLNs, int obsoleteINs,
0808: int obsoleteLNSize, int obsoleteLNSizeCounted) {
0809: this .obsoleteLNs = obsoleteLNs;
0810: this .obsoleteINs = obsoleteINs;
0811: this .obsoleteLNSize = obsoleteLNSize;
0812: this .obsoleteLNSizeCounted = obsoleteLNSizeCounted;
0813: }
0814:
0815: public String toString() {
0816: return "lns=" + obsoleteLNs + " ins=" + obsoleteINs
0817: + " lnSize=" + obsoleteLNSize + " lnSizeCounted="
0818: + obsoleteLNSizeCounted;
0819: }
0820: }
0821:
0822: private ObsoleteCounts verifyUtilization(ObsoleteCounts prev,
0823: long expectedLNs, int expectedINs) throws DatabaseException {
0824:
0825: return verifyUtilization(prev, expectedLNs, expectedINs, false);
0826: }
0827:
0828: /*
0829: * Check obsolete counts. If the expected IN count is zero, don't
0830: * check the obsolete IN count. Always check the obsolete LN count.
0831: */
0832: private ObsoleteCounts verifyUtilization(ObsoleteCounts prev,
0833: long expectedLNs, int expectedINs, boolean expectNonResident)
0834: throws DatabaseException {
0835:
0836: /*
0837: * If all nodes are resident OR we have explicitly configured
0838: * fetchObsoleteSize, then the size of every LN should have been
0839: * counted.
0840: */
0841: boolean expectAccurateObsoleteLNSize = !expectNonResident
0842: || fetchObsoleteSize;
0843:
0844: ObsoleteCounts now = getObsoleteCounts();
0845: String beforeAndAfter = "before: " + prev + " now: " + now;
0846: if (DEBUG) {
0847: System.out.println(beforeAndAfter);
0848: }
0849:
0850: assertEquals(beforeAndAfter, expectedLNs, now.obsoleteLNs
0851: - prev.obsoleteLNs);
0852: if (expectedLNs > 0) {
0853: int size = now.obsoleteLNSize - prev.obsoleteLNSize;
0854: int counted = now.obsoleteLNSizeCounted
0855: - prev.obsoleteLNSizeCounted;
0856: assertTrue(String.valueOf(size), size > 0);
0857:
0858: if (expectAccurateObsoleteLNSize) {
0859: assertEquals(beforeAndAfter, counted, now.obsoleteLNs
0860: - prev.obsoleteLNs);
0861: }
0862: }
0863: if (expectedINs > 0) {
0864: assertEquals(beforeAndAfter, expectedINs, now.obsoleteINs
0865: - prev.obsoleteINs);
0866: }
0867:
0868: /* Verify utilization using UtilizationFileReader. */
0869: CleanerTestUtils.verifyUtilization(DbInternal
0870: .envGetEnvironmentImpl(env), true, // expectAccurateObsoleteLNCount
0871: expectAccurateObsoleteLNSize);
0872:
0873: return now;
0874: }
0875:
0876: /**
0877: * Checks whether a given DB has a non-zero use count. Does nothing if
0878: * je.dbEviction is not enabled, since reference counts are only maintained
0879: * if that config parameter is enabled.
0880: */
0881: private void assertDbInUse(DatabaseImpl db, boolean inUse) {
0882: if (dbEviction) {
0883: assertEquals(inUse, db.isInUse());
0884: }
0885: }
0886:
0887: /**
0888: * Returns true if all files exist, or false if any file is deleted.
0889: */
0890: private boolean logFilesExist(Set fileNumbers) {
0891:
0892: Iterator iter = fileNumbers.iterator();
0893: while (iter.hasNext()) {
0894: long fileNum = ((Long) iter.next()).longValue();
0895: File file = new File(envHome, FileManager.getFileName(
0896: fileNum, FileManager.JE_SUFFIX));
0897: if (!file.exists()) {
0898: return false;
0899: }
0900: }
0901: return true;
0902: }
0903:
0904: /*
0905: * Run batch cleaning and verify that there are no files with these
0906: * log entries.
0907: */
0908: private void batchCleanAndVerify(DatabaseId dbId) throws Exception {
0909:
0910: /*
0911: * Open the environment, flip the log files to reduce mixing of new
0912: * records and old records and add more records to force the
0913: * utilization level of the removed records down.
0914: */
0915: openEnv(true);
0916: openDb(null, DB_NAME2);
0917: long lsn = DbInternal.envGetEnvironmentImpl(env)
0918: .forceLogFileFlip();
0919: CheckpointConfig force = new CheckpointConfig();
0920: force.setForce(true);
0921: env.checkpoint(force);
0922:
0923: writeAndCountRecords(null, RECORD_COUNT * 3);
0924: env.checkpoint(force);
0925:
0926: closeDb();
0927:
0928: /* Check log files, there should be entries with this database. */
0929: CheckReader checker = new CheckReader(env, dbId, true);
0930: while (checker.readNextEntry()) {
0931: }
0932:
0933: if (DEBUG) {
0934: System.out.println("entries for this db ="
0935: + checker.getCount());
0936: }
0937:
0938: assertTrue(checker.getCount() > 0);
0939:
0940: /* batch clean. */
0941: boolean anyCleaned = false;
0942: while (env.cleanLog() > 0) {
0943: anyCleaned = true;
0944: }
0945:
0946: assertTrue(anyCleaned);
0947:
0948: if (anyCleaned) {
0949: env.checkpoint(force);
0950: }
0951:
0952: /* Check log files, there should be no entries with this database. */
0953: checker = new CheckReader(env, dbId, false);
0954: while (checker.readNextEntry()) {
0955: }
0956:
0957: closeEnv();
0958:
0959: }
0960:
0961: class CheckReader extends DumpFileReader {
0962:
0963: private DatabaseId dbId;
0964: private boolean expectEntries;
0965: private int count;
0966:
0967: /*
0968: * @param databaseId we're looking for log entries for this database.
0969: * @param expectEntries if false, there should be no log entries
0970: * with this database id. If true, the log should have entries
0971: * with this database id.
0972: */
0973: CheckReader(Environment env, DatabaseId dbId,
0974: boolean expectEntries) throws DatabaseException,
0975: IOException {
0976:
0977: super (DbInternal.envGetEnvironmentImpl(env), 1000,
0978: DbLsn.NULL_LSN, DbLsn.NULL_LSN, null, null, false);
0979: this .dbId = dbId;
0980: this .expectEntries = expectEntries;
0981: }
0982:
0983: protected boolean processEntry(ByteBuffer entryBuffer)
0984: throws DatabaseException {
0985:
0986: /* Figure out what kind of log entry this is */
0987: byte type = currentEntryHeader.getType();
0988: byte version = currentEntryHeader.getVersion();
0989: LogEntryType lastEntryType = LogEntryType.findType(type,
0990: version);
0991: boolean isNode = LogEntryType.isNodeType(type, version);
0992:
0993: /* Read the entry. */
0994: LogEntry entry = lastEntryType.getSharedLogEntry();
0995: entry.readEntry(currentEntryHeader, entryBuffer, true); // readFullItem
0996:
0997: long lsn = getLastLsn();
0998: if (isNode) {
0999: boolean found = false;
1000: if (entry instanceof INLogEntry) {
1001: INLogEntry inEntry = (INLogEntry) entry;
1002: found = dbId.equals(inEntry.getDbId());
1003: } else {
1004: LNLogEntry lnEntry = (LNLogEntry) entry;
1005: found = dbId.equals(lnEntry.getDbId());
1006: }
1007: if (found) {
1008: if (expectEntries) {
1009: count++;
1010: } else {
1011: StringBuffer sb = new StringBuffer();
1012: entry.dumpEntry(sb, false);
1013: fail("lsn=" + DbLsn.getNoFormatString(lsn)
1014: + " dbId = " + dbId + " entry= "
1015: + sb.toString());
1016: }
1017: }
1018: }
1019:
1020: return true;
1021: }
1022:
1023: /* Num entries with this database id seen by reader. */
1024: int getCount() {
1025: return count;
1026: }
1027: }
1028: }
|