001: /*-
002: * See the file LICENSE for redistribution information.
003: *
004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
005: *
006: * $Id: DbScavengerTest.java,v 1.16.2.3 2008/01/07 15:14:34 cwl Exp $
007: */
008:
009: package com.sleepycat.je.util;
010:
011: import java.io.BufferedReader;
012: import java.io.File;
013: import java.io.FileInputStream;
014: import java.io.FilenameFilter;
015: import java.io.IOException;
016: import java.io.InputStreamReader;
017: import java.io.RandomAccessFile;
018: import java.util.HashMap;
019: import java.util.HashSet;
020: import java.util.Iterator;
021: import java.util.Map;
022: import java.util.Set;
023: import java.util.StringTokenizer;
024:
025: import junit.framework.TestCase;
026:
027: import com.sleepycat.bind.tuple.IntegerBinding;
028: import com.sleepycat.je.Cursor;
029: import com.sleepycat.je.Database;
030: import com.sleepycat.je.DatabaseConfig;
031: import com.sleepycat.je.DatabaseEntry;
032: import com.sleepycat.je.DatabaseException;
033: import com.sleepycat.je.DatabaseNotFoundException;
034: import com.sleepycat.je.DbInternal;
035: import com.sleepycat.je.Environment;
036: import com.sleepycat.je.EnvironmentConfig;
037: import com.sleepycat.je.OperationStatus;
038: import com.sleepycat.je.Transaction;
039: import com.sleepycat.je.config.EnvironmentParams;
040: import com.sleepycat.je.log.FileManager;
041: import com.sleepycat.je.utilint.DbLsn;
042: import com.sleepycat.je.utilint.DbScavenger;
043:
044: public class DbScavengerTest extends TestCase {
045:
046: private static final int TRANSACTIONAL = 1 << 0;
047: private static final int WRITE_MULTIPLE = 1 << 1;
048: private static final int PRINTABLE = 1 << 2;
049: private static final int ABORT_BEFORE = 1 << 3;
050: private static final int ABORT_AFTER = 1 << 4;
051: private static final int CORRUPT_LOG = 1 << 5;
052: private static final int DELETE_DATA = 1 << 6;
053: private static final int AGGRESSIVE = 1 << 7;
054:
055: private static final int N_DBS = 3;
056: private static final int N_KEYS = 100;
057: private static final int N_DATA_BYTES = 100;
058: private static final int LOG_SIZE = 10000;
059:
060: private String envHomeName;
061: private File envHome;
062:
063: private Environment env;
064:
065: private Database[] dbs = new Database[N_DBS];
066:
067: private boolean duplicatesAllowed = true;
068:
069: public DbScavengerTest() {
070: envHomeName = System.getProperty(TestUtils.DEST_DIR);
071: envHome = new File(envHomeName);
072: }
073:
074: public void setUp() throws IOException {
075:
076: TestUtils.removeLogFiles("Setup", envHome, false);
077: TestUtils.removeFiles("Setup", envHome, ".dump");
078: }
079:
080: public void tearDown() throws IOException {
081:
082: if (env != null) {
083: try {
084: env.close();
085: } catch (Exception e) {
086: System.out.println("TearDown: " + e);
087: }
088: env = null;
089: }
090: TestUtils.removeLogFiles("TearDown", envHome, false);
091: TestUtils.removeFiles("Teardown", envHome, ".dump");
092: }
093:
094: public void testScavenger1() throws Throwable {
095:
096: try {
097: doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_BEFORE
098: | ABORT_AFTER);
099: } catch (Throwable T) {
100: System.out.println("caught " + T);
101: T.printStackTrace();
102: }
103: }
104:
105: public void testScavenger2() throws Throwable {
106:
107: try {
108: doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_BEFORE);
109: } catch (Throwable T) {
110: System.out.println("caught " + T);
111: T.printStackTrace();
112: }
113: }
114:
115: public void testScavenger3() throws Throwable {
116:
117: try {
118: doScavengerTest(PRINTABLE | TRANSACTIONAL | ABORT_AFTER);
119: } catch (Throwable T) {
120: System.out.println("caught " + T);
121: T.printStackTrace();
122: }
123: }
124:
125: public void testScavenger4() throws Throwable {
126:
127: try {
128: doScavengerTest(PRINTABLE | TRANSACTIONAL);
129: } catch (Throwable T) {
130: System.out.println("caught " + T);
131: T.printStackTrace();
132: }
133: }
134:
135: public void testScavenger5() throws Throwable {
136:
137: try {
138: doScavengerTest(PRINTABLE | WRITE_MULTIPLE | TRANSACTIONAL);
139: } catch (Throwable T) {
140: System.out.println("caught " + T);
141: T.printStackTrace();
142: }
143: }
144:
145: public void testScavenger6() throws Throwable {
146:
147: try {
148: doScavengerTest(PRINTABLE);
149: } catch (Throwable T) {
150: System.out.println("caught " + T);
151: T.printStackTrace();
152: throw T;
153: }
154: }
155:
156: public void testScavenger7() throws Throwable {
157:
158: try {
159: doScavengerTest(TRANSACTIONAL | ABORT_BEFORE | ABORT_AFTER);
160: } catch (Throwable T) {
161: System.out.println("caught " + T);
162: T.printStackTrace();
163: }
164: }
165:
166: public void testScavenger8() throws Throwable {
167:
168: try {
169: doScavengerTest(TRANSACTIONAL | ABORT_BEFORE);
170: } catch (Throwable T) {
171: System.out.println("caught " + T);
172: T.printStackTrace();
173: }
174: }
175:
176: public void testScavenger9() throws Throwable {
177:
178: try {
179: doScavengerTest(TRANSACTIONAL);
180: } catch (Throwable T) {
181: System.out.println("caught " + T);
182: T.printStackTrace();
183: }
184: }
185:
186: public void testScavenger10() throws Throwable {
187:
188: try {
189: doScavengerTest(TRANSACTIONAL | ABORT_AFTER);
190: } catch (Throwable T) {
191: System.out.println("caught " + T);
192: T.printStackTrace();
193: }
194: }
195:
196: public void testScavenger11() throws Throwable {
197:
198: try {
199: doScavengerTest(0);
200: } catch (Throwable T) {
201: System.out.println("caught " + T);
202: T.printStackTrace();
203: }
204: }
205:
206: public void testScavenger12() throws Throwable {
207:
208: try {
209: doScavengerTest(CORRUPT_LOG);
210: } catch (Throwable T) {
211: System.out.println("caught " + T);
212: T.printStackTrace();
213: }
214: }
215:
216: public void testScavenger13() throws Throwable {
217:
218: try {
219: doScavengerTest(DELETE_DATA);
220: } catch (Throwable T) {
221: System.out.println("caught " + T);
222: T.printStackTrace();
223: }
224: }
225:
226: public void testScavenger14() throws Throwable {
227:
228: try {
229: doScavengerTest(AGGRESSIVE);
230: } catch (Throwable T) {
231: System.out.println("caught " + T);
232: T.printStackTrace();
233: }
234: }
235:
236: public void testScavengerAbortedDbLevelOperations()
237: throws Throwable {
238:
239: try {
240: createEnv(true, true);
241: boolean doAbort = true;
242: byte[] dataBytes = new byte[N_DATA_BYTES];
243: DatabaseEntry key = new DatabaseEntry();
244: DatabaseEntry data = new DatabaseEntry(dataBytes);
245: IntegerBinding.intToEntry(1, key);
246: TestUtils.generateRandomAlphaBytes(dataBytes);
247: for (int i = 0; i < 2; i++) {
248: Transaction txn = env.beginTransaction(null, null);
249: for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
250: String databaseName = null;
251: if (doAbort) {
252: databaseName = "abortedDb" + dbCnt;
253: } else {
254: databaseName = "simpleDb" + dbCnt;
255: }
256: DatabaseConfig dbConfig = new DatabaseConfig();
257: dbConfig.setAllowCreate(true);
258: dbConfig.setSortedDuplicates(duplicatesAllowed);
259: dbConfig.setTransactional(true);
260: if (dbs[dbCnt] != null) {
261: throw new DatabaseException(
262: "database already open");
263: }
264: Database db = env.openDatabase(txn, databaseName,
265: dbConfig);
266: dbs[dbCnt] = db;
267: db.put(txn, key, data);
268: }
269: if (doAbort) {
270: txn.abort();
271: dbs = new Database[N_DBS];
272: } else {
273: txn.commit();
274: }
275: doAbort = !doAbort;
276: }
277:
278: closeEnv();
279: createEnv(false, false);
280: openDbs(false, false, duplicatesAllowed, null);
281: dumpDbs(false, false);
282:
283: /* Close the environment, delete it completely from the disk. */
284: closeEnv();
285: TestUtils.removeLogFiles("doScavengerTest", envHome, false);
286:
287: /* Recreate and reload the environment from the scavenger files. */
288: createEnv(true, true);
289: loadDbs();
290:
291: /* Verify that the data is the same as when it was created. */
292: for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
293: String databaseName = "abortedDb" + dbCnt;
294: DatabaseConfig dbConfig = new DatabaseConfig();
295: dbConfig.setAllowCreate(false);
296: try {
297: env.openDatabase(null, databaseName, dbConfig);
298: fail("expected DatabaseNotFoundException");
299: } catch (DatabaseNotFoundException DNFE) {
300: /* Expected. */
301: }
302: }
303: closeEnv();
304:
305: } catch (Throwable T) {
306: System.out.println("caught " + T);
307: T.printStackTrace();
308: }
309: }
310:
311: private void doScavengerTest(int config) throws DatabaseException,
312: IOException {
313:
314: boolean printable = (config & PRINTABLE) != 0;
315: boolean transactional = (config & TRANSACTIONAL) != 0;
316: boolean writeMultiple = (config & WRITE_MULTIPLE) != 0;
317: boolean abortBefore = (config & ABORT_BEFORE) != 0;
318: boolean abortAfter = (config & ABORT_AFTER) != 0;
319: boolean corruptLog = (config & CORRUPT_LOG) != 0;
320: boolean deleteData = (config & DELETE_DATA) != 0;
321: boolean aggressive = (config & AGGRESSIVE) != 0;
322:
323: assert transactional || (!abortBefore && !abortAfter);
324:
325: Map[] dataMaps = new Map[N_DBS];
326: Set lsnsToCorrupt = new HashSet();
327: /* Create the environment and some data. */
328: createEnvAndDbs(dataMaps, writeMultiple, transactional,
329: abortBefore, abortAfter, corruptLog, lsnsToCorrupt,
330: deleteData);
331: closeEnv();
332: createEnv(false, false);
333: if (corruptLog) {
334: corruptFiles(lsnsToCorrupt);
335: }
336: openDbs(false, false, duplicatesAllowed, null);
337: dumpDbs(printable, aggressive);
338:
339: /* Close the environment, delete it completely from the disk. */
340: closeEnv();
341: TestUtils.removeLogFiles("doScavengerTest", envHome, false);
342:
343: /* Recreate the environment and load it from the scavenger files. */
344: createEnv(true, transactional);
345: loadDbs();
346:
347: /* Verify that the data is the same as when it was created. */
348: openDbs(false, false, duplicatesAllowed, null);
349: verifyDbs(dataMaps);
350: closeEnv();
351: }
352:
353: private void closeEnv() throws DatabaseException {
354:
355: for (int i = 0; i < N_DBS; i++) {
356: if (dbs[i] != null) {
357: dbs[i].close();
358: dbs[i] = null;
359: }
360: }
361:
362: env.close();
363: env = null;
364: }
365:
366: private void createEnv(boolean create, boolean transactional)
367: throws DatabaseException {
368:
369: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
370: DbInternal.disableParameterValidation(envConfig);
371: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
372: .getName(), "false");
373: envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX
374: .getName(), "" + LOG_SIZE);
375: envConfig.setTransactional(transactional);
376: envConfig.setAllowCreate(create);
377: env = new Environment(envHome, envConfig);
378: }
379:
380: private void createEnvAndDbs(Map[] dataMaps, boolean writeMultiple,
381: boolean transactional, boolean abortBefore,
382: boolean abortAfter, boolean corruptLog, Set lsnsToCorrupt,
383: boolean deleteData) throws DatabaseException {
384:
385: createEnv(true, transactional);
386: Transaction txn = null;
387: if (transactional) {
388: txn = env.beginTransaction(null, null);
389: }
390:
391: openDbs(true, transactional, duplicatesAllowed, txn);
392:
393: if (transactional) {
394: txn.commit();
395: }
396:
397: long lastCorruptedFile = -1;
398: for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
399: Map dataMap = new HashMap();
400: dataMaps[dbCnt] = dataMap;
401: Database db = dbs[dbCnt];
402:
403: for (int i = 0; i < N_KEYS; i++) {
404: byte[] dataBytes = new byte[N_DATA_BYTES];
405: DatabaseEntry key = new DatabaseEntry();
406: DatabaseEntry data = new DatabaseEntry(dataBytes);
407: IntegerBinding.intToEntry(i, key);
408: TestUtils.generateRandomAlphaBytes(dataBytes);
409:
410: boolean corruptedThisEntry = false;
411:
412: if (transactional) {
413: txn = env.beginTransaction(null, null);
414: }
415:
416: if (transactional && abortBefore) {
417: assertEquals(OperationStatus.SUCCESS, db.put(txn,
418: key, data));
419: txn.abort();
420: txn = env.beginTransaction(null, null);
421: }
422:
423: assertEquals(OperationStatus.SUCCESS, db.put(txn, key,
424: data));
425: if (corruptLog) {
426: long currentLsn = getLastLsn();
427: long fileNumber = DbLsn.getFileNumber(currentLsn);
428: long fileOffset = DbLsn.getFileOffset(currentLsn);
429: if (fileOffset > (LOG_SIZE >> 1) &&
430: /* We're writing in the second half of the file. */
431: fileNumber > lastCorruptedFile) {
432: /* Corrupt this file. */
433: lsnsToCorrupt.add(new Long(currentLsn));
434: lastCorruptedFile = fileNumber;
435: corruptedThisEntry = true;
436: }
437: }
438:
439: if (writeMultiple) {
440: assertEquals(OperationStatus.SUCCESS, db.delete(
441: txn, key));
442: assertEquals(OperationStatus.SUCCESS, db.put(txn,
443: key, data));
444: }
445:
446: if (deleteData) {
447: assertEquals(OperationStatus.SUCCESS, db.delete(
448: txn, key));
449: /* overload this for deleted data. */
450: corruptedThisEntry = true;
451: }
452:
453: if (!corruptedThisEntry) {
454: dataMap.put(new Integer(i), new String(dataBytes));
455: }
456:
457: if (transactional) {
458: txn.commit();
459: }
460:
461: if (transactional && abortAfter) {
462: txn = env.beginTransaction(null, null);
463: assertEquals(OperationStatus.SUCCESS, db.put(txn,
464: key, data));
465: txn.abort();
466: }
467: }
468: }
469: }
470:
471: private void openDbs(boolean create, boolean transactional,
472: boolean duplicatesAllowed, Transaction txn)
473: throws DatabaseException {
474:
475: for (int dbCnt = 0; dbCnt < N_DBS; dbCnt++) {
476: String databaseName = "simpleDb" + dbCnt;
477: DatabaseConfig dbConfig = new DatabaseConfig();
478: dbConfig.setAllowCreate(create);
479: dbConfig.setSortedDuplicates(duplicatesAllowed);
480: dbConfig.setTransactional(transactional);
481: if (dbs[dbCnt] != null) {
482: throw new DatabaseException("database already open");
483: }
484: dbs[dbCnt] = env.openDatabase(txn, databaseName, dbConfig);
485: }
486: }
487:
488: private void dumpDbs(boolean printable, boolean aggressive)
489: throws DatabaseException {
490:
491: try {
492: DbScavenger scavenger = new DbScavenger(env, null,
493: envHomeName, printable, aggressive, false /* verbose */);
494: scavenger.dump();
495: } catch (IOException IOE) {
496: throw new DatabaseException(IOE);
497: }
498: }
499:
500: private void loadDbs() throws DatabaseException {
501:
502: try {
503: String dbNameBase = "simpleDb";
504: for (int i = 0; i < N_DBS; i++) {
505: DbLoad loader = new DbLoad();
506: File file = new File(envHomeName, dbNameBase + i
507: + ".dump");
508: FileInputStream is = new FileInputStream(file);
509: BufferedReader reader = new BufferedReader(
510: new InputStreamReader(is));
511: loader.setEnv(env);
512: loader.setInputReader(reader);
513: loader.setNoOverwrite(false);
514: loader.setDbName(dbNameBase + i);
515: loader.load();
516: is.close();
517: }
518: } catch (IOException IOE) {
519: throw new DatabaseException(IOE);
520: }
521: }
522:
523: private void verifyDbs(Map[] dataMaps) throws DatabaseException {
524:
525: for (int i = 0; i < N_DBS; i++) {
526: Map dataMap = dataMaps[i];
527: Cursor cursor = dbs[i].openCursor(null, null);
528: DatabaseEntry key = new DatabaseEntry();
529: DatabaseEntry data = new DatabaseEntry();
530: while (cursor.getNext(key, data, null) == OperationStatus.SUCCESS) {
531: Integer keyInt = new Integer(IntegerBinding
532: .entryToInt(key));
533: String databaseString = new String(data.getData());
534: String originalString = (String) dataMap.get(keyInt);
535: if (originalString == null) {
536: fail("couldn't find " + keyInt);
537: } else if (databaseString.equals(originalString)) {
538: dataMap.remove(keyInt);
539: } else {
540: fail(" Mismatch: key=" + keyInt + " Expected: "
541: + originalString + " Found: "
542: + databaseString);
543: }
544: }
545:
546: if (dataMap.size() > 0) {
547: fail("entries still remain");
548: }
549:
550: cursor.close();
551: }
552: }
553:
554: private static DumpFileFilter dumpFileFilter = new DumpFileFilter();
555:
556: static class DumpFileFilter implements FilenameFilter {
557:
558: /**
559: * Accept files of this format:
560: * *.dump
561: */
562: public boolean accept(File dir, String name) {
563: StringTokenizer tokenizer = new StringTokenizer(name, ".");
564: /* There should be two parts. */
565: if (tokenizer.countTokens() == 2) {
566: String fileName = tokenizer.nextToken();
567: String fileSuffix = tokenizer.nextToken();
568:
569: /* Check the length and the suffix. */
570: if (fileSuffix.equals("dump")) {
571: return true;
572: }
573: }
574:
575: return false;
576: }
577: }
578:
579: private long getLastLsn() throws DatabaseException {
580:
581: return DbInternal.envGetEnvironmentImpl(env).getFileManager()
582: .getLastUsedLsn();
583: }
584:
585: private void corruptFiles(Set lsnsToCorrupt)
586: throws DatabaseException {
587:
588: Iterator iter = lsnsToCorrupt.iterator();
589: while (iter.hasNext()) {
590: long lsn = ((Long) iter.next()).longValue();
591: corruptFile(DbLsn.getFileNumber(lsn), DbLsn
592: .getFileOffset(lsn));
593: }
594: }
595:
596: private void corruptFile(long fileNumber, long fileOffset)
597: throws DatabaseException {
598:
599: String fileName = DbInternal.envGetEnvironmentImpl(env)
600: .getFileManager().getFullFileName(fileNumber,
601: FileManager.JE_SUFFIX);
602: /*
603: System.out.println("corrupting 1 byte at " +
604: DbLsn.makeLsn(fileNumber, fileOffset));
605: */
606: try {
607: RandomAccessFile raf = new RandomAccessFile(fileName, "rw");
608: raf.seek(fileOffset);
609: int current = raf.read();
610: raf.seek(fileOffset);
611: raf.write(current + 1);
612: raf.close();
613: } catch (IOException IOE) {
614: throw new DatabaseException(IOE);
615: }
616: }
617: }
|