001: /*-
002: * See the file LICENSE for redistribution information.
003: *
004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
005: *
006: * $Id: LastFileReaderTest.java,v 1.67.2.3 2008/01/07 15:14:29 cwl Exp $
007: */
008:
009: package com.sleepycat.je.log;
010:
011: import java.io.File;
012: import java.io.IOException;
013: import java.io.RandomAccessFile;
014: import java.util.ArrayList;
015: import java.util.List;
016:
017: import junit.framework.TestCase;
018:
019: import com.sleepycat.je.DatabaseException;
020: import com.sleepycat.je.DbInternal;
021: import com.sleepycat.je.EnvironmentConfig;
022: import com.sleepycat.je.config.EnvironmentParams;
023: import com.sleepycat.je.dbi.DbConfigManager;
024: import com.sleepycat.je.dbi.EnvironmentImpl;
025: import com.sleepycat.je.log.entry.SingleItemEntry;
026: import com.sleepycat.je.txn.TxnAbort;
027: import com.sleepycat.je.util.BadFileFilter;
028: import com.sleepycat.je.util.TestUtils;
029: import com.sleepycat.je.utilint.DbLsn;
030: import com.sleepycat.je.utilint.Tracer;
031:
032: public class LastFileReaderTest extends TestCase {
033:
034: private DbConfigManager configManager;
035: private FileManager fileManager;
036: private LogManager logManager;
037: private File envHome;
038: private EnvironmentImpl envImpl;
039:
040: public LastFileReaderTest() {
041: super ();
042: envHome = new File(System.getProperty(TestUtils.DEST_DIR));
043: }
044:
045: public void setUp() throws DatabaseException, IOException {
046:
047: TestUtils.removeFiles("Setup", envHome, FileManager.JE_SUFFIX);
048: TestUtils.removeFiles(envHome, new BadFileFilter());
049: }
050:
051: public void tearDown() throws DatabaseException, IOException {
052:
053: /*
054: * Pass false to skip checkpoint, since the file manager may hold an
055: * open file that we've trashed in the tests, so we don't want to
056: * write to it here.
057: */
058: try {
059: envImpl.close(false);
060: } catch (DatabaseException e) {
061: }
062:
063: TestUtils.removeFiles("TearDown", envHome,
064: FileManager.JE_SUFFIX);
065: TestUtils.removeFiles(envHome, new BadFileFilter());
066: }
067:
068: /* Create an environment, using the default log file size. */
069: private void initEnv() throws DatabaseException {
070:
071: initEnv(null);
072: }
073:
074: /* Create an environment, specifying the log file size. */
075: private void initEnv(String logFileSize) throws DatabaseException {
076:
077: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
078:
079: /* Don't run daemons; we do some abrupt shutdowns. */
080: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
081: .getName(), "false");
082: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CHECKPOINTER
083: .getName(), "false");
084: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_EVICTOR
085: .getName(), "false");
086:
087: envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(),
088: "6");
089: envConfig.setConfigParam(EnvironmentParams.JE_LOGGING_LEVEL
090: .getName(), "CONFIG");
091: if (logFileSize != null) {
092: DbInternal.disableParameterValidation(envConfig);
093: envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX
094: .getName(), logFileSize);
095: }
096:
097: /* Disable noisy UtilizationProfile database creation. */
098: DbInternal.setCreateUP(envConfig, false);
099: /* Don't checkpoint utilization info for this test. */
100: DbInternal.setCheckpointUP(envConfig, false);
101:
102: envConfig.setAllowCreate(true);
103: envImpl = new EnvironmentImpl(envHome, envConfig);
104: configManager = envImpl.getConfigManager();
105: fileManager = envImpl.getFileManager();
106: logManager = envImpl.getLogManager();
107: }
108:
109: /**
110: * Run with an empty file that has a file header but no log entries.
111: */
112: public void testEmptyAtEnd() throws Throwable {
113:
114: initEnv();
115:
116: /*
117: * Make a log file with a valid header, but no data.
118: */
119: FileManagerTestUtils.createLogFile(fileManager, envImpl, 100);
120: fileManager.clear();
121:
122: LastFileReader reader = new LastFileReader(envImpl, 1000);
123: assertTrue(reader.readNextEntry());
124: assertEquals(0, DbLsn.getFileOffset(reader.getLastLsn()));
125: }
126:
127: /**
128: * Run with an empty, 0 length file at the end. This has caused a
129: * BufferUnderflowException. [#SR 12631]
130: */
131: public void testLastFileEmpty() throws Throwable {
132:
133: initEnv("1000");
134: int numIters = 10;
135: List testObjs = new ArrayList();
136: List testLsns = new ArrayList();
137:
138: /*
139: * Create a log with one or more files. Use only Tracer objects so we
140: * can iterate through the entire log ... ?
141: */
142: for (int i = 0; i < numIters; i++) {
143: /* Add a debug record. */
144: Tracer msg = new Tracer("Hello there, rec " + (i + 1));
145: testObjs.add(msg);
146: testLsns.add(new Long(msg.log(logManager)));
147: }
148: /* Flush the log, files. */
149: logManager.flush();
150: fileManager.clear();
151:
152: int lastFileNum = fileManager.getAllFileNumbers().length - 1;
153:
154: /*
155: * Create an extra, totally empty file.
156: */
157: fileManager.syncLogEnd();
158: fileManager.clear();
159: String emptyLastFile = fileManager.getFullFileName(
160: lastFileNum + 1, FileManager.JE_SUFFIX);
161:
162: RandomAccessFile file = new RandomAccessFile(emptyLastFile,
163: FileManager.FileMode.READWRITE_MODE.getModeValue());
164: file.close();
165:
166: assertTrue(fileManager.getAllFileNumbers().length >= 2);
167:
168: /*
169: * Try a LastFileReader. It should give us a end-of-log position in the
170: * penultimate file.
171: */
172: LastFileReader reader = new LastFileReader(envImpl, 1000);
173: while (reader.readNextEntry()) {
174: }
175:
176: /*
177: * The reader should be positioned at the last, valid file, skipping
178: * this 0 length file.
179: */
180: assertEquals("lastValid="
181: + DbLsn.toString(reader.getLastValidLsn()),
182: lastFileNum, DbLsn.getFileNumber(reader
183: .getLastValidLsn()));
184: assertEquals(lastFileNum, DbLsn.getFileNumber(reader
185: .getEndOfLog()));
186: }
187:
188: /**
189: * Corrupt the file headers of the one and only log file.
190: */
191: public void testBadFileHeader() throws Throwable {
192:
193: initEnv();
194:
195: /*
196: * Handle a log file that has data and a bad header. First corrupt the
197: * existing log file. We will not be able to establish log end, but
198: * won't throw away the file because it has data.
199: */
200: long lastFileNum = fileManager.getLastFileNum().longValue();
201: String lastFile = fileManager.getFullFileName(lastFileNum,
202: FileManager.JE_SUFFIX);
203:
204: RandomAccessFile file = new RandomAccessFile(lastFile,
205: FileManager.FileMode.READWRITE_MODE.getModeValue());
206:
207: file.seek(15);
208: file.writeBytes("putting more junk in, mess up header");
209: file.close();
210:
211: /*
212: * We should see an exception on this one, because we made a file that
213: * looks like it has a bad header and bad data.
214: */
215: try {
216: LastFileReader reader = new LastFileReader(envImpl, 1000);
217: fail("Should see exception when creating " + reader);
218: } catch (DbChecksumException e) {
219: /* Eat exception, expected. */
220: }
221:
222: /*
223: * Now make a bad file header, but one that is less than the size of a
224: * file header. This file ought to get moved aside.
225: */
226: file = new RandomAccessFile(lastFile, "rw");
227: file.getChannel().truncate(0);
228: file.writeBytes("bad");
229: file.close();
230:
231: LastFileReader reader = new LastFileReader(envImpl, 1000);
232: /* Nothing comes back from reader. */
233: assertFalse(reader.readNextEntry());
234: File movedFile = new File(envHome, "00000000.bad");
235: assertTrue(movedFile.exists());
236:
237: /* Try a few more times, we ought to keep moving the file. */
238: file = new RandomAccessFile(lastFile, "rw");
239: file.getChannel().truncate(0);
240: file.writeBytes("bad");
241: file.close();
242:
243: reader = new LastFileReader(envImpl, 1000);
244: assertTrue(movedFile.exists());
245: File movedFile1 = new File(envHome, "00000000.bad.1");
246: assertTrue(movedFile1.exists());
247: }
248:
249: /**
250: * Run with defaults.
251: */
252: public void testBasic() throws Throwable {
253:
254: initEnv();
255: int numIters = 50;
256: List testObjs = new ArrayList();
257: List testLsns = new ArrayList();
258:
259: fillLogFile(numIters, testLsns, testObjs);
260: LastFileReader reader = new LastFileReader(
261: envImpl,
262: configManager
263: .getInt(EnvironmentParams.LOG_ITERATOR_READ_SIZE));
264:
265: checkLogEnd(reader, numIters, testLsns, testObjs);
266: }
267:
268: /**
269: * Run with very small read buffer.
270: */
271: public void testSmallBuffers() throws Throwable {
272:
273: initEnv();
274: int numIters = 50;
275: List testObjs = new ArrayList();
276: List testLsns = new ArrayList();
277:
278: fillLogFile(numIters, testLsns, testObjs);
279: LastFileReader reader = new LastFileReader(envImpl, 10);
280: checkLogEnd(reader, numIters, testLsns, testObjs);
281: }
282:
283: /**
284: * Run with medium buffers.
285: */
286: public void testMedBuffers() throws Throwable {
287:
288: initEnv();
289: int numIters = 50;
290: List testObjs = new ArrayList();
291: List testLsns = new ArrayList();
292:
293: fillLogFile(numIters, testLsns, testObjs);
294: LastFileReader reader = new LastFileReader(envImpl, 100);
295: checkLogEnd(reader, numIters, testLsns, testObjs);
296: }
297:
298: /**
299: * Put junk at the end of the file.
300: */
301: public void testJunk() throws Throwable {
302:
303: initEnv();
304: int numIters = 50;
305: List testObjs = new ArrayList();
306: List testLsns = new ArrayList();
307:
308: /* Write junk into the end of the file. */
309: fillLogFile(numIters, testLsns, testObjs);
310: long lastFileNum = fileManager.getLastFileNum().longValue();
311: String lastFile = fileManager.getFullFileName(lastFileNum,
312: FileManager.JE_SUFFIX);
313:
314: RandomAccessFile file = new RandomAccessFile(lastFile,
315: FileManager.FileMode.READWRITE_MODE.getModeValue());
316: file.seek(file.length());
317: file.writeBytes("hello, some junk");
318: file.close();
319:
320: /* Read. */
321: LastFileReader reader = new LastFileReader(envImpl, 100);
322: checkLogEnd(reader, numIters, testLsns, testObjs);
323: }
324:
325: /**
326: * Make a log, then make a few extra files at the end, one empty, one with
327: * a bad file header.
328: */
329: public void testExtraEmpty() throws Throwable {
330:
331: initEnv();
332: int numIters = 50;
333: List testObjs = new ArrayList();
334: List testLsns = new ArrayList();
335: int defaultBufferSize = configManager
336: .getInt(EnvironmentParams.LOG_ITERATOR_READ_SIZE);
337:
338: /*
339: * Make a valid log with data, then put a couple of extra files after
340: * it. Make the file numbers non-consecutive. We should have three log
341: * files.
342: */
343: /* Create a log */
344: fillLogFile(numIters, testLsns, testObjs);
345:
346: /* First empty log file -- header, no data. */
347: fileManager.bumpLsn(100000000);
348: fileManager.bumpLsn(100000000);
349: FileManagerTestUtils.createLogFile(fileManager, envImpl, 10);
350:
351: /* Second empty log file -- header, no data. */
352: fileManager.bumpLsn(100000000);
353: fileManager.bumpLsn(100000000);
354: FileManagerTestUtils.createLogFile(fileManager, envImpl, 10);
355:
356: assertEquals(3, fileManager.getAllFileNumbers().length);
357:
358: /*
359: * Corrupt the last empty file and then search for the correct last
360: * file.
361: */
362: long lastFileNum = fileManager.getLastFileNum().longValue();
363: String lastFile = fileManager.getFullFileName(lastFileNum,
364: FileManager.JE_SUFFIX);
365: RandomAccessFile file = new RandomAccessFile(lastFile,
366: FileManager.FileMode.READWRITE_MODE.getModeValue());
367: file.getChannel().truncate(10);
368: file.close();
369: fileManager.clear();
370:
371: /*
372: * Make a reader, read the log. After the reader returns, we should
373: * only have 2 log files.
374: */
375: LastFileReader reader = new LastFileReader(envImpl,
376: defaultBufferSize);
377: checkLogEnd(reader, numIters, testLsns, testObjs);
378: assertEquals(2, fileManager.getAllFileNumbers().length);
379:
380: /*
381: * Corrupt the now "last" empty file and try again. This is actually
382: * the first empty file we made.
383: */
384: lastFileNum = fileManager.getLastFileNum().longValue();
385: lastFile = fileManager.getFullFileName(lastFileNum,
386: FileManager.JE_SUFFIX);
387: file = new RandomAccessFile(lastFile,
388: FileManager.FileMode.READWRITE_MODE.getModeValue());
389: file.getChannel().truncate(10);
390: file.close();
391:
392: /*
393: * Validate that we have the right number of log entries, and only one
394: * valid log file.
395: */
396: reader = new LastFileReader(envImpl, defaultBufferSize);
397: checkLogEnd(reader, numIters, testLsns, testObjs);
398: assertEquals(1, fileManager.getAllFileNumbers().length);
399: }
400:
401: /**
402: * Write a logfile of entries, then read the end.
403: */
404: private void fillLogFile(int numIters, List testLsns, List testObjs)
405: throws Throwable {
406:
407: /*
408: * Create a log file full of LNs and Debug Records.
409: */
410: for (int i = 0; i < numIters; i++) {
411: /* Add a debug record. */
412: Tracer msg = new Tracer("Hello there, rec " + (i + 1));
413: testObjs.add(msg);
414: testLsns.add(new Long(msg.log(logManager)));
415:
416: /* Add a txn abort */
417: TxnAbort abort = new TxnAbort(10L, 200L);
418: SingleItemEntry entry = new SingleItemEntry(
419: LogEntryType.LOG_TXN_ABORT, abort);
420: testObjs.add(abort);
421: testLsns.add(new Long(logManager.log(entry)));
422: }
423:
424: /* Flush the log, files. */
425: logManager.flush();
426: fileManager.clear();
427: }
428:
429: /**
430: * Use the LastFileReader to check this file, see if the log end is set
431: * right.
432: */
433: private void checkLogEnd(LastFileReader reader, int numIters,
434: List testLsns, List testObjs) throws Throwable {
435:
436: reader.setTargetType(LogEntryType.LOG_ROOT);
437: reader.setTargetType(LogEntryType.LOG_TXN_COMMIT);
438: reader.setTargetType(LogEntryType.LOG_TXN_ABORT);
439: reader.setTargetType(LogEntryType.LOG_TRACE);
440: reader.setTargetType(LogEntryType.LOG_IN);
441: reader.setTargetType(LogEntryType.LOG_LN_TRANSACTIONAL);
442:
443: /* Now ask the LastFileReader to read it back. */
444: while (reader.readNextEntry()) {
445: }
446:
447: /* Truncate the file. */
448: reader.setEndOfFile();
449:
450: /*
451: * How many entries did the iterator go over? We should see
452: * numIters * 2 + 7
453: * (the extra 7 are the root, debug records and checkpoints and file
454: * header written by recovery.
455: */
456: assertEquals("should have seen this many entries",
457: (numIters * 2) + 7, reader.getNumRead());
458:
459: /* Check last used LSN. */
460: int numLsns = testLsns.size();
461: long lastLsn = DbLsn
462: .longToLsn((Long) testLsns.get(numLsns - 1));
463: assertEquals("last LSN", lastLsn, reader.getLastLsn());
464:
465: /* Check last offset. */
466: assertEquals("prev offset", DbLsn.getFileOffset(lastLsn),
467: reader.getPrevOffset());
468:
469: /* Check next available LSN. */
470: int lastSize = ((Loggable) testObjs.get(testObjs.size() - 1))
471: .getLogSize();
472: assertEquals("next available", DbLsn.makeLsn(DbLsn
473: .getFileNumber(lastLsn), DbLsn.getFileOffset(lastLsn)
474: + LogEntryHeader.MIN_HEADER_SIZE + lastSize), reader
475: .getEndOfLog());
476:
477: /* The log should be truncated to just the right size. */
478: FileHandle handle = fileManager.getFileHandle(0L);
479: RandomAccessFile file = handle.getFile();
480: assertEquals(DbLsn.getFileOffset(reader.getEndOfLog()), file
481: .getChannel().size());
482: handle.release();
483: fileManager.clear();
484:
485: /* Check the last tracked LSNs. */
486: assertTrue(reader.getLastSeen(LogEntryType.LOG_ROOT) != DbLsn.NULL_LSN);
487: assertTrue(reader.getLastSeen(LogEntryType.LOG_IN) == DbLsn.NULL_LSN);
488: assertTrue(reader
489: .getLastSeen(LogEntryType.LOG_LN_TRANSACTIONAL) == DbLsn.NULL_LSN);
490: assertEquals(reader.getLastSeen(LogEntryType.LOG_TRACE), DbLsn
491: .longToLsn((Long) testLsns.get(numLsns - 2)));
492: assertEquals(reader.getLastSeen(LogEntryType.LOG_TXN_ABORT),
493: lastLsn);
494: }
495: }
|