001: /*-
002: * See the file LICENSE for redistribution information.
003: *
004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
005: *
006: * $Id: DbBackupTest.java,v 1.3.2.3 2008/01/07 15:14:34 cwl Exp $
007: */
008:
009: package com.sleepycat.je.util;
010:
011: import java.io.File;
012: import java.io.FileInputStream;
013: import java.io.FileOutputStream;
014: import java.io.IOException;
015: import java.nio.channels.FileChannel;
016:
017: import junit.framework.TestCase;
018:
019: import com.sleepycat.bind.tuple.IntegerBinding;
020: import com.sleepycat.je.CheckpointConfig;
021: import com.sleepycat.je.Cursor;
022: import com.sleepycat.je.Database;
023: import com.sleepycat.je.DatabaseConfig;
024: import com.sleepycat.je.DatabaseEntry;
025: import com.sleepycat.je.DatabaseException;
026: import com.sleepycat.je.DbInternal;
027: import com.sleepycat.je.Environment;
028: import com.sleepycat.je.EnvironmentConfig;
029: import com.sleepycat.je.EnvironmentStats;
030: import com.sleepycat.je.LockMode;
031: import com.sleepycat.je.OperationStatus;
032: import com.sleepycat.je.StatsConfig;
033: import com.sleepycat.je.config.EnvironmentParams;
034: import com.sleepycat.je.dbi.EnvironmentImpl;
035: import com.sleepycat.je.log.FileManager;
036: import com.sleepycat.je.utilint.DbLsn;
037:
038: public class DbBackupTest extends TestCase {
039:
040: private static StatsConfig CLEAR_CONFIG = new StatsConfig();
041: static {
042: CLEAR_CONFIG.setClear(true);
043: }
044:
045: private static CheckpointConfig FORCE_CONFIG = new CheckpointConfig();
046: static {
047: FORCE_CONFIG.setForce(true);
048: }
049:
050: private static final String SAVE1 = "save1";
051: private static final String SAVE2 = "save2";
052: private static final String SAVE3 = "save3";
053: private static final int NUM_RECS = 50;
054:
055: private File envHome;
056: private Environment env;
057: private FileManager fileManager;
058:
059: public DbBackupTest() {
060: envHome = new File(System.getProperty(TestUtils.DEST_DIR));
061: }
062:
063: public void setUp() throws IOException {
064:
065: TestUtils.removeLogFiles("Setup", envHome, false);
066: deleteSaveDir(SAVE1);
067: deleteSaveDir(SAVE2);
068: deleteSaveDir(SAVE3);
069: }
070:
071: public void tearDown() throws Exception {
072:
073: TestUtils.removeLogFiles("TearDown", envHome, false);
074: deleteSaveDir(SAVE1);
075: deleteSaveDir(SAVE2);
076: deleteSaveDir(SAVE3);
077: }
078:
079: /**
080: * Test basic backup, make sure log cleaning isn't running.
081: */
082: public void testBackupVsCleaning() throws Throwable {
083:
084: env = createEnv(false, envHome); /* read-write env */
085: EnvironmentImpl envImpl = DbInternal.envGetEnvironmentImpl(env);
086: fileManager = envImpl.getFileManager();
087:
088: try {
089:
090: /*
091: * Grow files, creating obsolete entries to create cleaner
092: * opportunity.
093: */
094: growFiles("db1", env, 8);
095:
096: /* Start backup. */
097: DbBackup backupHelper = new DbBackup(env);
098: backupHelper.startBackup();
099:
100: long lastFileNum = backupHelper.getLastFileInBackupSet();
101: long checkLastFileNum = lastFileNum;
102:
103: /* Copy the backup set. */
104: saveFiles(backupHelper, -1, lastFileNum, SAVE1);
105:
106: /*
107: * Try to clean and checkpoint. Check that the logs grew as
108: * a result.
109: */
110: batchClean(0);
111: long newLastFileNum = (fileManager.getLastFileNum())
112: .longValue();
113: assertTrue(checkLastFileNum < newLastFileNum);
114: checkLastFileNum = newLastFileNum;
115:
116: /* Copy the backup set after attempting cleaning */
117: saveFiles(backupHelper, -1, lastFileNum, SAVE2);
118:
119: /* Insert more data. */
120: growFiles("db2", env, 8);
121:
122: /*
123: * Try to clean and checkpoint. Check that the logs grew as
124: * a result.
125: */
126: batchClean(0);
127: newLastFileNum = fileManager.getLastFileNum().longValue();
128: assertTrue(checkLastFileNum < newLastFileNum);
129: checkLastFileNum = newLastFileNum;
130:
131: /* Copy the backup set after inserting more data */
132: saveFiles(backupHelper, -1, lastFileNum, SAVE3);
133:
134: /* Check the membership of the saved set. */
135: long lastFile = backupHelper.getLastFileInBackupSet();
136: String[] backupSet = backupHelper.getLogFilesInBackupSet();
137: assertEquals((lastFile + 1), backupSet.length);
138:
139: /* End backup. */
140: backupHelper.endBackup();
141:
142: /*
143: * Run cleaning, and verify that quite a few files are deleted.
144: */
145: long numCleaned = batchClean(100);
146: assertTrue(numCleaned > 5);
147: env.close();
148: env = null;
149:
150: /* Verify backups. */
151: TestUtils.removeLogFiles("Verify", envHome, false);
152: verifyDb1(SAVE1, true);
153: TestUtils.removeLogFiles("Verify", envHome, false);
154: verifyDb1(SAVE2, true);
155: TestUtils.removeLogFiles("Verify", envHome, false);
156: verifyDb1(SAVE3, true);
157: } finally {
158: if (env != null) {
159: env.close();
160: }
161: }
162: }
163:
164: /**
165: * Test multiple backup passes
166: */
167: public void testIncrementalBackup() throws Throwable {
168:
169: env = createEnv(false, envHome); /* read-write env */
170: EnvironmentImpl envImpl = DbInternal.envGetEnvironmentImpl(env);
171: fileManager = envImpl.getFileManager();
172:
173: try {
174:
175: /*
176: * Grow files, creating obsolete entries to create cleaner
177: * opportunity.
178: */
179: growFiles("db1", env, 8);
180:
181: /* Backup1. */
182: DbBackup backupHelper = new DbBackup(env);
183: backupHelper.startBackup();
184: long b1LastFile = backupHelper.getLastFileInBackupSet();
185: saveFiles(backupHelper, -1, b1LastFile, SAVE1);
186: String lastName = fileManager.getFullFileName(b1LastFile,
187: FileManager.JE_SUFFIX);
188: File f = new File(lastName);
189: long savedLength = f.length();
190: backupHelper.endBackup();
191:
192: /*
193: * Add more data. Check that the file did flip, and is not modified
194: * by the additional data.
195: */
196: growFiles("db2", env, 8);
197: checkFileLen(b1LastFile, savedLength);
198:
199: /* Backup2. */
200: backupHelper.startBackup();
201: long b2LastFile = backupHelper.getLastFileInBackupSet();
202: saveFiles(backupHelper, b1LastFile, b2LastFile, SAVE2);
203: backupHelper.endBackup();
204:
205: env.close();
206: env = null;
207:
208: /* Verify backups. */
209: TestUtils.removeLogFiles("Verify", envHome, false);
210: verifyDb1(SAVE1, false);
211: TestUtils.removeLogFiles("Verify", envHome, false);
212: verifyBothDbs(SAVE1, SAVE2);
213: } finally {
214: if (env != null) {
215: env.close();
216: }
217: }
218: }
219:
220: public void testBadUsage() throws Exception {
221:
222: Environment env = createEnv(false, envHome); /* read-write env */
223:
224: try {
225: DbBackup backup = new DbBackup(env);
226:
227: /* end can only be called after start. */
228: try {
229: backup.endBackup();
230: fail("should fail");
231: } catch (DatabaseException expected) {
232: }
233:
234: /* start can't be called twice. */
235: backup.startBackup();
236: try {
237: backup.startBackup();
238: fail("should fail");
239: } catch (DatabaseException expected) {
240: }
241:
242: /*
243: * You can only get the backup set when you're in between start
244: * and end.
245: */
246: backup.endBackup();
247:
248: try {
249: backup.getLastFileInBackupSet();
250: fail("should fail");
251: } catch (DatabaseException expected) {
252: }
253:
254: try {
255: backup.getLogFilesInBackupSet();
256: fail("should fail");
257: } catch (DatabaseException expected) {
258: }
259:
260: try {
261: backup.getLogFilesInBackupSet(0);
262: fail("should fail");
263: } catch (DatabaseException expected) {
264: }
265: } finally {
266: env.close();
267: }
268: }
269:
270: /*
271: * This test can't be run by default, because it makes a directory
272: * read/only, and Java doesn't support a way to make it writable again
273: * except in Mustang. There's no way to clean up a read-only directory.
274: */
275: public void xtestReadOnly() throws Exception {
276:
277: /* Make a read-only handle on a read-write environment directory.*/
278: Environment env = createEnv(true, envHome);
279:
280: try {
281: DbBackup backup = new DbBackup(env);
282: fail("Should fail because env is read/only.");
283: } catch (DatabaseException expected) {
284: }
285:
286: env.close();
287:
288: /*
289: * Make a read-only handle on a read-only environment directory. Use a
290: * new environment directory because we're going to set it read0nly and
291: * there doesn't seem to be a way of undoing that.
292: */
293: File tempEnvDir = new File(envHome, SAVE1);
294: assertTrue(tempEnvDir.mkdirs());
295: env = createEnv(false, tempEnvDir);
296: growFiles("db1", env, 8);
297: env.close();
298: //assertTrue(tempEnvDir.setReadOnly());
299:
300: env = createEnv(true, tempEnvDir);
301:
302: DbBackup backupHelper = new DbBackup(env);
303: backupHelper.startBackup();
304:
305: FileManager fileManager = DbInternal.envGetEnvironmentImpl(env)
306: .getFileManager();
307: long lastFile = fileManager.getLastFileNum().longValue();
308: assertEquals(lastFile, backupHelper.getLastFileInBackupSet());
309:
310: backupHelper.endBackup();
311: env.close();
312: assertTrue(tempEnvDir.delete());
313: }
314:
315: private Environment createEnv(boolean readOnly, File envDir)
316: throws DatabaseException {
317:
318: EnvironmentConfig envConfig = TestUtils.initEnvConfig();
319: DbInternal.disableParameterValidation(envConfig);
320: envConfig.setAllowCreate(true);
321: envConfig.setReadOnly(readOnly);
322: envConfig.setConfigParam(EnvironmentParams.LOG_FILE_MAX
323: .getName(), "400");
324: envConfig.setConfigParam(EnvironmentParams.ENV_RUN_CLEANER
325: .getName(), "false");
326:
327: Environment env = new Environment(envDir, envConfig);
328:
329: return env;
330: }
331:
332: private long growFiles(String dbName, Environment env,
333: int minNumFiles) throws DatabaseException {
334:
335: DatabaseConfig dbConfig = new DatabaseConfig();
336: dbConfig.setAllowCreate(true);
337: Database db = env.openDatabase(null, dbName, dbConfig);
338: FileManager fileManager = DbInternal.envGetEnvironmentImpl(env)
339: .getFileManager();
340: long startLastFileNum = DbLsn.getFileNumber(fileManager
341: .getLastUsedLsn());
342:
343: DatabaseEntry key = new DatabaseEntry();
344: DatabaseEntry data = new DatabaseEntry();
345: /* Update twice, in order to create plenty of cleaning opportunity. */
346: for (int i = 0; i < NUM_RECS; i++) {
347: IntegerBinding.intToEntry(i, key);
348: IntegerBinding.intToEntry(i, data);
349: assertEquals(OperationStatus.SUCCESS, db.put(null, key,
350: data));
351: }
352:
353: for (int i = 0; i < NUM_RECS; i++) {
354: IntegerBinding.intToEntry(i, key);
355: IntegerBinding.intToEntry(i + 5, data);
356: assertEquals(OperationStatus.SUCCESS, db.put(null, key,
357: data));
358: }
359:
360: db.close();
361:
362: long endLastFileNum = DbLsn.getFileNumber(fileManager
363: .getLastUsedLsn());
364: assertTrue((endLastFileNum - startLastFileNum) >= minNumFiles);
365: return endLastFileNum;
366: }
367:
368: private int batchClean(int expectedDeletions)
369: throws DatabaseException {
370:
371: EnvironmentStats stats = env.getStats(CLEAR_CONFIG);
372: while (env.cleanLog() > 0) {
373: }
374: env.checkpoint(FORCE_CONFIG);
375: stats = env.getStats(CLEAR_CONFIG);
376: assertTrue(stats.getNCleanerDeletions() <= expectedDeletions);
377:
378: return stats.getNCleanerDeletions();
379: }
380:
381: private void saveFiles(DbBackup backupHelper,
382: long lastFileFromPrevBackup, long lastFileNum,
383: String saveDirName) throws IOException, DatabaseException {
384:
385: /* Check that the backup set contains only the files it should have. */
386: String[] fileList = backupHelper
387: .getLogFilesInBackupSet(lastFileFromPrevBackup);
388: assertEquals(lastFileNum, fileManager.getNumFromName(
389: fileList[fileList.length - 1]).longValue());
390:
391: /* Make a new save directory. */
392: File saveDir = new File(envHome, saveDirName);
393: assertTrue(saveDir.mkdir());
394: copyFiles(envHome, saveDir, fileList);
395: }
396:
397: private void copyFiles(File sourceDir, File destDir,
398: String[] fileList) throws DatabaseException {
399:
400: try {
401: for (int i = 0; i < fileList.length; i++) {
402: File source = new File(sourceDir, fileList[i]);
403: FileChannel sourceChannel = new FileInputStream(source)
404: .getChannel();
405: File save = new File(destDir, fileList[i]);
406: FileChannel saveChannel = new FileOutputStream(save)
407: .getChannel();
408:
409: saveChannel.transferFrom(sourceChannel, 0,
410: sourceChannel.size());
411:
412: // Close the channels
413: sourceChannel.close();
414: saveChannel.close();
415: }
416: } catch (IOException e) {
417: throw new DatabaseException(e);
418: }
419: }
420:
421: /**
422: * Delete all the contents and the directory itself.
423: */
424: private void deleteSaveDir(String saveDirName) throws IOException {
425:
426: File saveDir = new File(envHome, saveDirName);
427: if (saveDir.exists()) {
428: String[] savedFiles = saveDir.list();
429: if (savedFiles != null) {
430: for (int i = 0; i < savedFiles.length; i++) {
431: File f = new File(saveDir, savedFiles[i]);
432: assertTrue(f.delete());
433: }
434: assertTrue(saveDir.delete());
435: }
436: }
437: }
438:
439: /**
440: * Copy the saved files in, check values.
441: */
442: private void verifyDb1(String saveDirName, boolean rename)
443: throws DatabaseException {
444:
445: File saveDir = new File(envHome, saveDirName);
446: String[] savedFiles = saveDir.list();
447: if (rename) {
448: for (int i = 0; i < savedFiles.length; i++) {
449: File saved = new File(saveDir, savedFiles[i]);
450: File dest = new File(envHome, savedFiles[i]);
451: assertTrue(saved.renameTo(dest));
452: }
453: } else {
454: /* copy. */
455: copyFiles(saveDir, envHome, savedFiles);
456: }
457: env = createEnv(false, envHome);
458: try {
459: checkDb("db1");
460:
461: /* Db 2 should not exist. */
462: DatabaseConfig dbConfig = new DatabaseConfig();
463: try {
464: Database db = env.openDatabase(null, "db2", dbConfig);
465: fail("db2 should not exist");
466: } catch (DatabaseException expected) {
467: }
468:
469: } finally {
470: env.close();
471: env = null;
472: }
473: }
474:
475: /**
476: * Copy the saved files in, check values.
477: */
478: private void verifyBothDbs(String saveDirName1, String saveDirName2)
479: throws DatabaseException {
480:
481: File saveDir = new File(envHome, saveDirName1);
482: String[] savedFiles = saveDir.list();
483: for (int i = 0; i < savedFiles.length; i++) {
484: File saved = new File(saveDir, savedFiles[i]);
485: File dest = new File(envHome, savedFiles[i]);
486: assertTrue(saved.renameTo(dest));
487: }
488:
489: saveDir = new File(envHome, saveDirName2);
490: savedFiles = saveDir.list();
491: for (int i = 0; i < savedFiles.length; i++) {
492: File saved = new File(saveDir, savedFiles[i]);
493: File dest = new File(envHome, savedFiles[i]);
494: assertTrue(saved.renameTo(dest));
495: }
496:
497: env = createEnv(false, envHome);
498: try {
499: checkDb("db1");
500: checkDb("db2");
501: } finally {
502: env.close();
503: env = null;
504: }
505: }
506:
507: private void checkDb(String dbName) throws DatabaseException {
508:
509: DatabaseConfig dbConfig = new DatabaseConfig();
510: Database db = env.openDatabase(null, dbName, dbConfig);
511: Cursor c = null;
512: try {
513: DatabaseEntry key = new DatabaseEntry();
514: DatabaseEntry data = new DatabaseEntry();
515: c = db.openCursor(null, null);
516:
517: for (int i = 0; i < NUM_RECS; i++) {
518: assertEquals(OperationStatus.SUCCESS, c.getNext(key,
519: data, LockMode.DEFAULT));
520: assertEquals(i, IntegerBinding.entryToInt(key));
521: assertEquals(i + 5, IntegerBinding.entryToInt(data));
522: }
523: assertEquals(OperationStatus.NOTFOUND, c.getNext(key, data,
524: LockMode.DEFAULT));
525: } finally {
526: if (c != null)
527: c.close();
528: db.close();
529: }
530: }
531:
532: private void checkFileLen(long fileNum, long length)
533: throws IOException {
534: String fileName = fileManager.getFullFileName(fileNum,
535: FileManager.JE_SUFFIX);
536: File f = new File(fileName);
537: assertEquals(length, f.length());
538: }
539: }
|