001: /*-
002: * See the file LICENSE for redistribution information.
003: *
004: * Copyright (c) 2002,2008 Oracle. All rights reserved.
005: *
006: * $Id: DbLoad.java,v 1.46.2.3 2008/01/07 15:14:17 cwl Exp $
007: */
008:
009: package com.sleepycat.je.util;
010:
011: import java.io.BufferedReader;
012: import java.io.File;
013: import java.io.FileInputStream;
014: import java.io.IOException;
015: import java.io.InputStream;
016: import java.io.InputStreamReader;
017: import java.util.Date;
018: import java.util.logging.Level;
019:
020: import com.sleepycat.je.Database;
021: import com.sleepycat.je.DatabaseConfig;
022: import com.sleepycat.je.DatabaseEntry;
023: import com.sleepycat.je.DatabaseException;
024: import com.sleepycat.je.DbInternal;
025: import com.sleepycat.je.Environment;
026: import com.sleepycat.je.EnvironmentConfig;
027: import com.sleepycat.je.JEVersion;
028: import com.sleepycat.je.OperationStatus;
029: import com.sleepycat.je.utilint.CmdUtil;
030: import com.sleepycat.je.utilint.Tracer;
031:
032: public class DbLoad {
033: private static final boolean DEBUG = false;
034:
035: protected Environment env;
036: private boolean formatUsingPrintable;
037: private String dbName;
038: private BufferedReader reader;
039: private boolean noOverwrite;
040: private boolean textFileMode;
041: private boolean dupSort;
042: private boolean ignoreUnknownConfig;
043: private boolean commandLine;
044: private long progressInterval;
045: private long totalLoadBytes;
046:
047: private static final String usageString = "usage: "
048: + CmdUtil.getJavaCommand(DbLoad.class)
049: + "\n"
050: + " -h <dir> # environment home directory\n"
051: + " [-f <fileName>] # input file\n"
052: + " [-n ] # no overwrite mode\n"
053: + " [-T] # input file is in text mode\n"
054: + " [-I] # ignore unknown parameters\n"
055: + " [-c name=value] # config values\n"
056: + " [-s <databaseName> ] # database to load\n"
057: + " [-v] # show progress\n"
058: + " [-V] # print JE version number";
059:
060: static public void main(String argv[]) throws DatabaseException,
061: IOException {
062:
063: DbLoad loader = parseArgs(argv);
064:
065: try {
066: loader.load();
067: } catch (Throwable e) {
068: e.printStackTrace();
069: }
070:
071: loader.env.close();
072: }
073:
074: static private void printUsage(String msg) {
075: System.err.println(msg);
076: System.err.println(usageString);
077: System.exit(-1);
078: }
079:
080: static private DbLoad parseArgs(String argv[]) throws IOException,
081: DatabaseException {
082:
083: boolean noOverwrite = false;
084: boolean textFileMode = false;
085: boolean ignoreUnknownConfig = false;
086: boolean showProgressInterval = false;
087:
088: int argc = 0;
089: int nArgs = argv.length;
090: String inputFileName = null;
091: File envHome = null;
092: String dbName = null;
093: long progressInterval = 0;
094: DbLoad ret = new DbLoad();
095: ret.setCommandLine(true);
096:
097: while (argc < nArgs) {
098: String this Arg = argv[argc++].trim();
099: if (this Arg.equals("-n")) {
100: noOverwrite = true;
101: } else if (this Arg.equals("-T")) {
102: textFileMode = true;
103: } else if (this Arg.equals("-I")) {
104: ignoreUnknownConfig = true;
105: } else if (this Arg.equals("-V")) {
106: System.out.println(JEVersion.CURRENT_VERSION);
107: System.exit(0);
108: } else if (this Arg.equals("-f")) {
109: if (argc < nArgs) {
110: inputFileName = argv[argc++];
111: } else {
112: printUsage("-f requires an argument");
113: }
114: } else if (this Arg.equals("-h")) {
115: if (argc < nArgs) {
116: envHome = new File(argv[argc++]);
117: } else {
118: printUsage("-h requires an argument");
119: }
120: } else if (this Arg.equals("-s")) {
121: if (argc < nArgs) {
122: dbName = argv[argc++];
123: } else {
124: printUsage("-s requires an argument");
125: }
126: } else if (this Arg.equals("-c")) {
127: if (argc < nArgs) {
128: try {
129: ret.loadConfigLine(argv[argc++]);
130: } catch (IllegalArgumentException e) {
131: printUsage("-c: " + e.getMessage());
132: }
133: } else {
134: printUsage("-c requires an argument");
135: }
136: } else if (this Arg.equals("-v")) {
137: showProgressInterval = true;
138: }
139: }
140:
141: if (envHome == null) {
142: printUsage("-h is a required argument");
143: }
144:
145: long totalLoadBytes = 0;
146: InputStream is;
147: if (inputFileName == null) {
148: is = System.in;
149: if (showProgressInterval) {
150:
151: /*
152: * Can't show progress if we don't know how big the stream
153: * is.
154: */
155: printUsage("-v requires -f");
156: }
157: } else {
158: is = new FileInputStream(inputFileName);
159: if (showProgressInterval) {
160: totalLoadBytes = ((FileInputStream) is).getChannel()
161: .size();
162: /* Use 5% intervals. */
163: progressInterval = totalLoadBytes / 20;
164: }
165: }
166: BufferedReader reader = new BufferedReader(
167: new InputStreamReader(is));
168:
169: EnvironmentConfig envConfig = new EnvironmentConfig();
170: envConfig.setAllowCreate(true);
171: Environment env = new Environment(envHome, envConfig);
172: ret.setEnv(env);
173: ret.setDbName(dbName);
174: ret.setInputReader(reader);
175: ret.setNoOverwrite(noOverwrite);
176: ret.setTextFileMode(textFileMode);
177: ret.setIgnoreUnknownConfig(ignoreUnknownConfig);
178: ret.setProgressInterval(progressInterval);
179: ret.setTotalLoadBytes(totalLoadBytes);
180: return ret;
181: }
182:
183: /*
184: * Begin DbLoad API. From here on there should be no calls to printUsage,
185: * System.xxx.print, or System.exit.
186: */
187:
188: public DbLoad() {
189: }
190:
191: /**
192: * If true, enables output of warning messages. Command line behavior is
193: * not available via the public API.
194: */
195: private void setCommandLine(boolean commandLine) {
196: this .commandLine = commandLine;
197: }
198:
199: public void setEnv(Environment env) {
200: this .env = env;
201: }
202:
203: public void setDbName(String dbName) {
204: this .dbName = dbName;
205: }
206:
207: public void setInputReader(BufferedReader reader) {
208: this .reader = reader;
209: }
210:
211: public void setNoOverwrite(boolean noOverwrite) {
212: this .noOverwrite = noOverwrite;
213: }
214:
215: public void setTextFileMode(boolean textFileMode) {
216: this .textFileMode = textFileMode;
217: }
218:
219: public void setIgnoreUnknownConfig(boolean ignoreUnknownConfigMode) {
220: this .ignoreUnknownConfig = ignoreUnknownConfigMode;
221: }
222:
223: public void setProgressInterval(long progressInterval) {
224: this .progressInterval = progressInterval;
225: }
226:
227: public void setTotalLoadBytes(long totalLoadBytes) {
228: this .totalLoadBytes = totalLoadBytes;
229: }
230:
231: public boolean load() throws IOException, DatabaseException {
232:
233: Tracer.trace(Level.INFO, DbInternal.envGetEnvironmentImpl(env),
234: "DbLoad.load of " + dbName + " starting");
235:
236: if (progressInterval > 0) {
237: System.out.println("Load start: " + new Date());
238: }
239:
240: if (textFileMode) {
241: formatUsingPrintable = true;
242: } else {
243: loadHeader();
244: }
245:
246: if (dbName == null) {
247: throw new IllegalArgumentException(
248: "Must supply a database name if -l not supplied.");
249: }
250:
251: DatabaseConfig dbConfig = new DatabaseConfig();
252: dbConfig.setSortedDuplicates(dupSort);
253: dbConfig.setAllowCreate(true);
254: Database db = env.openDatabase(null, dbName, dbConfig);
255:
256: loadData(db);
257:
258: db.close();
259:
260: Tracer.trace(Level.INFO, DbInternal.envGetEnvironmentImpl(env),
261: "DbLoad.load of " + dbName + " ending.");
262:
263: if (progressInterval > 0) {
264: System.out.println("Load end: " + new Date());
265: }
266:
267: return true;
268: }
269:
270: private void loadConfigLine(String line) throws DatabaseException {
271:
272: int equalsIdx = line.indexOf('=');
273: if (equalsIdx < 0) {
274: throw new IllegalArgumentException(
275: "Invalid header parameter: " + line);
276: }
277:
278: String keyword = line.substring(0, equalsIdx).trim()
279: .toLowerCase();
280: String value = line.substring(equalsIdx + 1).trim();
281:
282: if (keyword.equals("version")) {
283: if (DEBUG) {
284: System.out.println("Found version: " + line);
285: }
286: if (!value.equals("3")) {
287: throw new IllegalArgumentException("Version " + value
288: + " is not supported.");
289: }
290: } else if (keyword.equals("format")) {
291: value = value.toLowerCase();
292: if (value.equals("print")) {
293: formatUsingPrintable = true;
294: } else if (value.equals("bytevalue")) {
295: formatUsingPrintable = false;
296: } else {
297: throw new IllegalArgumentException(value
298: + " is an unknown value for the format keyword");
299: }
300: if (DEBUG) {
301: System.out.println("Found format: "
302: + formatUsingPrintable);
303: }
304: } else if (keyword.equals("dupsort")) {
305: value = value.toLowerCase();
306: if (value.equals("true") || value.equals("1")) {
307: dupSort = true;
308: } else if (value.equals("false") || value.equals("0")) {
309: dupSort = false;
310: } else {
311: throw new IllegalArgumentException(
312: value
313: + " is an unknown value for the dupsort keyword");
314: }
315: if (DEBUG) {
316: System.out.println("Found dupsort: " + dupSort);
317: }
318: } else if (keyword.equals("type")) {
319: value = value.toLowerCase();
320: if (!value.equals("btree")) {
321: throw new IllegalArgumentException(value
322: + " is not a supported database type.");
323: }
324: if (DEBUG) {
325: System.out.println("Found type: " + line);
326: }
327: } else if (keyword.equals("database")) {
328: if (dbName == null) {
329: dbName = value;
330: }
331: if (DEBUG) {
332: System.out.println("DatabaseImpl: " + dbName);
333: }
334: } else if (!ignoreUnknownConfig) {
335: throw new IllegalArgumentException("'" + line
336: + "' is not understood.");
337: }
338: }
339:
340: private void loadHeader() throws IOException, DatabaseException {
341:
342: if (DEBUG) {
343: System.out.println("loading header");
344: }
345: String line = reader.readLine();
346: while (line != null && !line.equals("HEADER=END")) {
347: loadConfigLine(line);
348: line = reader.readLine();
349: }
350: }
351:
352: private void loadData(Database db) throws DatabaseException,
353: IOException {
354:
355: String keyLine = reader.readLine();
356: String dataLine = null;
357: int count = 0;
358: long totalBytesRead = 0;
359: long lastTime = System.currentTimeMillis();
360: long bytesReadThisInterval = 0;
361:
362: while (keyLine != null && !keyLine.equals("DATA=END")) {
363: dataLine = reader.readLine();
364: if (dataLine == null) {
365: throw new DatabaseException("No data to match key "
366: + keyLine);
367: }
368: /* Add one for \n or \r. */
369: bytesReadThisInterval += dataLine.length() + 1;
370: byte[] keyBytes = loadLine(keyLine.trim());
371: byte[] dataBytes = loadLine(dataLine.trim());
372:
373: DatabaseEntry key = new DatabaseEntry(keyBytes);
374: DatabaseEntry data = new DatabaseEntry(dataBytes);
375:
376: if (noOverwrite) {
377: if (db.putNoOverwrite(null, key, data) == OperationStatus.KEYEXIST) {
378: /* Calling println is OK only from command line. */
379: if (commandLine) {
380: System.err.println("Key exists: " + key);
381: }
382: }
383: } else {
384: db.put(null, key, data);
385: }
386:
387: count++;
388: if ((progressInterval > 0)
389: && (bytesReadThisInterval > progressInterval)) {
390: totalBytesRead += bytesReadThisInterval;
391: bytesReadThisInterval -= progressInterval;
392: long now = System.currentTimeMillis();
393: System.out.println("loaded " + count + " records "
394: + (now - lastTime) + " ms - % completed: "
395: + ((100 * totalBytesRead) / totalLoadBytes));
396: lastTime = now;
397: }
398:
399: keyLine = reader.readLine();
400: if (keyLine == null) {
401: throw new DatabaseException("No \"DATA=END\"");
402: }
403: bytesReadThisInterval += keyLine.length() + 1;
404: }
405: }
406:
407: private byte[] loadLine(String line) throws DatabaseException {
408:
409: if (formatUsingPrintable) {
410: return readPrintableLine(line);
411: }
412: int nBytes = line.length() / 2;
413: byte[] ret = new byte[nBytes];
414: int charIdx = 0;
415: for (int i = 0; i < nBytes; i++, charIdx += 2) {
416: int b2 = Character.digit(line.charAt(charIdx), 16);
417: b2 <<= 4;
418: b2 += Character.digit(line.charAt(charIdx + 1), 16);
419: ret[i] = (byte) b2;
420: }
421: return ret;
422: }
423:
424: static private byte backSlashValue = (byte) (new Character('\\')
425: .charValue() & 0xff);
426:
427: private byte[] readPrintableLine(String line)
428: throws DatabaseException {
429:
430: /* nBytes is the max number of bytes that this line could turn into. */
431: int maxNBytes = line.length();
432: byte[] ba = new byte[maxNBytes];
433: int actualNBytes = 0;
434:
435: for (int charIdx = 0; charIdx < maxNBytes; charIdx++) {
436: char c = line.charAt(charIdx);
437: if (c == '\\') {
438: if (++charIdx < maxNBytes) {
439: char c1 = line.charAt(charIdx);
440: if (c1 == '\\') {
441: ba[actualNBytes++] = backSlashValue;
442: } else {
443: if (++charIdx < maxNBytes) {
444: char c2 = line.charAt(charIdx);
445: int b = Character.digit(c1, 16);
446: b <<= 4;
447: b += Character.digit(c2, 16);
448: ba[actualNBytes++] = (byte) b;
449: } else {
450: throw new DatabaseException(
451: "Corrupted file");
452: }
453: }
454: } else {
455: throw new DatabaseException("Corrupted file");
456: }
457: } else {
458: ba[actualNBytes++] = (byte) (c & 0xff);
459: }
460: }
461:
462: if (maxNBytes == actualNBytes) {
463: return ba;
464: } else {
465: byte[] ret = new byte[actualNBytes];
466: System.arraycopy(ba, 0, ret, 0, actualNBytes);
467: return ret;
468: }
469: }
470: }
|