001: /*
002: * Copyright 2004-2006 the original author or authors.
003: *
004: * Licensed under the Apache License, Version 2.0 (the "License");
005: * you may not use this file except in compliance with the License.
006: * You may obtain a copy of the License at
007: *
008: * http://www.apache.org/licenses/LICENSE-2.0
009: *
010: * Unless required by applicable law or agreed to in writing, software
011: * distributed under the License is distributed on an "AS IS" BASIS,
012: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013: * See the License for the specific language governing permissions and
014: * limitations under the License.
015: */
016:
017: package org.apache.lucene.store.jdbc;
018:
019: import java.io.IOException;
020: import java.sql.CallableStatement;
021: import java.sql.PreparedStatement;
022: import java.sql.ResultSet;
023: import java.sql.Timestamp;
024: import java.util.ArrayList;
025: import java.util.HashMap;
026: import java.util.Iterator;
027: import java.util.List;
028: import java.util.Map;
029: import javax.sql.DataSource;
030:
031: import org.apache.lucene.store.Directory;
032: import org.apache.lucene.store.DirectoryTemplate;
033: import org.apache.lucene.store.IndexInput;
034: import org.apache.lucene.store.IndexOutput;
035: import org.apache.lucene.store.Lock;
036: import org.apache.lucene.store.MultiDeleteDirectory;
037: import org.apache.lucene.store.jdbc.dialect.Dialect;
038: import org.apache.lucene.store.jdbc.dialect.DialectResolver;
039: import org.apache.lucene.store.jdbc.handler.FileEntryHandler;
040: import org.apache.lucene.store.jdbc.lock.JdbcLock;
041: import org.apache.lucene.store.jdbc.support.JdbcTable;
042: import org.apache.lucene.store.jdbc.support.JdbcTemplate;
043:
044: /**
045: * A Jdbc based implementation of a Lucene <code>Directory</code> allowing the storage of a Lucene index
046: * within a database. Uses a jdbc <code>DataSource</code>, {@link Dialect} specific for the database used,
047: * and an optional {@link JdbcDirectorySettings} and {@link JdbcTable} for configuration.
048: * <p/>
049: * The directory works against a single table, where the binary data is stored in <code>Blob</code>. Each "file"
050: * has an entry in the database, and different {@link FileEntryHandler} can be defines for different files (or
051: * files groups).
052: * <p/>
053: * Most of the files will not be deleted from the database when the directory delete method is called, but will
054: * only be marked to be deleted (see {@link org.apache.lucene.store.jdbc.handler.MarkDeleteFileEntryHandler}. It is
055: * done since other readers or searchers might be working with the database, and still use the files. The ability to
056: * purge mark deleted files based on a "delta" is acheived using {@link #deleteMarkDeleted()} and
057: * {@link #deleteMarkDeleted(long)}. Note, the purging process is not called by the directory code, so it will have
058: * to be managed by the application using the jdbc directory.
059: * <p/>
060: * For transaction management, all the operations performed against the database do not call <code>commit</code> or
061: * <code>rollback</code>. They simply open a connection (using
062: * {@link org.apache.lucene.store.jdbc.datasource.DataSourceUtils#getConnection(javax.sql.DataSource)} ), and close it
063: * using {@link org.apache.lucene.store.jdbc.datasource.DataSourceUtils#releaseConnection(java.sql.Connection)}). This
064: * results in the fact that transcation management is simple and wraps the directory operations, allowing it to span
065: * as many operations as needed.
066: * <p/>
067: * For none managed applications (i.e. applications that do not use JTA or Spring transaction manager), the jdbc directory
068: * implementation comes with {@link org.apache.lucene.store.jdbc.datasource.TransactionAwareDataSourceProxy} which wraps
069: * a <code>DataSource</code> (should be a pooled one, like Jakartat DBCP). Using it with the
070: * {@link org.apache.lucene.store.jdbc.datasource.DataSourceUtils}, or the provided {@link DirectoryTemplate} should make
071: * integrating or using jdbc directory simple.
072: * <p/>
073: * Also, for none managed applications, there is an option working with autoCommit=true mode. The system will work much
074: * slower, and it is only supported on a portion of the databases, but any existing code that uses Lucene with any
075: * other <code>Directory</code> implemenation should work as is.
076: * <p/>
077: * If working within managed environments, an external transaction management should be performed (using JTA for example).
078: * Simple solutions can be using CMT or Spring Framework abstraction of transaction managers. Currently, the jdbc directory
079: * implementation does not implement a transaction management abstraction, since there is a very good solution out there
080: * already (Spring and JTA). Note, when using Spring and the <code>DataSourceTransactionManager</code>, to provide the jdbc directory
081: * with a Spring's <code>TransactionAwareDataSourceProxy</code>.
082: *
083: * @author kimchy
084: */
085: public class JdbcDirectory extends Directory implements
086: MultiDeleteDirectory {
087:
088: private Dialect dialect;
089:
090: private DataSource dataSource;
091:
092: private JdbcTable table;
093:
094: private JdbcDirectorySettings settings;
095:
096: private HashMap fileEntryHandlers = new HashMap();
097:
098: private JdbcTemplate jdbcTemplate;
099:
100: /**
101: * Creates a new jdbc directory. Creates new {@link JdbcDirectorySettings} using it's default values.
102: * Uses {@link DialectResolver} to try and automatically reolve the {@link Dialect}.
103: *
104: * @param dataSource The data source to use
105: * @param tableName The table name
106: * @throws JdbcStoreException
107: */
108: public JdbcDirectory(DataSource dataSource, String tableName)
109: throws JdbcStoreException {
110: Dialect dialect = new DialectResolver().getDialect(dataSource);
111: initialize(dataSource, new JdbcTable(
112: new JdbcDirectorySettings(), dialect, tableName));
113: }
114:
115: /**
116: * Creates a new jdbc directory. Creates new {@link JdbcDirectorySettings} using it's default values.
117: *
118: * @param dataSource The data source to use
119: * @param dialect The dialect
120: * @param tableName The table name
121: */
122: public JdbcDirectory(DataSource dataSource, Dialect dialect,
123: String tableName) {
124: initialize(dataSource, new JdbcTable(
125: new JdbcDirectorySettings(), dialect, tableName));
126: }
127:
128: /**
129: * Creates a new jdbc directory. Uses {@link DialectResolver} to try and automatically reolve the {@link Dialect}.
130: *
131: * @param dataSource The data source to use
132: * @param settings The settings to configure the directory
133: * @param tableName The table name that will be used
134: */
135: public JdbcDirectory(DataSource dataSource,
136: JdbcDirectorySettings settings, String tableName)
137: throws JdbcStoreException {
138: Dialect dialect = new DialectResolver().getDialect(dataSource);
139: initialize(dataSource, new JdbcTable(settings, dialect,
140: tableName));
141: }
142:
143: /**
144: * Creates a new jdbc directory.
145: *
146: * @param dataSource The data source to use
147: * @param dialect The dialect
148: * @param settings The settings to configure the directory
149: * @param tableName The table name that will be used
150: */
151: public JdbcDirectory(DataSource dataSource, Dialect dialect,
152: JdbcDirectorySettings settings, String tableName) {
153: initialize(dataSource, new JdbcTable(settings, dialect,
154: tableName));
155: }
156:
157: /**
158: * Creates a new jdbc directory.
159: *
160: * @param dataSource The data source to use
161: * @param table The Jdbc table definitions
162: */
163: public JdbcDirectory(DataSource dataSource, JdbcTable table) {
164: initialize(dataSource, table);
165: }
166:
167: private void initialize(DataSource dataSource, JdbcTable table) {
168: this .dataSource = dataSource;
169: this .jdbcTemplate = new JdbcTemplate(dataSource, table
170: .getSettings());
171: this .dialect = table.getDialect();
172: this .table = table;
173: this .settings = table.getSettings();
174: dialect.processSettings(settings);
175: Map fileEntrySettings = settings.getFileEntrySettings();
176: // go over all the file entry settings and configure them
177: for (Iterator it = fileEntrySettings.keySet().iterator(); it
178: .hasNext();) {
179: String name = (String) it.next();
180: JdbcFileEntrySettings feSettings = ((JdbcFileEntrySettings) fileEntrySettings
181: .get(name));
182: try {
183: Class fileEntryHandlerClass = feSettings
184: .getSettingAsClass(
185: JdbcFileEntrySettings.FILE_ENTRY_HANDLER_TYPE,
186: null);
187: FileEntryHandler fileEntryHandler = (FileEntryHandler) fileEntryHandlerClass
188: .newInstance();
189: fileEntryHandler.configure(this );
190: fileEntryHandlers.put(name, fileEntryHandler);
191: } catch (Exception e) {
192: throw new IllegalArgumentException(
193: "Failed to create FileEntryHandler ["
194: + feSettings
195: .getSetting(JdbcFileEntrySettings.FILE_ENTRY_HANDLER_TYPE)
196: + "]");
197: }
198: }
199: }
200:
201: /**
202: * Returns <code>true</code> if the database table exists.
203: *
204: * @return <code>true</code> if the database table exists, <code>false</code> otherwise
205: * @throws IOException
206: * @throws UnsupportedOperationException If the database dialect does not support it
207: */
208: public boolean tableExists() throws IOException,
209: UnsupportedOperationException {
210: Boolean tableExists = (Boolean) jdbcTemplate.executeSelect(
211: dialect.sqlTableExists(table.getCatalog(), table
212: .getSchema()),
213: new JdbcTemplate.ExecuteSelectCallback() {
214: public void fillPrepareStatement(
215: PreparedStatement ps) throws Exception {
216: ps.setFetchSize(1);
217: ps.setString(1, table.getName().toLowerCase());
218: }
219:
220: public Object execute(ResultSet rs)
221: throws Exception {
222: if (rs.next()) {
223: return Boolean.TRUE;
224: }
225: return Boolean.FALSE;
226: }
227: });
228: return tableExists.booleanValue();
229: }
230:
231: /**
232: * Deletes the database table (drops it) from the database.
233: *
234: * @throws IOException
235: */
236: public void delete() throws IOException {
237: if (!dialect.supportsIfExistsAfterTableName()
238: && !dialect.supportsIfExistsBeforeTableName()) {
239: // there are databases where the fact that an exception was thrown, invalidates the connection
240: // so if they do not support "if exists" in the drop clause, we will try to check first if the
241: // table exists.
242: if (dialect.supportsTableExists() && !tableExists()) {
243: return;
244: }
245: }
246: jdbcTemplate.executeUpdate(table.sqlDrop());
247: }
248:
249: /**
250: * Creates a new database table. Drops it before hand.
251: *
252: * @throws IOException
253: */
254: public void create() throws IOException {
255: try {
256: delete();
257: } catch (Exception e) {
258: //e.printStackTrace();
259: }
260: jdbcTemplate.executeUpdate(table.sqlCreate());
261: ((JdbcLock) createLock()).initializeDatabase(this );
262: }
263:
264: /**
265: * Deletes the contents of the database, except for the commit and write lock.
266: *
267: * @throws IOException
268: */
269: public void deleteContent() throws IOException {
270: jdbcTemplate.executeUpdate(table.sqlDeletaAll());
271: }
272:
273: /**
274: * Delets all the file entries that are marked to be deleted, and they were marked
275: * "delta" time ago (base on database time, if possible by dialect). The delta is
276: * taken from {@link org.apache.lucene.store.jdbc.JdbcDirectorySettings#getDeleteMarkDeletedDelta()}.
277: */
278: public void deleteMarkDeleted() throws IOException {
279: deleteMarkDeleted(settings.getDeleteMarkDeletedDelta());
280: }
281:
282: /**
283: * Delets all the file entries that are marked to be deleted, and they were marked
284: * "delta" time ago (base on database time, if possible by dialect).
285: */
286: public void deleteMarkDeleted(long delta) throws IOException {
287: long currentTime = System.currentTimeMillis();
288: if (dialect.supportsCurrentTimestampSelection()) {
289: String timestampSelectString = dialect
290: .getCurrentTimestampSelectString();
291: if (dialect.isCurrentTimestampSelectStringCallable()) {
292: currentTime = ((Long) jdbcTemplate.executeCallable(
293: timestampSelectString,
294: new JdbcTemplate.CallableStatementCallback() {
295: public void fillCallableStatement(
296: CallableStatement cs)
297: throws Exception {
298: cs.registerOutParameter(1,
299: java.sql.Types.TIMESTAMP);
300: }
301:
302: public Object readCallableData(
303: CallableStatement cs)
304: throws Exception {
305: Timestamp timestamp = cs
306: .getTimestamp(1);
307: return new Long(timestamp.getTime());
308: }
309: })).longValue();
310: } else {
311: currentTime = ((Long) jdbcTemplate.executeSelect(
312: timestampSelectString,
313: new JdbcTemplate.ExecuteSelectCallback() {
314: public void fillPrepareStatement(
315: PreparedStatement ps)
316: throws Exception {
317: // nothing to do here
318: }
319:
320: public Object execute(ResultSet rs)
321: throws Exception {
322: rs.next();
323: Timestamp timestamp = rs
324: .getTimestamp(1);
325: return new Long(timestamp.getTime());
326: }
327: })).longValue();
328: }
329: }
330: final long deleteBefore = currentTime - delta;
331: jdbcTemplate.executeUpdate(table.sqlDeletaMarkDeleteByDelta(),
332: new JdbcTemplate.PrepateStatementAwareCallback() {
333: public void fillPrepareStatement(
334: PreparedStatement ps) throws Exception {
335: ps.setBoolean(1, true);
336: ps.setTimestamp(2, new Timestamp(deleteBefore));
337: }
338: });
339: }
340:
341: public String[] list() throws IOException {
342: return (String[]) jdbcTemplate.executeSelect(table
343: .sqlSelectNames(),
344: new JdbcTemplate.ExecuteSelectCallback() {
345: public void fillPrepareStatement(
346: PreparedStatement ps) throws Exception {
347: ps.setBoolean(1, false);
348: }
349:
350: public Object execute(ResultSet rs)
351: throws Exception {
352: ArrayList names = new ArrayList();
353: while (rs.next()) {
354: names.add(rs.getString(1));
355: }
356: return (String[]) names
357: .toArray(new String[names.size()]);
358: }
359: });
360: }
361:
362: public boolean fileExists(final String name) throws IOException {
363: return getFileEntryHandler(name).fileExists(name);
364: }
365:
366: public long fileModified(final String name) throws IOException {
367: return getFileEntryHandler(name).fileModified(name);
368: }
369:
370: public void touchFile(final String name) throws IOException {
371: getFileEntryHandler(name).touchFile(name);
372: }
373:
374: public void deleteFile(final String name) throws IOException {
375: getFileEntryHandler(name).deleteFile(name);
376: }
377:
378: public List deleteFiles(List names) throws IOException {
379: HashMap tempMap = new HashMap();
380: for (Iterator it = names.iterator(); it.hasNext();) {
381: String name = (String) it.next();
382: FileEntryHandler fileEntryHandler = getFileEntryHandler(name);
383: ArrayList tempNames = (ArrayList) tempMap
384: .get(fileEntryHandler);
385: if (tempNames == null) {
386: tempNames = new ArrayList(names.size());
387: tempMap.put(fileEntryHandler, tempNames);
388: }
389: tempNames.add(name);
390: }
391: ArrayList notDeleted = new ArrayList(names.size() / 2);
392: for (Iterator it = tempMap.keySet().iterator(); it.hasNext();) {
393: FileEntryHandler fileEntryHandler = (FileEntryHandler) it
394: .next();
395: List tempNames = (ArrayList) tempMap.get(fileEntryHandler);
396: tempNames = fileEntryHandler.deleteFiles(tempNames);
397: if (tempNames != null) {
398: notDeleted.addAll(tempNames);
399: }
400: }
401: return notDeleted;
402: }
403:
404: public void renameFile(final String from, final String to)
405: throws IOException {
406: getFileEntryHandler(from).renameFile(from, to);
407: }
408:
409: public long fileLength(final String name) throws IOException {
410: return getFileEntryHandler(name).fileLength(name);
411: }
412:
413: public IndexInput openInput(String name) throws IOException {
414: return getFileEntryHandler(name).openInput(name);
415: }
416:
417: public IndexOutput createOutput(String name) throws IOException {
418: return getFileEntryHandler(name).createOutput(name);
419: }
420:
421: public Lock makeLock(final String name) {
422: try {
423: Lock lock = createLock();
424: ((JdbcLock) lock).configure(this , name);
425: return lock;
426: } catch (IOException e) {
427: // shoule not happen
428: return null;
429: }
430: }
431:
432: /**
433: * Closes the directory.
434: */
435: public void close() throws IOException {
436: IOException last = null;
437: for (Iterator it = fileEntryHandlers.values().iterator(); it
438: .hasNext();) {
439: FileEntryHandler fileEntryHandler = (FileEntryHandler) it
440: .next();
441: try {
442: fileEntryHandler.close();
443: } catch (IOException e) {
444: last = e;
445: }
446: }
447: if (last != null) {
448: throw last;
449: }
450: }
451:
452: protected FileEntryHandler getFileEntryHandler(String name) {
453: FileEntryHandler handler = (FileEntryHandler) fileEntryHandlers
454: .get(name.substring(name.length() - 3));
455: if (handler != null) {
456: return handler;
457: }
458: handler = (FileEntryHandler) fileEntryHandlers.get(name);
459: if (handler != null) {
460: return handler;
461: }
462: return (FileEntryHandler) fileEntryHandlers
463: .get(JdbcDirectorySettings.DEFAULT_FILE_ENTRY);
464:
465: }
466:
467: protected Lock createLock() throws IOException {
468: try {
469: return (Lock) settings.getLockClass().newInstance();
470: } catch (Exception e) {
471: throw new JdbcStoreException(
472: "Failed to create lock class ["
473: + settings.getLockClass() + "]");
474: }
475: }
476:
477: public Dialect getDialect() {
478: return dialect;
479: }
480:
481: public JdbcTemplate getJdbcTemplate() {
482: return this .jdbcTemplate;
483: }
484:
485: public JdbcTable getTable() {
486: return this .table;
487: }
488:
489: public JdbcDirectorySettings getSettings() {
490: return settings;
491: }
492:
493: public DataSource getDataSource() {
494: return dataSource;
495: }
496: }
|