01: package it.unimi.dsi.mg4j.index;
02:
03: /*
04: * MG4J: Managing Gigabytes for Java
05: *
06: * Copyright (C) 2007 Sebastiano Vigna
07: *
08: * This library is free software; you can redistribute it and/or modify it
09: * under the terms of the GNU Lesser General Public License as published by the Free
10: * Software Foundation; either version 2.1 of the License, or (at your option)
11: * any later version.
12: *
13: * This library is distributed in the hope that it will be useful, but
14: * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15: * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
16: * for more details.
17: *
18: * You should have received a copy of the GNU Lesser General Public License
19: * along with this program; if not, write to the Free Software
20: * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21: *
22: */
23:
24: import it.unimi.dsi.fastutil.ints.IntList;
25: import it.unimi.dsi.fastutil.longs.LongList;
26: import it.unimi.dsi.mg4j.index.CompressionFlags.Coding;
27: import it.unimi.dsi.mg4j.index.payload.Payload;
28: import it.unimi.dsi.io.ByteBufferInputStream;
29: import it.unimi.dsi.io.InputBitStream;
30: import it.unimi.dsi.mg4j.tool.PartitionLexically;
31: import it.unimi.dsi.util.Properties;
32: import it.unimi.dsi.util.StringMap;
33:
34: import java.nio.ByteBuffer;
35: import java.nio.MappedByteBuffer;
36:
37: import it.unimi.dsi.util.PrefixMap;
38:
39: /** A local memory-mapped bistream index.
40: *
41: * <p>Memory-mapped indices are created by mapping the index file into memory
42: * using a {@link MappedByteBuffer}. The main advantage over an {@link InMemoryIndex}
43: * is that only the most frequently used parts of the index will be loaded in core memory.
44: *
45: * <p>Note that due to insurmountable Java limitations, it is impossible to map an index larger than
46: * 2GiB. However, you can {@linkplain PartitionLexically partition lexically} an index so that
47: * the resulting segments are smaller than 2GiB, and modify the property file of the resulting
48: * cluster so that the URIs of the local indices require memory mapping. This will effectively memory-map
49: * the whole index.
50: *
51: * @author Sebastiano Vigna
52: * @since 1.2
53: */
54:
55: public class MemoryMappedIndex extends BitStreamIndex {
56: private static final long serialVersionUID = 0L;
57:
58: /** The byte buffer containing the index. */
59: protected final ByteBuffer index;
60:
61: public MemoryMappedIndex(final ByteBuffer index,
62: final int numberOfDocuments, final int numberOfTerms,
63: final long numberOfPostings,
64: final long numberOfOccurrences, final int maxCount,
65: final Payload payload, final Coding frequencyCoding,
66: final Coding pointerCoding, final Coding countCoding,
67: final Coding positionCoding, final int quantum,
68: final int height, final TermProcessor termProcessor,
69: final String field, final Properties properties,
70: final StringMap<? extends CharSequence> termMap,
71: final PrefixMap<? extends CharSequence> prefixMap,
72: final IntList sizes, final LongList offsets) {
73: super (numberOfDocuments, numberOfTerms, numberOfPostings,
74: numberOfOccurrences, maxCount, payload,
75: frequencyCoding, pointerCoding, countCoding,
76: positionCoding, quantum, height, -1, termProcessor,
77: field, properties, termMap, prefixMap, sizes, offsets);
78: this .index = index;
79: }
80:
81: @Override
82: public InputBitStream getInputBitStream(int bufferSizeUnused) {
83: return new InputBitStream(getInputStream());
84: }
85:
86: @Override
87: public ByteBufferInputStream getInputStream() {
88: return new ByteBufferInputStream((ByteBuffer) index.duplicate()
89: .position(0));
90: }
91: }
|