001: package org.apache.lucene.analysis;
002:
003: /**
004: * Licensed to the Apache Software Foundation (ASF) under one or more
005: * contributor license agreements. See the NOTICE file distributed with
006: * this work for additional information regarding copyright ownership.
007: * The ASF licenses this file to You under the Apache License, Version 2.0
008: * (the "License"); you may not use this file except in compliance with
009: * the License. You may obtain a copy of the License at
010: *
011: * http://www.apache.org/licenses/LICENSE-2.0
012: *
013: * Unless required by applicable law or agreed to in writing, software
014: * distributed under the License is distributed on an "AS IS" BASIS,
015: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016: * See the License for the specific language governing permissions and
017: * limitations under the License.
018: */
019:
020: import java.io.IOException;
021: import java.util.Arrays;
022: import java.util.Set;
023:
024: /**
025: * Removes stop words from a token stream.
026: */
027:
028: public final class StopFilter extends TokenFilter {
029:
030: private static boolean ENABLE_POSITION_INCREMENTS_DEFAULT = false;
031:
032: private final CharArraySet stopWords;
033: private boolean enablePositionIncrements = ENABLE_POSITION_INCREMENTS_DEFAULT;
034:
035: /**
036: * Construct a token stream filtering the given input.
037: */
038: public StopFilter(TokenStream input, String[] stopWords) {
039: this (input, stopWords, false);
040: }
041:
042: /**
043: * Constructs a filter which removes words from the input
044: * TokenStream that are named in the array of words.
045: */
046: public StopFilter(TokenStream in, String[] stopWords,
047: boolean ignoreCase) {
048: super (in);
049: this .stopWords = (CharArraySet) makeStopSet(stopWords,
050: ignoreCase);
051: }
052:
053: /**
054: * Construct a token stream filtering the given input.
055: * If <code>stopWords</code> is an instance of {@link CharArraySet} (true if
056: * <code>makeStopSet()</code> was used to construct the set) it will be directly used
057: * and <code>ignoreCase</code> will be ignored since <code>CharArraySet</code>
058: * directly controls case sensitivity.
059: * <p/>
060: * If <code>stopWords</code> is not an instance of {@link CharArraySet},
061: * a new CharArraySet will be constructed and <code>ignoreCase</code> will be
062: * used to specify the case sensitivity of that set.
063: *
064: * @param input
065: * @param stopWords The set of Stop Words.
066: * @param ignoreCase -Ignore case when stopping.
067: */
068: public StopFilter(TokenStream input, Set stopWords,
069: boolean ignoreCase) {
070: super (input);
071: if (stopWords instanceof CharArraySet) {
072: this .stopWords = (CharArraySet) stopWords;
073: } else {
074: this .stopWords = new CharArraySet(stopWords.size(),
075: ignoreCase);
076: this .stopWords.addAll(stopWords);
077: }
078: }
079:
080: /**
081: * Constructs a filter which removes words from the input
082: * TokenStream that are named in the Set.
083: *
084: * @see #makeStopSet(java.lang.String[])
085: */
086: public StopFilter(TokenStream in, Set stopWords) {
087: this (in, stopWords, false);
088: }
089:
090: /**
091: * Builds a Set from an array of stop words,
092: * appropriate for passing into the StopFilter constructor.
093: * This permits this stopWords construction to be cached once when
094: * an Analyzer is constructed.
095: *
096: * @see #makeStopSet(java.lang.String[], boolean) passing false to ignoreCase
097: */
098: public static final Set makeStopSet(String[] stopWords) {
099: return makeStopSet(stopWords, false);
100: }
101:
102: /**
103: *
104: * @param stopWords
105: * @param ignoreCase If true, all words are lower cased first.
106: * @return a Set containing the words
107: */
108: public static final Set makeStopSet(String[] stopWords,
109: boolean ignoreCase) {
110: CharArraySet stopSet = new CharArraySet(stopWords.length,
111: ignoreCase);
112: stopSet.addAll(Arrays.asList(stopWords));
113: return stopSet;
114: }
115:
116: /**
117: * Returns the next input Token whose termText() is not a stop word.
118: */
119: public final Token next(Token result) throws IOException {
120: // return the first non-stop word found
121: int skippedPositions = 0;
122: while ((result = input.next(result)) != null) {
123: if (!stopWords.contains(result.termBuffer(), 0,
124: result.termLength)) {
125: if (enablePositionIncrements) {
126: result.setPositionIncrement(result
127: .getPositionIncrement()
128: + skippedPositions);
129: }
130: return result;
131: }
132: skippedPositions += result.getPositionIncrement();
133: }
134: // reached EOS -- return null
135: return null;
136: }
137:
138: /**
139: * @see #setEnablePositionIncrementsDefault(boolean).
140: */
141: public static boolean getEnablePositionIncrementsDefault() {
142: return ENABLE_POSITION_INCREMENTS_DEFAULT;
143: }
144:
145: /**
146: * Set the default position increments behavior of every StopFilter created from now on.
147: * <p>
148: * Note: behavior of a single StopFilter instance can be modified
149: * with {@link #setEnablePositionIncrements(boolean)}.
150: * This static method allows control over behavior of classes using StopFilters internally,
151: * for example {@link org.apache.lucene.analysis.standard.StandardAnalyzer StandardAnalyzer}.
152: * <p>
153: * Default : false.
154: * @see #setEnablePositionIncrements(boolean).
155: */
156: public static void setEnablePositionIncrementsDefault(
157: boolean defaultValue) {
158: ENABLE_POSITION_INCREMENTS_DEFAULT = defaultValue;
159: }
160:
161: /**
162: * @see #setEnablePositionIncrements(boolean).
163: */
164: public boolean getEnablePositionIncrements() {
165: return enablePositionIncrements;
166: }
167:
168: /**
169: * Set to <code>true</code> to make <b>this</b> StopFilter enable position increments to result tokens.
170: * <p>
171: * When set, when a token is stopped (omitted), the position increment of
172: * the following token is incremented.
173: * <p>
174: * Default: see {@link #setEnablePositionIncrementsDefault(boolean)}.
175: */
176: public void setEnablePositionIncrements(boolean enable) {
177: this.enablePositionIncrements = enable;
178: }
179: }
|