001: /*
002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
003: *
004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
005: *
006: * The contents of this file are subject to the terms of either the GNU
007: * General Public License Version 2 only ("GPL") or the Common
008: * Development and Distribution License("CDDL") (collectively, the
009: * "License"). You may not use this file except in compliance with the
010: * License. You can obtain a copy of the License at
011: * http://www.netbeans.org/cddl-gplv2.html
012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
013: * specific language governing permissions and limitations under the
014: * License. When distributing the software, include this License Header
015: * Notice in each file and include the License file at
016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
017: * particular file as subject to the "Classpath" exception as provided
018: * by Sun in the GPL Version 2 section of the License file that
019: * accompanied this code. If applicable, add the following below the
020: * License Header, with the fields enclosed by brackets [] replaced by
021: * your own identifying information:
022: * "Portions Copyrighted [year] [name of copyright owner]"
023: *
024: * Contributor(s):
025: *
026: * The Original Software is NetBeans. The Initial Developer of the Original
027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
028: * Microsystems, Inc. All Rights Reserved.
029: *
030: * If you wish your version of this file to be governed by only the CDDL
031: * or only the GPL Version 2, indicate your decision by adding
032: * "[Contributor] elects to include this software in this distribution
033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
034: * single choice of license, a recipient has the option to distribute
035: * your version of this file under either the CDDL, the GPL Version 2 or
036: * to extend the choice of license to its licensees as provided above.
037: * However, if you add GPL Version 2 code and therefore, elected the GPL
038: * Version 2 license, then the option applies only if the new code is
039: * made subject to such option by the copyright holder.
040: */
041:
042: package org.netbeans.lib.lexer;
043:
044: import java.util.List;
045: import java.util.Set;
046: import org.netbeans.api.lexer.LanguagePath;
047: import org.netbeans.lib.editor.util.FlyOffsetGapList;
048: import org.netbeans.lib.lexer.inc.MutableTokenList;
049: import org.netbeans.api.lexer.InputAttributes;
050: import org.netbeans.api.lexer.Token;
051: import org.netbeans.api.lexer.TokenId;
052: import org.netbeans.lib.lexer.inc.TokenListChange;
053: import org.netbeans.spi.lexer.LanguageEmbedding;
054: import org.netbeans.lib.lexer.token.AbstractToken;
055: import org.netbeans.lib.lexer.token.TextToken;
056:
057: /**
058: * Embedded token list maintains a list of tokens
059: * on a particular embedded language level .
060: * <br>
061: * The physical storage contains a gap to speed up list modifications
062: * during typing in a document when tokens are typically added/removed
063: * at the same index in the list.
064: *
065: * <p>
066: * There is an intent to not degrade performance significantly
067: * with each extra language embedding level so the token list maintains direct
068: * link to the root level.
069: *
070: * @author Miloslav Metelka
071: * @version 1.00
072: */
073:
074: public final class EmbeddedTokenList<T extends TokenId> extends
075: FlyOffsetGapList<Object> implements MutableTokenList<T> {
076:
077: /** Flag for additional correctness checks (may degrade performance). */
078: private static final boolean testing = Boolean
079: .getBoolean("netbeans.debug.lexer.test");
080:
081: /**
082: * Marker value that represents that an attempt to create default embedding was
083: * made but was unsuccessful.
084: */
085: public static final EmbeddedTokenList<TokenId> NO_DEFAULT_EMBEDDING = new EmbeddedTokenList<TokenId>(
086: null, null, null, null);
087:
088: /**
089: * Embedding container carries info about the token into which this
090: * token list is embedded.
091: */
092: private EmbeddingContainer<?> embeddingContainer; // 36 bytes (32-super + 4)
093:
094: /**
095: * Language embedding for this embedded token list.
096: */
097: private final LanguageEmbedding<T> embedding; // 40 bytes
098:
099: /**
100: * Language path of this token list.
101: */
102: private final LanguagePath languagePath; // 44 bytes
103:
104: /**
105: * Storage for lookaheads and states.
106: * <br/>
107: * It's non-null only initialized for mutable token lists
108: * or when in testing environment.
109: */
110: private LAState laState; // 48 bytes
111:
112: /**
113: * Next embedded token list forming a single-linked list.
114: */
115: private EmbeddedTokenList<?> nextEmbeddedTokenList; // 52 bytes
116:
117: public EmbeddedTokenList(EmbeddingContainer<?> embeddingContainer,
118: LanguagePath languagePath, LanguageEmbedding<T> embedding,
119: EmbeddedTokenList<?> nextEmbedding) {
120: this .embeddingContainer = embeddingContainer;
121: this .languagePath = languagePath;
122: this .embedding = embedding;
123: this .nextEmbeddedTokenList = nextEmbedding;
124:
125: if (embeddingContainer != null) { // ec may be null for NO_DEFAULT_EMBEDDING only
126: laState = LAState.initState();
127: embeddingContainer.updateStatusImpl(); // Ensure startOffset() is up-to-date
128: }
129: }
130:
131: private void init() {
132: if (embedding.joinSections()) {
133: // Find the token list list - it should also init this token list
134: root().tokenHierarchyOperation()
135: .tokenListList(languagePath);
136: } else { // not joining => can lex individually
137: init(null);
138: }
139: }
140:
141: public void init(Object relexState) {
142: laState = (modCount() != -1 || testing) ? LAState.empty()
143: : null;
144:
145: // Lex the whole input represented by token at once
146: LexerInputOperation<T> lexerInputOperation = createLexerInputOperation(
147: 0, startOffset(), relexState);
148: AbstractToken<T> token = lexerInputOperation.nextToken();
149: while (token != null) {
150: updateElementOffsetAdd(token); // must subtract startOffset()
151: add(token);
152: if (laState != null) {
153: laState = laState.add(lexerInputOperation.lookahead(),
154: lexerInputOperation.lexerState());
155: }
156: token = lexerInputOperation.nextToken();
157: }
158: lexerInputOperation.release();
159: lexerInputOperation = null;
160:
161: trimToSize(); // Compact storage
162: if (laState != null)
163: laState.trimToSize();
164: }
165:
166: /**
167: * Check whether this embedded token list is initialized.
168: * <br/>
169: * If not then the updating process should not touch it unless
170: * the token list list exists for this particular language path.
171: */
172: public boolean isInited() {
173: return (laState != LAState.initState());
174: }
175:
176: EmbeddedTokenList<?> nextEmbeddedTokenList() {
177: return nextEmbeddedTokenList;
178: }
179:
180: void setNextEmbeddedTokenList(
181: EmbeddedTokenList<?> nextEmbeddedTokenList) {
182: this .nextEmbeddedTokenList = nextEmbeddedTokenList;
183: }
184:
185: public LanguagePath languagePath() {
186: return languagePath;
187: }
188:
189: public LanguageEmbedding embedding() {
190: return embedding;
191: }
192:
193: public int tokenCount() {
194: synchronized (root()) {
195: if (laState == LAState.initState())
196: init();
197: return size();
198: }
199: }
200:
201: public Object tokenOrEmbeddingContainer(int index) {
202: synchronized (root()) {
203: if (laState == LAState.initState())
204: init();
205: return (index < size()) ? get(index) : null;
206: }
207: }
208:
209: private Token existingToken(int index) {
210: return LexerUtilsConstants
211: .token(tokenOrEmbeddingContainer(index));
212: }
213:
214: public int lookahead(int index) {
215: return (laState != null) ? laState.lookahead(index) : -1;
216: }
217:
218: public Object state(int index) {
219: return (laState != null) ? laState.state(index) : null;
220: }
221:
222: /**
223: * Returns absolute offset of the token at the given index
224: * (startOffset gets added to the child token's real offset).
225: * <br/>
226: * For token hierarchy snapshots the returned value is corrected
227: * in the TokenSequence explicitly by adding TokenSequence.tokenOffsetDiff.
228: */
229: public int tokenOffset(int index) {
230: // embeddingContainer().checkStatusUpdated();
231: return elementOffset(index);
232: }
233:
234: public int childTokenOffset(int rawOffset) {
235: // Need to make sure that the startOffset is up-to-date
236: embeddingContainer.updateStatus();
237: return childTokenOffsetNoUpdate(rawOffset);
238: }
239:
240: public int childTokenOffsetNoUpdate(int rawOffset) {
241: // embeddingContainer().checkStatusUpdated();
242: return embeddingContainer.tokenStartOffset()
243: + embedding.startSkipLength()
244: + childTokenRelOffset(rawOffset);
245: }
246:
247: /**
248: * Get difference between start offset of the particular child token
249: * against start offset of the root token.
250: */
251: public int childTokenOffsetShift(int rawOffset) {
252: // Need to make sure that the startOffsetShift is up-to-date
253: embeddingContainer.updateStatus();
254: return embeddingContainer.rootTokenOffsetShift()
255: + childTokenRelOffset(rawOffset);
256: }
257:
258: /**
259: * Get child token's real offset which is always a relative value
260: * to startOffset value.
261: */
262: private int childTokenRelOffset(int rawOffset) {
263: return (rawOffset < offsetGapStart()) ? rawOffset : rawOffset
264: - offsetGapLength();
265: }
266:
267: public char childTokenCharAt(int rawOffset, int index) {
268: // embeddingContainer().checkStatusUpdated();
269: // Do not update the start offset shift - the token.text()
270: // did it before returning its result and its contract
271: // specifies that.
272: // Return chars by delegating to rootToken
273: return embeddingContainer.charAt(embedding.startSkipLength()
274: + childTokenRelOffset(rawOffset) + index);
275: }
276:
277: public int modCount() {
278: // Delegate to root to have the most up-to-date value for token sequence's check.
279: // Extra synchronization should not be necessary since the TokenSequence.embedded()
280: // calls EmbeddingContainer.embeddedTokenList()
281: // which calls which contains the synchronization and calls updateStatusImpl().
282: return embeddingContainer.cachedModCount();
283: }
284:
285: @Override
286: public int startOffset() { // used by FlyOffsetGapList
287: // embeddingContainer.checkStatusUpdated();
288: return embeddingContainer.tokenStartOffset()
289: + embedding.startSkipLength();
290: }
291:
292: public int endOffset() {
293: // embeddingContainer.checkStatusUpdated();
294: return embeddingContainer.tokenStartOffset()
295: + embeddingContainer.token().length()
296: - embedding.endSkipLength();
297: }
298:
299: public boolean isRemoved() {
300: embeddingContainer.updateStatusImpl();
301: return embeddingContainer.isRemoved();
302: }
303:
304: public TokenList<?> root() {
305: return embeddingContainer.rootTokenList();
306: }
307:
308: public TokenHierarchyOperation<?, ?> tokenHierarchyOperation() {
309: return root().tokenHierarchyOperation();
310: }
311:
312: public AbstractToken<?> rootToken() {
313: return embeddingContainer.rootToken();
314: }
315:
316: protected int elementRawOffset(Object elem) {
317: return (elem.getClass() == EmbeddingContainer.class) ? ((EmbeddingContainer) elem)
318: .token().rawOffset()
319: : ((AbstractToken<?>) elem).rawOffset();
320: }
321:
322: protected void setElementRawOffset(Object elem, int rawOffset) {
323: if (elem.getClass() == EmbeddingContainer.class)
324: ((EmbeddingContainer) elem).token().setRawOffset(rawOffset);
325: else
326: ((AbstractToken<?>) elem).setRawOffset(rawOffset);
327: }
328:
329: protected boolean isElementFlyweight(Object elem) {
330: // token wrapper always contains non-flyweight token
331: return (elem.getClass() != EmbeddingContainer.class)
332: && ((AbstractToken<?>) elem).isFlyweight();
333: }
334:
335: protected int elementLength(Object elem) {
336: return LexerUtilsConstants.token(elem).length();
337: }
338:
339: public AbstractToken<T> replaceFlyToken(int index,
340: AbstractToken<T> flyToken, int offset) {
341: synchronized (root()) {
342: TextToken<T> nonFlyToken = ((TextToken<T>) flyToken)
343: .createCopy(this , offset2Raw(offset));
344: set(index, nonFlyToken);
345: return nonFlyToken;
346: }
347: }
348:
349: public void wrapToken(int index,
350: EmbeddingContainer embeddingContainer) {
351: synchronized (root()) {
352: set(index, embeddingContainer);
353: }
354: }
355:
356: public InputAttributes inputAttributes() {
357: return root().inputAttributes();
358: }
359:
360: // MutableTokenList extra methods
361: public Object tokenOrEmbeddingContainerUnsync(int index) {
362: return get(index);
363: }
364:
365: public int tokenCountCurrent() {
366: return size();
367: }
368:
369: public LexerInputOperation<T> createLexerInputOperation(
370: int tokenIndex, int relexOffset, Object relexState) {
371: // embeddingContainer.checkStatusUpdated();
372: CharSequence tokenText = embeddingContainer.token().text();
373: int tokenStartOffset = embeddingContainer.tokenStartOffset();
374: if (tokenText == null) { // Should not normally happen - debug the state
375: throw new IllegalStateException(
376: "Text of parent token is null. tokenStartOffset="
377: + tokenStartOffset + ", tokenIndex="
378: + tokenIndex + ", relexOffset="
379: + relexOffset + ", relexState="
380: + relexState + ", languagePath="
381: + languagePath() + ", inited=" + isInited());
382: }
383: int endOffset = tokenStartOffset + tokenText.length()
384: - embedding.endSkipLength();
385: return new TextLexerInputOperation<T>(this , tokenIndex,
386: relexState, tokenText, tokenStartOffset, relexOffset,
387: endOffset);
388: }
389:
390: public boolean isFullyLexed() {
391: return true;
392: }
393:
394: public void replaceTokens(TokenListChange<T> change,
395: int removeTokenCount, int diffLength) {
396: int index = change.index();
397: // Remove obsolete tokens (original offsets are retained)
398: Object[] removedTokensOrEmbeddingContainers = new Object[removeTokenCount];
399: copyElements(index, index + removeTokenCount,
400: removedTokensOrEmbeddingContainers, 0);
401: int offset = change.offset();
402: for (int i = 0; i < removeTokenCount; i++) {
403: Object tokenOrEmbeddingContainer = removedTokensOrEmbeddingContainers[i];
404: AbstractToken<?> token;
405: // It's necessary to update-status of all removed tokens' contained embeddings
406: // since otherwise (if they would not be up-to-date) they could not be updated later
407: // as they lose their parent token list which the update-status relies on.
408: if (tokenOrEmbeddingContainer.getClass() == EmbeddingContainer.class) {
409: EmbeddingContainer<?> ec = (EmbeddingContainer<?>) tokenOrEmbeddingContainer;
410: ec.updateStatusAndInvalidate();
411: token = ec.token();
412: } else { // Regular token
413: token = (AbstractToken<?>) tokenOrEmbeddingContainer;
414: }
415: if (!token.isFlyweight()) {
416: updateElementOffsetRemove(token);
417: token.setTokenList(null);
418: }
419: offset += token.length();
420: }
421: remove(index, removeTokenCount); // Retain original offsets
422: laState.remove(index, removeTokenCount); // Remove lookaheads and states
423: change.setRemovedTokens(removedTokensOrEmbeddingContainers);
424: change.setRemovedEndOffset(offset);
425:
426: // Move and fix the gap according to the performed modification.
427: int startOffset = startOffset(); // updateStatus() should already be called
428: if (offsetGapStart() != change.offset() - startOffset) {
429: // Minimum of the index of the first removed index and original computed index
430: moveOffsetGap(change.offset() - startOffset, Math.min(
431: index, change.offsetGapIndex()));
432: }
433: updateOffsetGapLength(-diffLength);
434:
435: // Add created tokens.
436: // This should be called early when all the members are true tokens
437: List<Object> addedTokensOrBranches = change
438: .addedTokensOrBranches();
439: if (addedTokensOrBranches != null) {
440: for (Object tokenOrBranch : addedTokensOrBranches) {
441: @SuppressWarnings("unchecked")
442: AbstractToken<T> token = (AbstractToken<T>) tokenOrBranch;
443: updateElementOffsetAdd(token);
444: }
445: addAll(index, addedTokensOrBranches);
446: laState = laState.addAll(index, change.laState());
447: change.syncAddedTokenCount();
448: // Check for bounds change only
449: if (removeTokenCount == 1
450: && addedTokensOrBranches.size() == 1) {
451: // Compare removed and added token ids and part types
452: AbstractToken<T> removedToken = LexerUtilsConstants
453: .token(removedTokensOrEmbeddingContainers[0]);
454: AbstractToken<T> addedToken = change.addedToken(0);
455: if (removedToken.id() == addedToken.id()
456: && removedToken.partType() == addedToken
457: .partType()) {
458: change.markBoundsChange();
459: }
460: }
461: }
462: }
463:
464: public boolean isContinuous() {
465: return true;
466: }
467:
468: public Set<T> skipTokenIds() {
469: return null;
470: }
471:
472: public EmbeddingContainer<?> embeddingContainer() {
473: return embeddingContainer;
474: }
475:
476: public void setEmbeddingContainer(
477: EmbeddingContainer<?> embeddingContainer) {
478: this .embeddingContainer = embeddingContainer;
479: }
480:
481: public String toStringHeader() {
482: StringBuilder sb = new StringBuilder(50);
483: sb.append("ETL: <").append(startOffset());
484: sb.append(",").append(endOffset());
485: sb.append(">");
486: sb.append(" IHC=").append(System.identityHashCode(this ));
487: sb.append('\n');
488: return sb.toString();
489: }
490:
491: @Override
492: public String toString() {
493: StringBuilder sb = new StringBuilder(256);
494: sb.append(toStringHeader());
495: LexerUtilsConstants.appendTokenList(sb, this);
496: return sb.toString();
497: }
498:
499: }
|