0001: /*
0002: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
0003: *
0004: * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
0005: *
0006: * The contents of this file are subject to the terms of either the GNU
0007: * General Public License Version 2 only ("GPL") or the Common
0008: * Development and Distribution License("CDDL") (collectively, the
0009: * "License"). You may not use this file except in compliance with the
0010: * License. You can obtain a copy of the License at
0011: * http://www.netbeans.org/cddl-gplv2.html
0012: * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
0013: * specific language governing permissions and limitations under the
0014: * License. When distributing the software, include this License Header
0015: * Notice in each file and include the License file at
0016: * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
0017: * particular file as subject to the "Classpath" exception as provided
0018: * by Sun in the GPL Version 2 section of the License file that
0019: * accompanied this code. If applicable, add the following below the
0020: * License Header, with the fields enclosed by brackets [] replaced by
0021: * your own identifying information:
0022: * "Portions Copyrighted [year] [name of copyright owner]"
0023: *
0024: * Contributor(s):
0025: *
0026: * The Original Software is NetBeans. The Initial Developer of the Original
0027: * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
0028: * Microsystems, Inc. All Rights Reserved.
0029: *
0030: * If you wish your version of this file to be governed by only the CDDL
0031: * or only the GPL Version 2, indicate your decision by adding
0032: * "[Contributor] elects to include this software in this distribution
0033: * under the [CDDL or GPL Version 2] license." If you do not indicate a
0034: * single choice of license, a recipient has the option to distribute
0035: * your version of this file under either the CDDL, the GPL Version 2 or
0036: * to extend the choice of license to its licensees as provided above.
0037: * However, if you add GPL Version 2 code and therefore, elected the GPL
0038: * Version 2 license, then the option applies only if the new code is
0039: * made subject to such option by the copyright holder.
0040: */
0041:
0042: package org.netbeans.modules.cnd.lexer;
0043:
0044: import junit.framework.TestCase;
0045: import org.netbeans.cnd.api.lexer.CppStringTokenId;
0046: import org.netbeans.cnd.api.lexer.CppTokenId;
0047: import org.netbeans.cnd.api.lexer.DoxygenTokenId;
0048: import org.netbeans.api.lexer.PartType;
0049: import org.netbeans.api.lexer.TokenHierarchy;
0050: import org.netbeans.api.lexer.TokenSequence;
0051: import org.netbeans.lib.lexer.test.LexerTestUtilities;
0052:
0053: /**
0054: * Test several lexer inputs.
0055: *
0056: * @author Vladimir Voskresensky
0057: */
0058: public class CppLexerBatchTestCase extends TestCase {
0059:
0060: public CppLexerBatchTestCase(String testName) {
0061: super (testName);
0062: }
0063:
0064: @Override
0065: protected void setUp() throws java.lang.Exception {
0066: // Set-up testing environment
0067: LexerTestUtilities.setTesting(true);
0068: }
0069:
0070: @Override
0071: protected void tearDown() throws java.lang.Exception {
0072: }
0073:
0074: public void testPreprocEmbedding() {
0075: String text = "#define C 1 \"/*\" /* \n@see C */";
0076: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0077: .languageCpp());
0078: TokenSequence<?> ts = hi.tokenSequence();
0079:
0080: LexerTestUtilities.assertNextTokenEquals(ts,
0081: CppTokenId.PREPROCESSOR_DIRECTIVE,
0082: "#define C 1 \"/*\" /* \n@see C */");
0083:
0084: TokenSequence<?> ep = ts.embedded();
0085: LexerTestUtilities.assertNextTokenEquals(ep,
0086: CppTokenId.PREPROCESSOR_START, "#");
0087: LexerTestUtilities.assertNextTokenEquals(ep,
0088: CppTokenId.PREPROCESSOR_DEFINE, "define");
0089: LexerTestUtilities.assertNextTokenEquals(ep,
0090: CppTokenId.WHITESPACE, " ");
0091: LexerTestUtilities.assertNextTokenEquals(ep,
0092: CppTokenId.PREPROCESSOR_IDENTIFIER, "C");
0093: LexerTestUtilities.assertNextTokenEquals(ep,
0094: CppTokenId.WHITESPACE, " ");
0095: LexerTestUtilities.assertNextTokenEquals(ep,
0096: CppTokenId.INT_LITERAL, "1");
0097: LexerTestUtilities.assertNextTokenEquals(ep,
0098: CppTokenId.WHITESPACE, " ");
0099: LexerTestUtilities.assertNextTokenEquals(ep,
0100: CppTokenId.STRING_LITERAL, "\"/*\"");
0101: LexerTestUtilities.assertNextTokenEquals(ep,
0102: CppTokenId.WHITESPACE, " ");
0103: LexerTestUtilities.assertNextTokenEquals(ep,
0104: CppTokenId.BLOCK_COMMENT, "/* \n@see C */");
0105: assertFalse("No more tokens", ep.moveNext());
0106:
0107: assertFalse("No more tokens", ts.moveNext());
0108:
0109: }
0110:
0111: public void testComments() {
0112: String text = "/*ml-comment*//**//***//**\n*doxygen-comment*//* a";
0113: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0114: .languageCpp());
0115: TokenSequence<?> ts = hi.tokenSequence();
0116: LexerTestUtilities.assertNextTokenEquals(ts,
0117: CppTokenId.BLOCK_COMMENT, "/*ml-comment*/");
0118: LexerTestUtilities.assertNextTokenEquals(ts,
0119: CppTokenId.BLOCK_COMMENT, "/**/");
0120: LexerTestUtilities.assertNextTokenEquals(ts,
0121: CppTokenId.DOXYGEN_COMMENT, "/***/");
0122: LexerTestUtilities.assertNextTokenEquals(ts,
0123: CppTokenId.DOXYGEN_COMMENT, "/**\n*doxygen-comment*/");
0124: LexerTestUtilities.assertNextTokenEquals(ts,
0125: CppTokenId.BLOCK_COMMENT, "/* a");
0126: assertEquals(PartType.START, ts.token().partType());
0127: }
0128:
0129: public void testIdentifiers() {
0130: String text = "a ab aB2 2a x\nyZ\r\nz";
0131: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0132: .languageCpp());
0133: TokenSequence<?> ts = hi.tokenSequence();
0134: LexerTestUtilities.assertNextTokenEquals(ts,
0135: CppTokenId.IDENTIFIER, "a");
0136: LexerTestUtilities.assertNextTokenEquals(ts,
0137: CppTokenId.WHITESPACE, " ");
0138: LexerTestUtilities.assertNextTokenEquals(ts,
0139: CppTokenId.IDENTIFIER, "ab");
0140: LexerTestUtilities.assertNextTokenEquals(ts,
0141: CppTokenId.WHITESPACE, " ");
0142: LexerTestUtilities.assertNextTokenEquals(ts,
0143: CppTokenId.IDENTIFIER, "aB2");
0144: LexerTestUtilities.assertNextTokenEquals(ts,
0145: CppTokenId.WHITESPACE, " ");
0146: LexerTestUtilities.assertNextTokenEquals(ts,
0147: CppTokenId.INT_LITERAL, "2");
0148: LexerTestUtilities.assertNextTokenEquals(ts,
0149: CppTokenId.IDENTIFIER, "a");
0150: LexerTestUtilities.assertNextTokenEquals(ts,
0151: CppTokenId.WHITESPACE, " ");
0152: LexerTestUtilities.assertNextTokenEquals(ts,
0153: CppTokenId.IDENTIFIER, "x");
0154: LexerTestUtilities.assertNextTokenEquals(ts,
0155: CppTokenId.NEW_LINE, "\n");
0156: LexerTestUtilities.assertNextTokenEquals(ts,
0157: CppTokenId.IDENTIFIER, "yZ");
0158: LexerTestUtilities.assertNextTokenEquals(ts,
0159: CppTokenId.NEW_LINE, "\r\n");
0160: LexerTestUtilities.assertNextTokenEquals(ts,
0161: CppTokenId.IDENTIFIER, "z");
0162: }
0163:
0164: public void testCharLiterals() {
0165: String text = "'' 'a''' '\\'' '\\\\' '\\\\\\'' '\\n' 'a";
0166: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0167: .languageCpp());
0168: TokenSequence<?> ts = hi.tokenSequence();
0169: LexerTestUtilities.assertNextTokenEquals(ts,
0170: CppTokenId.CHAR_LITERAL, "''");
0171: LexerTestUtilities.assertNextTokenEquals(ts,
0172: CppTokenId.WHITESPACE, " ");
0173: LexerTestUtilities.assertNextTokenEquals(ts,
0174: CppTokenId.CHAR_LITERAL, "'a'");
0175: LexerTestUtilities.assertNextTokenEquals(ts,
0176: CppTokenId.CHAR_LITERAL, "''");
0177: LexerTestUtilities.assertNextTokenEquals(ts,
0178: CppTokenId.WHITESPACE, " ");
0179: LexerTestUtilities.assertNextTokenEquals(ts,
0180: CppTokenId.CHAR_LITERAL, "'\\''");
0181: LexerTestUtilities.assertNextTokenEquals(ts,
0182: CppTokenId.WHITESPACE, " ");
0183: LexerTestUtilities.assertNextTokenEquals(ts,
0184: CppTokenId.CHAR_LITERAL, "'\\\\'");
0185: LexerTestUtilities.assertNextTokenEquals(ts,
0186: CppTokenId.WHITESPACE, " ");
0187: LexerTestUtilities.assertNextTokenEquals(ts,
0188: CppTokenId.CHAR_LITERAL, "'\\\\\\''");
0189: LexerTestUtilities.assertNextTokenEquals(ts,
0190: CppTokenId.WHITESPACE, " ");
0191: LexerTestUtilities.assertNextTokenEquals(ts,
0192: CppTokenId.CHAR_LITERAL, "'\\n'");
0193: LexerTestUtilities.assertNextTokenEquals(ts,
0194: CppTokenId.WHITESPACE, " ");
0195: LexerTestUtilities.assertNextTokenEquals(ts,
0196: CppTokenId.CHAR_LITERAL, "'a");
0197: assertEquals(PartType.START, ts.token().partType());
0198: }
0199:
0200: public void testStringLiterals() {
0201: String text = "\"\" \"a\"\"\" \"\\\"\" \"\\\\\" \"\\\\\\\"\" \"\\n\" \"a";
0202: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0203: .languageCpp());
0204: TokenSequence<?> ts = hi.tokenSequence();
0205: LexerTestUtilities.assertNextTokenEquals(ts,
0206: CppTokenId.STRING_LITERAL, "\"\"");
0207: LexerTestUtilities.assertNextTokenEquals(ts,
0208: CppTokenId.WHITESPACE, " ");
0209: LexerTestUtilities.assertNextTokenEquals(ts,
0210: CppTokenId.STRING_LITERAL, "\"a\"");
0211: LexerTestUtilities.assertNextTokenEquals(ts,
0212: CppTokenId.STRING_LITERAL, "\"\"");
0213: LexerTestUtilities.assertNextTokenEquals(ts,
0214: CppTokenId.WHITESPACE, " ");
0215: LexerTestUtilities.assertNextTokenEquals(ts,
0216: CppTokenId.STRING_LITERAL, "\"\\\"\"");
0217: LexerTestUtilities.assertNextTokenEquals(ts,
0218: CppTokenId.WHITESPACE, " ");
0219: LexerTestUtilities.assertNextTokenEquals(ts,
0220: CppTokenId.STRING_LITERAL, "\"\\\\\"");
0221: LexerTestUtilities.assertNextTokenEquals(ts,
0222: CppTokenId.WHITESPACE, " ");
0223: LexerTestUtilities.assertNextTokenEquals(ts,
0224: CppTokenId.STRING_LITERAL, "\"\\\\\\\"\"");
0225: LexerTestUtilities.assertNextTokenEquals(ts,
0226: CppTokenId.WHITESPACE, " ");
0227: LexerTestUtilities.assertNextTokenEquals(ts,
0228: CppTokenId.STRING_LITERAL, "\"\\n\"");
0229: LexerTestUtilities.assertNextTokenEquals(ts,
0230: CppTokenId.WHITESPACE, " ");
0231: LexerTestUtilities.assertNextTokenEquals(ts,
0232: CppTokenId.STRING_LITERAL, "\"a");
0233: assertEquals(PartType.START, ts.token().partType());
0234: }
0235:
0236: public void testNumberLiterals() {
0237: String text = "0 00 09 1 12 0L 1l 12L 0x1 0xf 0XdE 0Xbcy"
0238: + " 09.5 1.5f 1.6F 6u 7U 7e3 6.1E-7f .3";
0239: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0240: .languageCpp());
0241: TokenSequence<?> ts = hi.tokenSequence();
0242: LexerTestUtilities.assertNextTokenEquals(ts,
0243: CppTokenId.INT_LITERAL, "0");
0244: LexerTestUtilities.assertNextTokenEquals(ts,
0245: CppTokenId.WHITESPACE, " ");
0246: LexerTestUtilities.assertNextTokenEquals(ts,
0247: CppTokenId.INT_LITERAL, "00");
0248: LexerTestUtilities.assertNextTokenEquals(ts,
0249: CppTokenId.WHITESPACE, " ");
0250: LexerTestUtilities.assertNextTokenEquals(ts,
0251: CppTokenId.INT_LITERAL, "09");
0252: LexerTestUtilities.assertNextTokenEquals(ts,
0253: CppTokenId.WHITESPACE, " ");
0254: LexerTestUtilities.assertNextTokenEquals(ts,
0255: CppTokenId.INT_LITERAL, "1");
0256: LexerTestUtilities.assertNextTokenEquals(ts,
0257: CppTokenId.WHITESPACE, " ");
0258: LexerTestUtilities.assertNextTokenEquals(ts,
0259: CppTokenId.INT_LITERAL, "12");
0260: LexerTestUtilities.assertNextTokenEquals(ts,
0261: CppTokenId.WHITESPACE, " ");
0262: LexerTestUtilities.assertNextTokenEquals(ts,
0263: CppTokenId.LONG_LITERAL, "0L");
0264: LexerTestUtilities.assertNextTokenEquals(ts,
0265: CppTokenId.WHITESPACE, " ");
0266: LexerTestUtilities.assertNextTokenEquals(ts,
0267: CppTokenId.LONG_LITERAL, "1l");
0268: LexerTestUtilities.assertNextTokenEquals(ts,
0269: CppTokenId.WHITESPACE, " ");
0270: LexerTestUtilities.assertNextTokenEquals(ts,
0271: CppTokenId.LONG_LITERAL, "12L");
0272: LexerTestUtilities.assertNextTokenEquals(ts,
0273: CppTokenId.WHITESPACE, " ");
0274: LexerTestUtilities.assertNextTokenEquals(ts,
0275: CppTokenId.INT_LITERAL, "0x1");
0276: LexerTestUtilities.assertNextTokenEquals(ts,
0277: CppTokenId.WHITESPACE, " ");
0278: LexerTestUtilities.assertNextTokenEquals(ts,
0279: CppTokenId.INT_LITERAL, "0xf");
0280: LexerTestUtilities.assertNextTokenEquals(ts,
0281: CppTokenId.WHITESPACE, " ");
0282: LexerTestUtilities.assertNextTokenEquals(ts,
0283: CppTokenId.INT_LITERAL, "0XdE");
0284: LexerTestUtilities.assertNextTokenEquals(ts,
0285: CppTokenId.WHITESPACE, " ");
0286: LexerTestUtilities.assertNextTokenEquals(ts,
0287: CppTokenId.INT_LITERAL, "0Xbc");
0288: LexerTestUtilities.assertNextTokenEquals(ts,
0289: CppTokenId.IDENTIFIER, "y");
0290: LexerTestUtilities.assertNextTokenEquals(ts,
0291: CppTokenId.WHITESPACE, " ");
0292: LexerTestUtilities.assertNextTokenEquals(ts,
0293: CppTokenId.DOUBLE_LITERAL, "09.5");
0294: LexerTestUtilities.assertNextTokenEquals(ts,
0295: CppTokenId.WHITESPACE, " ");
0296: LexerTestUtilities.assertNextTokenEquals(ts,
0297: CppTokenId.FLOAT_LITERAL, "1.5f");
0298: LexerTestUtilities.assertNextTokenEquals(ts,
0299: CppTokenId.WHITESPACE, " ");
0300: LexerTestUtilities.assertNextTokenEquals(ts,
0301: CppTokenId.FLOAT_LITERAL, "1.6F");
0302: LexerTestUtilities.assertNextTokenEquals(ts,
0303: CppTokenId.WHITESPACE, " ");
0304: LexerTestUtilities.assertNextTokenEquals(ts,
0305: CppTokenId.UNSIGNED_LITERAL, "6u");
0306: LexerTestUtilities.assertNextTokenEquals(ts,
0307: CppTokenId.WHITESPACE, " ");
0308: LexerTestUtilities.assertNextTokenEquals(ts,
0309: CppTokenId.UNSIGNED_LITERAL, "7U");
0310: LexerTestUtilities.assertNextTokenEquals(ts,
0311: CppTokenId.WHITESPACE, " ");
0312: LexerTestUtilities.assertNextTokenEquals(ts,
0313: CppTokenId.DOUBLE_LITERAL, "7e3");
0314: LexerTestUtilities.assertNextTokenEquals(ts,
0315: CppTokenId.WHITESPACE, " ");
0316: LexerTestUtilities.assertNextTokenEquals(ts,
0317: CppTokenId.FLOAT_LITERAL, "6.1E-7f");
0318: LexerTestUtilities.assertNextTokenEquals(ts,
0319: CppTokenId.WHITESPACE, " ");
0320: LexerTestUtilities.assertNextTokenEquals(ts,
0321: CppTokenId.DOUBLE_LITERAL, ".3");
0322: }
0323:
0324: public void testOperators() {
0325: String text = "= > < ! ~ ? : == <= >= != && || ++ -- + - * / & | ^ % << >> += -= *= /= &= |= ^= %= <<= >>=";
0326: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0327: .languageCpp());
0328: TokenSequence<?> ts = hi.tokenSequence();
0329: LexerTestUtilities
0330: .assertNextTokenEquals(ts, CppTokenId.EQ, "=");
0331: LexerTestUtilities.assertNextTokenEquals(ts,
0332: CppTokenId.WHITESPACE, " ");
0333: LexerTestUtilities
0334: .assertNextTokenEquals(ts, CppTokenId.GT, ">");
0335: LexerTestUtilities.assertNextTokenEquals(ts,
0336: CppTokenId.WHITESPACE, " ");
0337: LexerTestUtilities
0338: .assertNextTokenEquals(ts, CppTokenId.LT, "<");
0339: LexerTestUtilities.assertNextTokenEquals(ts,
0340: CppTokenId.WHITESPACE, " ");
0341: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.NOT,
0342: "!");
0343: LexerTestUtilities.assertNextTokenEquals(ts,
0344: CppTokenId.WHITESPACE, " ");
0345: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.TILDE,
0346: "~");
0347: LexerTestUtilities.assertNextTokenEquals(ts,
0348: CppTokenId.WHITESPACE, " ");
0349: LexerTestUtilities.assertNextTokenEquals(ts,
0350: CppTokenId.QUESTION, "?");
0351: LexerTestUtilities.assertNextTokenEquals(ts,
0352: CppTokenId.WHITESPACE, " ");
0353: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.COLON,
0354: ":");
0355: LexerTestUtilities.assertNextTokenEquals(ts,
0356: CppTokenId.WHITESPACE, " ");
0357: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.EQEQ,
0358: "==");
0359: LexerTestUtilities.assertNextTokenEquals(ts,
0360: CppTokenId.WHITESPACE, " ");
0361: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LTEQ,
0362: "<=");
0363: LexerTestUtilities.assertNextTokenEquals(ts,
0364: CppTokenId.WHITESPACE, " ");
0365: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.GTEQ,
0366: ">=");
0367: LexerTestUtilities.assertNextTokenEquals(ts,
0368: CppTokenId.WHITESPACE, " ");
0369: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.NOTEQ,
0370: "!=");
0371: LexerTestUtilities.assertNextTokenEquals(ts,
0372: CppTokenId.WHITESPACE, " ");
0373: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.AMPAMP,
0374: "&&");
0375: LexerTestUtilities.assertNextTokenEquals(ts,
0376: CppTokenId.WHITESPACE, " ");
0377: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.BARBAR,
0378: "||");
0379: LexerTestUtilities.assertNextTokenEquals(ts,
0380: CppTokenId.WHITESPACE, " ");
0381: LexerTestUtilities.assertNextTokenEquals(ts,
0382: CppTokenId.PLUSPLUS, "++");
0383: LexerTestUtilities.assertNextTokenEquals(ts,
0384: CppTokenId.WHITESPACE, " ");
0385: LexerTestUtilities.assertNextTokenEquals(ts,
0386: CppTokenId.MINUSMINUS, "--");
0387: LexerTestUtilities.assertNextTokenEquals(ts,
0388: CppTokenId.WHITESPACE, " ");
0389: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.PLUS,
0390: "+");
0391: LexerTestUtilities.assertNextTokenEquals(ts,
0392: CppTokenId.WHITESPACE, " ");
0393: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.MINUS,
0394: "-");
0395: LexerTestUtilities.assertNextTokenEquals(ts,
0396: CppTokenId.WHITESPACE, " ");
0397: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STAR,
0398: "*");
0399: LexerTestUtilities.assertNextTokenEquals(ts,
0400: CppTokenId.WHITESPACE, " ");
0401: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SLASH,
0402: "/");
0403: LexerTestUtilities.assertNextTokenEquals(ts,
0404: CppTokenId.WHITESPACE, " ");
0405: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.AMP,
0406: "&");
0407: LexerTestUtilities.assertNextTokenEquals(ts,
0408: CppTokenId.WHITESPACE, " ");
0409: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.BAR,
0410: "|");
0411: LexerTestUtilities.assertNextTokenEquals(ts,
0412: CppTokenId.WHITESPACE, " ");
0413: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CARET,
0414: "^");
0415: LexerTestUtilities.assertNextTokenEquals(ts,
0416: CppTokenId.WHITESPACE, " ");
0417: LexerTestUtilities.assertNextTokenEquals(ts,
0418: CppTokenId.PERCENT, "%");
0419: LexerTestUtilities.assertNextTokenEquals(ts,
0420: CppTokenId.WHITESPACE, " ");
0421: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LTLT,
0422: "<<");
0423: LexerTestUtilities.assertNextTokenEquals(ts,
0424: CppTokenId.WHITESPACE, " ");
0425: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.GTGT,
0426: ">>");
0427: LexerTestUtilities.assertNextTokenEquals(ts,
0428: CppTokenId.WHITESPACE, " ");
0429: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.PLUSEQ,
0430: "+=");
0431: LexerTestUtilities.assertNextTokenEquals(ts,
0432: CppTokenId.WHITESPACE, " ");
0433: LexerTestUtilities.assertNextTokenEquals(ts,
0434: CppTokenId.MINUSEQ, "-=");
0435: LexerTestUtilities.assertNextTokenEquals(ts,
0436: CppTokenId.WHITESPACE, " ");
0437: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STAREQ,
0438: "*=");
0439: LexerTestUtilities.assertNextTokenEquals(ts,
0440: CppTokenId.WHITESPACE, " ");
0441: LexerTestUtilities.assertNextTokenEquals(ts,
0442: CppTokenId.SLASHEQ, "/=");
0443: LexerTestUtilities.assertNextTokenEquals(ts,
0444: CppTokenId.WHITESPACE, " ");
0445: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.AMPEQ,
0446: "&=");
0447: LexerTestUtilities.assertNextTokenEquals(ts,
0448: CppTokenId.WHITESPACE, " ");
0449: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.BAREQ,
0450: "|=");
0451: LexerTestUtilities.assertNextTokenEquals(ts,
0452: CppTokenId.WHITESPACE, " ");
0453: LexerTestUtilities.assertNextTokenEquals(ts,
0454: CppTokenId.CARETEQ, "^=");
0455: LexerTestUtilities.assertNextTokenEquals(ts,
0456: CppTokenId.WHITESPACE, " ");
0457: LexerTestUtilities.assertNextTokenEquals(ts,
0458: CppTokenId.PERCENTEQ, "%=");
0459: LexerTestUtilities.assertNextTokenEquals(ts,
0460: CppTokenId.WHITESPACE, " ");
0461: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LTLTEQ,
0462: "<<=");
0463: LexerTestUtilities.assertNextTokenEquals(ts,
0464: CppTokenId.WHITESPACE, " ");
0465: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.GTGTEQ,
0466: ">>=");
0467:
0468: assertFalse("No more tokens", ts.moveNext());
0469: }
0470:
0471: public void testLineContinuation() {
0472: String text = " #de\\\n" + "fine A\\\r"
0473: + "AA 1 // comment \\\n" + " comment-again\n"
0474: + "ch\\\n" + "ar* a = \"str\\\n" + "0\"\n";
0475: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0476: .languageCpp());
0477: TokenSequence<?> ts = hi.tokenSequence();
0478: LexerTestUtilities.assertNextTokenEquals(ts,
0479: CppTokenId.WHITESPACE, " ");
0480: assertEquals(0, ts.offset());
0481: LexerTestUtilities
0482: .assertNextTokenEquals(ts,
0483: CppTokenId.PREPROCESSOR_DIRECTIVE,
0484: "#de\\\nfine A\\\rAA 1 // comment \\\n comment-again\n");
0485: assertEquals(2, ts.offset());
0486: TokenSequence<?> ep = ts.embedded();
0487:
0488: LexerTestUtilities.assertNextTokenEquals(ep,
0489: CppTokenId.PREPROCESSOR_START, "#");
0490: assertEquals(2, ep.offset());
0491: LexerTestUtilities.assertNextTokenEquals(ep,
0492: CppTokenId.PREPROCESSOR_DEFINE, "de\\\nfine");
0493: assertEquals(3, ep.offset());
0494: LexerTestUtilities.assertNextTokenEquals(ep,
0495: CppTokenId.WHITESPACE, " ");
0496: assertEquals(11, ep.offset());
0497: LexerTestUtilities.assertNextTokenEquals(ep,
0498: CppTokenId.PREPROCESSOR_IDENTIFIER, "A\\\rAA");
0499: assertEquals(12, ep.offset());
0500: LexerTestUtilities.assertNextTokenEquals(ep,
0501: CppTokenId.WHITESPACE, " ");
0502: assertEquals(17, ep.offset());
0503: LexerTestUtilities.assertNextTokenEquals(ep,
0504: CppTokenId.INT_LITERAL, "1");
0505: assertEquals(18, ep.offset());
0506: LexerTestUtilities.assertNextTokenEquals(ep,
0507: CppTokenId.WHITESPACE, " ");
0508: assertEquals(19, ep.offset());
0509: LexerTestUtilities.assertNextTokenEquals(ep,
0510: CppTokenId.LINE_COMMENT,
0511: "// comment \\\n comment-again");
0512: assertEquals(20, ep.offset());
0513: LexerTestUtilities.assertNextTokenEquals(ep,
0514: CppTokenId.NEW_LINE, "\n");
0515: assertEquals(47, ep.offset());
0516: assertFalse("No more tokens", ep.moveNext());
0517:
0518: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CHAR,
0519: "ch\\\nar");
0520: assertEquals(48, ts.offset());
0521: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STAR,
0522: "*");
0523: assertEquals(54, ts.offset());
0524: LexerTestUtilities.assertNextTokenEquals(ts,
0525: CppTokenId.WHITESPACE, " ");
0526: assertEquals(55, ts.offset());
0527: LexerTestUtilities.assertNextTokenEquals(ts,
0528: CppTokenId.IDENTIFIER, "a");
0529: assertEquals(56, ts.offset());
0530: LexerTestUtilities.assertNextTokenEquals(ts,
0531: CppTokenId.WHITESPACE, " ");
0532: assertEquals(57, ts.offset());
0533: LexerTestUtilities
0534: .assertNextTokenEquals(ts, CppTokenId.EQ, "=");
0535: assertEquals(58, ts.offset());
0536: LexerTestUtilities.assertNextTokenEquals(ts,
0537: CppTokenId.WHITESPACE, " ");
0538: assertEquals(59, ts.offset());
0539: LexerTestUtilities.assertNextTokenEquals(ts,
0540: CppTokenId.STRING_LITERAL, "\"str\\\n0\"");
0541: assertEquals(60, ts.offset());
0542:
0543: TokenSequence<?> es = ts.embedded();
0544:
0545: LexerTestUtilities.assertNextTokenEquals(es,
0546: CppStringTokenId.TEXT, "str\\\n0");
0547: assertEquals(61, es.offset());
0548:
0549: assertFalse("No more tokens", es.moveNext());
0550:
0551: LexerTestUtilities.assertNextTokenEquals(ts,
0552: CppTokenId.NEW_LINE, "\n");
0553: assertEquals(68, ts.offset());
0554: assertFalse("No more tokens", ts.moveNext());
0555:
0556: }
0557:
0558: private String getAllKeywords() {
0559: return "asm auto bool break case catch char class const const_cast continue "
0560: + "default delete do double dynamic_cast else enum finally float for friend goto if "
0561: + "inline int long mutable namespace new operator "
0562: + "private protected public register reinterpret_cast restrict return "
0563: + "short signed sizeof static static_cast struct switch "
0564: + "template this throw try typedef typeid typename typeof "
0565: + "union unsigned using virtual void volatile wchar_t while "
0566: + "_Bool _Complex _Imaginary " + "null true false";
0567: }
0568:
0569: public void testCppKeywords() {
0570: String text = getAllKeywords();
0571: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0572: .languageCpp());
0573: TokenSequence<?> ts = hi.tokenSequence();
0574: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.ASM,
0575: "asm");
0576: LexerTestUtilities.assertNextTokenEquals(ts,
0577: CppTokenId.WHITESPACE, " ");
0578: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.AUTO,
0579: "auto");
0580: LexerTestUtilities.assertNextTokenEquals(ts,
0581: CppTokenId.WHITESPACE, " ");
0582: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.BOOL,
0583: "bool");
0584: LexerTestUtilities.assertNextTokenEquals(ts,
0585: CppTokenId.WHITESPACE, " ");
0586: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.BREAK,
0587: "break");
0588: LexerTestUtilities.assertNextTokenEquals(ts,
0589: CppTokenId.WHITESPACE, " ");
0590: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CASE,
0591: "case");
0592: LexerTestUtilities.assertNextTokenEquals(ts,
0593: CppTokenId.WHITESPACE, " ");
0594: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CATCH,
0595: "catch");
0596: LexerTestUtilities.assertNextTokenEquals(ts,
0597: CppTokenId.WHITESPACE, " ");
0598: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CHAR,
0599: "char");
0600: LexerTestUtilities.assertNextTokenEquals(ts,
0601: CppTokenId.WHITESPACE, " ");
0602: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CLASS,
0603: "class");
0604: LexerTestUtilities.assertNextTokenEquals(ts,
0605: CppTokenId.WHITESPACE, " ");
0606: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CONST,
0607: "const");
0608: LexerTestUtilities.assertNextTokenEquals(ts,
0609: CppTokenId.WHITESPACE, " ");
0610: LexerTestUtilities.assertNextTokenEquals(ts,
0611: CppTokenId.CONST_CAST, "const_cast");
0612: LexerTestUtilities.assertNextTokenEquals(ts,
0613: CppTokenId.WHITESPACE, " ");
0614: LexerTestUtilities.assertNextTokenEquals(ts,
0615: CppTokenId.CONTINUE, "continue");
0616: LexerTestUtilities.assertNextTokenEquals(ts,
0617: CppTokenId.WHITESPACE, " ");
0618: LexerTestUtilities.assertNextTokenEquals(ts,
0619: CppTokenId.DEFAULT, "default");
0620: LexerTestUtilities.assertNextTokenEquals(ts,
0621: CppTokenId.WHITESPACE, " ");
0622: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DELETE,
0623: "delete");
0624: LexerTestUtilities.assertNextTokenEquals(ts,
0625: CppTokenId.WHITESPACE, " ");
0626: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DO,
0627: "do");
0628: LexerTestUtilities.assertNextTokenEquals(ts,
0629: CppTokenId.WHITESPACE, " ");
0630: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DOUBLE,
0631: "double");
0632: LexerTestUtilities.assertNextTokenEquals(ts,
0633: CppTokenId.WHITESPACE, " ");
0634: LexerTestUtilities.assertNextTokenEquals(ts,
0635: CppTokenId.DYNAMIC_CAST, "dynamic_cast");
0636: LexerTestUtilities.assertNextTokenEquals(ts,
0637: CppTokenId.WHITESPACE, " ");
0638: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.ELSE,
0639: "else");
0640: LexerTestUtilities.assertNextTokenEquals(ts,
0641: CppTokenId.WHITESPACE, " ");
0642: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.ENUM,
0643: "enum");
0644: LexerTestUtilities.assertNextTokenEquals(ts,
0645: CppTokenId.WHITESPACE, " ");
0646: LexerTestUtilities.assertNextTokenEquals(ts,
0647: CppTokenId.FINALLY, "finally");
0648: LexerTestUtilities.assertNextTokenEquals(ts,
0649: CppTokenId.WHITESPACE, " ");
0650: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.FLOAT,
0651: "float");
0652: LexerTestUtilities.assertNextTokenEquals(ts,
0653: CppTokenId.WHITESPACE, " ");
0654: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.FOR,
0655: "for");
0656: LexerTestUtilities.assertNextTokenEquals(ts,
0657: CppTokenId.WHITESPACE, " ");
0658: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.FRIEND,
0659: "friend");
0660: LexerTestUtilities.assertNextTokenEquals(ts,
0661: CppTokenId.WHITESPACE, " ");
0662: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.GOTO,
0663: "goto");
0664: LexerTestUtilities.assertNextTokenEquals(ts,
0665: CppTokenId.WHITESPACE, " ");
0666: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.IF,
0667: "if");
0668: LexerTestUtilities.assertNextTokenEquals(ts,
0669: CppTokenId.WHITESPACE, " ");
0670: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.INLINE,
0671: "inline");
0672: LexerTestUtilities.assertNextTokenEquals(ts,
0673: CppTokenId.WHITESPACE, " ");
0674: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.INT,
0675: "int");
0676: LexerTestUtilities.assertNextTokenEquals(ts,
0677: CppTokenId.WHITESPACE, " ");
0678: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LONG,
0679: "long");
0680: LexerTestUtilities.assertNextTokenEquals(ts,
0681: CppTokenId.WHITESPACE, " ");
0682: LexerTestUtilities.assertNextTokenEquals(ts,
0683: CppTokenId.MUTABLE, "mutable");
0684: LexerTestUtilities.assertNextTokenEquals(ts,
0685: CppTokenId.WHITESPACE, " ");
0686: LexerTestUtilities.assertNextTokenEquals(ts,
0687: CppTokenId.NAMESPACE, "namespace");
0688: LexerTestUtilities.assertNextTokenEquals(ts,
0689: CppTokenId.WHITESPACE, " ");
0690: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.NEW,
0691: "new");
0692: LexerTestUtilities.assertNextTokenEquals(ts,
0693: CppTokenId.WHITESPACE, " ");
0694: LexerTestUtilities.assertNextTokenEquals(ts,
0695: CppTokenId.OPERATOR, "operator");
0696: LexerTestUtilities.assertNextTokenEquals(ts,
0697: CppTokenId.WHITESPACE, " ");
0698: LexerTestUtilities.assertNextTokenEquals(ts,
0699: CppTokenId.PRIVATE, "private");
0700: LexerTestUtilities.assertNextTokenEquals(ts,
0701: CppTokenId.WHITESPACE, " ");
0702: LexerTestUtilities.assertNextTokenEquals(ts,
0703: CppTokenId.PROTECTED, "protected");
0704: LexerTestUtilities.assertNextTokenEquals(ts,
0705: CppTokenId.WHITESPACE, " ");
0706: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.PUBLIC,
0707: "public");
0708: LexerTestUtilities.assertNextTokenEquals(ts,
0709: CppTokenId.WHITESPACE, " ");
0710: LexerTestUtilities.assertNextTokenEquals(ts,
0711: CppTokenId.REGISTER, "register");
0712: LexerTestUtilities.assertNextTokenEquals(ts,
0713: CppTokenId.WHITESPACE, " ");
0714: LexerTestUtilities.assertNextTokenEquals(ts,
0715: CppTokenId.REINTERPRET_CAST, "reinterpret_cast");
0716: LexerTestUtilities.assertNextTokenEquals(ts,
0717: CppTokenId.WHITESPACE, " ");
0718: LexerTestUtilities.assertNextTokenEquals(ts,
0719: CppTokenId.IDENTIFIER, "restrict"); // C
0720: LexerTestUtilities.assertNextTokenEquals(ts,
0721: CppTokenId.WHITESPACE, " ");
0722: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.RETURN,
0723: "return");
0724: LexerTestUtilities.assertNextTokenEquals(ts,
0725: CppTokenId.WHITESPACE, " ");
0726: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SHORT,
0727: "short");
0728: LexerTestUtilities.assertNextTokenEquals(ts,
0729: CppTokenId.WHITESPACE, " ");
0730: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SIZNED,
0731: "signed");
0732: LexerTestUtilities.assertNextTokenEquals(ts,
0733: CppTokenId.WHITESPACE, " ");
0734: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SIZEOF,
0735: "sizeof");
0736: LexerTestUtilities.assertNextTokenEquals(ts,
0737: CppTokenId.WHITESPACE, " ");
0738: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STATIC,
0739: "static");
0740: LexerTestUtilities.assertNextTokenEquals(ts,
0741: CppTokenId.WHITESPACE, " ");
0742: LexerTestUtilities.assertNextTokenEquals(ts,
0743: CppTokenId.STATIC_CAST, "static_cast");
0744: LexerTestUtilities.assertNextTokenEquals(ts,
0745: CppTokenId.WHITESPACE, " ");
0746: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STRUCT,
0747: "struct");
0748: LexerTestUtilities.assertNextTokenEquals(ts,
0749: CppTokenId.WHITESPACE, " ");
0750: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SWITCH,
0751: "switch");
0752: LexerTestUtilities.assertNextTokenEquals(ts,
0753: CppTokenId.WHITESPACE, " ");
0754: LexerTestUtilities.assertNextTokenEquals(ts,
0755: CppTokenId.TEMPLATE, "template");
0756: LexerTestUtilities.assertNextTokenEquals(ts,
0757: CppTokenId.WHITESPACE, " ");
0758: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.THIS,
0759: "this");
0760: LexerTestUtilities.assertNextTokenEquals(ts,
0761: CppTokenId.WHITESPACE, " ");
0762: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.THROW,
0763: "throw");
0764: LexerTestUtilities.assertNextTokenEquals(ts,
0765: CppTokenId.WHITESPACE, " ");
0766: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.TRY,
0767: "try");
0768: LexerTestUtilities.assertNextTokenEquals(ts,
0769: CppTokenId.WHITESPACE, " ");
0770: LexerTestUtilities.assertNextTokenEquals(ts,
0771: CppTokenId.TYPEDEF, "typedef");
0772: LexerTestUtilities.assertNextTokenEquals(ts,
0773: CppTokenId.WHITESPACE, " ");
0774: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.TYPEID,
0775: "typeid");
0776: LexerTestUtilities.assertNextTokenEquals(ts,
0777: CppTokenId.WHITESPACE, " ");
0778: LexerTestUtilities.assertNextTokenEquals(ts,
0779: CppTokenId.TYPENAME, "typename");
0780: LexerTestUtilities.assertNextTokenEquals(ts,
0781: CppTokenId.WHITESPACE, " ");
0782: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.TYPEOF,
0783: "typeof");
0784: LexerTestUtilities.assertNextTokenEquals(ts,
0785: CppTokenId.WHITESPACE, " ");
0786: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.UNION,
0787: "union");
0788: LexerTestUtilities.assertNextTokenEquals(ts,
0789: CppTokenId.WHITESPACE, " ");
0790: LexerTestUtilities.assertNextTokenEquals(ts,
0791: CppTokenId.UNSIGNED, "unsigned");
0792: LexerTestUtilities.assertNextTokenEquals(ts,
0793: CppTokenId.WHITESPACE, " ");
0794: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.USING,
0795: "using");
0796: LexerTestUtilities.assertNextTokenEquals(ts,
0797: CppTokenId.WHITESPACE, " ");
0798: LexerTestUtilities.assertNextTokenEquals(ts,
0799: CppTokenId.VIRTUAL, "virtual");
0800: LexerTestUtilities.assertNextTokenEquals(ts,
0801: CppTokenId.WHITESPACE, " ");
0802: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.VOID,
0803: "void");
0804: LexerTestUtilities.assertNextTokenEquals(ts,
0805: CppTokenId.WHITESPACE, " ");
0806: LexerTestUtilities.assertNextTokenEquals(ts,
0807: CppTokenId.VOLATILE, "volatile");
0808: LexerTestUtilities.assertNextTokenEquals(ts,
0809: CppTokenId.WHITESPACE, " ");
0810: LexerTestUtilities.assertNextTokenEquals(ts,
0811: CppTokenId.WCHAR_T, "wchar_t");
0812: LexerTestUtilities.assertNextTokenEquals(ts,
0813: CppTokenId.WHITESPACE, " ");
0814: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.WHILE,
0815: "while");
0816: LexerTestUtilities.assertNextTokenEquals(ts,
0817: CppTokenId.WHITESPACE, " ");
0818:
0819: LexerTestUtilities.assertNextTokenEquals(ts,
0820: CppTokenId.IDENTIFIER, "_Bool"); //C
0821: LexerTestUtilities.assertNextTokenEquals(ts,
0822: CppTokenId.WHITESPACE, " ");
0823: LexerTestUtilities.assertNextTokenEquals(ts,
0824: CppTokenId.IDENTIFIER, "_Complex"); //C
0825: LexerTestUtilities.assertNextTokenEquals(ts,
0826: CppTokenId.WHITESPACE, " ");
0827: LexerTestUtilities.assertNextTokenEquals(ts,
0828: CppTokenId.IDENTIFIER, "_Imaginary"); //C
0829: LexerTestUtilities.assertNextTokenEquals(ts,
0830: CppTokenId.WHITESPACE, " ");
0831:
0832: LexerTestUtilities.assertNextTokenEquals(ts,
0833: CppTokenId.IDENTIFIER, "null");
0834: LexerTestUtilities.assertNextTokenEquals(ts,
0835: CppTokenId.WHITESPACE, " ");
0836: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.TRUE,
0837: "true");
0838: LexerTestUtilities.assertNextTokenEquals(ts,
0839: CppTokenId.WHITESPACE, " ");
0840: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.FALSE,
0841: "false");
0842: }
0843:
0844: public void testCKeywords() {
0845: String text = getAllKeywords();
0846: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
0847: .languageC());
0848: TokenSequence<?> ts = hi.tokenSequence();
0849: LexerTestUtilities.assertNextTokenEquals(ts,
0850: CppTokenId.IDENTIFIER, "asm"); // C++
0851: LexerTestUtilities.assertNextTokenEquals(ts,
0852: CppTokenId.WHITESPACE, " ");
0853: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.AUTO,
0854: "auto");
0855: LexerTestUtilities.assertNextTokenEquals(ts,
0856: CppTokenId.WHITESPACE, " ");
0857: LexerTestUtilities.assertNextTokenEquals(ts,
0858: CppTokenId.IDENTIFIER, "bool"); //C++
0859: LexerTestUtilities.assertNextTokenEquals(ts,
0860: CppTokenId.WHITESPACE, " ");
0861: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.BREAK,
0862: "break");
0863: LexerTestUtilities.assertNextTokenEquals(ts,
0864: CppTokenId.WHITESPACE, " ");
0865: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CASE,
0866: "case");
0867: LexerTestUtilities.assertNextTokenEquals(ts,
0868: CppTokenId.WHITESPACE, " ");
0869: LexerTestUtilities.assertNextTokenEquals(ts,
0870: CppTokenId.IDENTIFIER, "catch"); // C++
0871: LexerTestUtilities.assertNextTokenEquals(ts,
0872: CppTokenId.WHITESPACE, " ");
0873: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CHAR,
0874: "char");
0875: LexerTestUtilities.assertNextTokenEquals(ts,
0876: CppTokenId.WHITESPACE, " ");
0877: LexerTestUtilities.assertNextTokenEquals(ts,
0878: CppTokenId.IDENTIFIER, "class"); // C++
0879: LexerTestUtilities.assertNextTokenEquals(ts,
0880: CppTokenId.WHITESPACE, " ");
0881: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CONST,
0882: "const");
0883: LexerTestUtilities.assertNextTokenEquals(ts,
0884: CppTokenId.WHITESPACE, " ");
0885: LexerTestUtilities.assertNextTokenEquals(ts,
0886: CppTokenId.IDENTIFIER, "const_cast"); // C++
0887: LexerTestUtilities.assertNextTokenEquals(ts,
0888: CppTokenId.WHITESPACE, " ");
0889: LexerTestUtilities.assertNextTokenEquals(ts,
0890: CppTokenId.CONTINUE, "continue");
0891: LexerTestUtilities.assertNextTokenEquals(ts,
0892: CppTokenId.WHITESPACE, " ");
0893: LexerTestUtilities.assertNextTokenEquals(ts,
0894: CppTokenId.DEFAULT, "default");
0895: LexerTestUtilities.assertNextTokenEquals(ts,
0896: CppTokenId.WHITESPACE, " ");
0897: LexerTestUtilities.assertNextTokenEquals(ts,
0898: CppTokenId.IDENTIFIER, "delete"); // C++
0899: LexerTestUtilities.assertNextTokenEquals(ts,
0900: CppTokenId.WHITESPACE, " ");
0901: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DO,
0902: "do");
0903: LexerTestUtilities.assertNextTokenEquals(ts,
0904: CppTokenId.WHITESPACE, " ");
0905: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DOUBLE,
0906: "double");
0907: LexerTestUtilities.assertNextTokenEquals(ts,
0908: CppTokenId.WHITESPACE, " ");
0909: LexerTestUtilities.assertNextTokenEquals(ts,
0910: CppTokenId.IDENTIFIER, "dynamic_cast"); // C++
0911: LexerTestUtilities.assertNextTokenEquals(ts,
0912: CppTokenId.WHITESPACE, " ");
0913: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.ELSE,
0914: "else");
0915: LexerTestUtilities.assertNextTokenEquals(ts,
0916: CppTokenId.WHITESPACE, " ");
0917: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.ENUM,
0918: "enum");
0919: LexerTestUtilities.assertNextTokenEquals(ts,
0920: CppTokenId.WHITESPACE, " ");
0921: LexerTestUtilities.assertNextTokenEquals(ts,
0922: CppTokenId.IDENTIFIER, "finally"); // C++
0923: LexerTestUtilities.assertNextTokenEquals(ts,
0924: CppTokenId.WHITESPACE, " ");
0925: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.FLOAT,
0926: "float");
0927: LexerTestUtilities.assertNextTokenEquals(ts,
0928: CppTokenId.WHITESPACE, " ");
0929: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.FOR,
0930: "for");
0931: LexerTestUtilities.assertNextTokenEquals(ts,
0932: CppTokenId.WHITESPACE, " ");
0933: LexerTestUtilities.assertNextTokenEquals(ts,
0934: CppTokenId.IDENTIFIER, "friend"); // C++
0935: LexerTestUtilities.assertNextTokenEquals(ts,
0936: CppTokenId.WHITESPACE, " ");
0937: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.GOTO,
0938: "goto");
0939: LexerTestUtilities.assertNextTokenEquals(ts,
0940: CppTokenId.WHITESPACE, " ");
0941: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.IF,
0942: "if");
0943: LexerTestUtilities.assertNextTokenEquals(ts,
0944: CppTokenId.WHITESPACE, " ");
0945: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.INLINE,
0946: "inline");
0947: LexerTestUtilities.assertNextTokenEquals(ts,
0948: CppTokenId.WHITESPACE, " ");
0949: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.INT,
0950: "int");
0951: LexerTestUtilities.assertNextTokenEquals(ts,
0952: CppTokenId.WHITESPACE, " ");
0953: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LONG,
0954: "long");
0955: LexerTestUtilities.assertNextTokenEquals(ts,
0956: CppTokenId.WHITESPACE, " ");
0957: LexerTestUtilities.assertNextTokenEquals(ts,
0958: CppTokenId.IDENTIFIER, "mutable"); // C++
0959: LexerTestUtilities.assertNextTokenEquals(ts,
0960: CppTokenId.WHITESPACE, " ");
0961: LexerTestUtilities.assertNextTokenEquals(ts,
0962: CppTokenId.IDENTIFIER, "namespace"); // C++
0963: LexerTestUtilities.assertNextTokenEquals(ts,
0964: CppTokenId.WHITESPACE, " ");
0965: LexerTestUtilities.assertNextTokenEquals(ts,
0966: CppTokenId.IDENTIFIER, "new"); // C++
0967: LexerTestUtilities.assertNextTokenEquals(ts,
0968: CppTokenId.WHITESPACE, " ");
0969: LexerTestUtilities.assertNextTokenEquals(ts,
0970: CppTokenId.IDENTIFIER, "operator"); // C++
0971: LexerTestUtilities.assertNextTokenEquals(ts,
0972: CppTokenId.WHITESPACE, " ");
0973: LexerTestUtilities.assertNextTokenEquals(ts,
0974: CppTokenId.IDENTIFIER, "private"); // C++
0975: LexerTestUtilities.assertNextTokenEquals(ts,
0976: CppTokenId.WHITESPACE, " ");
0977: LexerTestUtilities.assertNextTokenEquals(ts,
0978: CppTokenId.IDENTIFIER, "protected"); // C++
0979: LexerTestUtilities.assertNextTokenEquals(ts,
0980: CppTokenId.WHITESPACE, " ");
0981: LexerTestUtilities.assertNextTokenEquals(ts,
0982: CppTokenId.IDENTIFIER, "public"); // C++
0983: LexerTestUtilities.assertNextTokenEquals(ts,
0984: CppTokenId.WHITESPACE, " ");
0985: LexerTestUtilities.assertNextTokenEquals(ts,
0986: CppTokenId.REGISTER, "register");
0987: LexerTestUtilities.assertNextTokenEquals(ts,
0988: CppTokenId.WHITESPACE, " ");
0989: LexerTestUtilities.assertNextTokenEquals(ts,
0990: CppTokenId.IDENTIFIER, "reinterpret_cast"); // C++
0991: LexerTestUtilities.assertNextTokenEquals(ts,
0992: CppTokenId.WHITESPACE, " ");
0993: LexerTestUtilities.assertNextTokenEquals(ts,
0994: CppTokenId.RESTRICT, "restrict"); // C
0995: LexerTestUtilities.assertNextTokenEquals(ts,
0996: CppTokenId.WHITESPACE, " ");
0997: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.RETURN,
0998: "return");
0999: LexerTestUtilities.assertNextTokenEquals(ts,
1000: CppTokenId.WHITESPACE, " ");
1001: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SHORT,
1002: "short");
1003: LexerTestUtilities.assertNextTokenEquals(ts,
1004: CppTokenId.WHITESPACE, " ");
1005: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SIZNED,
1006: "signed");
1007: LexerTestUtilities.assertNextTokenEquals(ts,
1008: CppTokenId.WHITESPACE, " ");
1009: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SIZEOF,
1010: "sizeof");
1011: LexerTestUtilities.assertNextTokenEquals(ts,
1012: CppTokenId.WHITESPACE, " ");
1013: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STATIC,
1014: "static");
1015: LexerTestUtilities.assertNextTokenEquals(ts,
1016: CppTokenId.WHITESPACE, " ");
1017: LexerTestUtilities.assertNextTokenEquals(ts,
1018: CppTokenId.IDENTIFIER, "static_cast"); // C++
1019: LexerTestUtilities.assertNextTokenEquals(ts,
1020: CppTokenId.WHITESPACE, " ");
1021: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STRUCT,
1022: "struct");
1023: LexerTestUtilities.assertNextTokenEquals(ts,
1024: CppTokenId.WHITESPACE, " ");
1025: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SWITCH,
1026: "switch");
1027: LexerTestUtilities.assertNextTokenEquals(ts,
1028: CppTokenId.WHITESPACE, " ");
1029: LexerTestUtilities.assertNextTokenEquals(ts,
1030: CppTokenId.IDENTIFIER, "template"); // C++
1031: LexerTestUtilities.assertNextTokenEquals(ts,
1032: CppTokenId.WHITESPACE, " ");
1033: LexerTestUtilities.assertNextTokenEquals(ts,
1034: CppTokenId.IDENTIFIER, "this"); // C++
1035: LexerTestUtilities.assertNextTokenEquals(ts,
1036: CppTokenId.WHITESPACE, " ");
1037: LexerTestUtilities.assertNextTokenEquals(ts,
1038: CppTokenId.IDENTIFIER, "throw"); // C++
1039: LexerTestUtilities.assertNextTokenEquals(ts,
1040: CppTokenId.WHITESPACE, " ");
1041: LexerTestUtilities.assertNextTokenEquals(ts,
1042: CppTokenId.IDENTIFIER, "try"); // C++
1043: LexerTestUtilities.assertNextTokenEquals(ts,
1044: CppTokenId.WHITESPACE, " ");
1045: LexerTestUtilities.assertNextTokenEquals(ts,
1046: CppTokenId.TYPEDEF, "typedef");
1047: LexerTestUtilities.assertNextTokenEquals(ts,
1048: CppTokenId.WHITESPACE, " ");
1049: LexerTestUtilities.assertNextTokenEquals(ts,
1050: CppTokenId.IDENTIFIER, "typeid"); // C++
1051: LexerTestUtilities.assertNextTokenEquals(ts,
1052: CppTokenId.WHITESPACE, " ");
1053: LexerTestUtilities.assertNextTokenEquals(ts,
1054: CppTokenId.IDENTIFIER, "typename"); // C++
1055: LexerTestUtilities.assertNextTokenEquals(ts,
1056: CppTokenId.WHITESPACE, " ");
1057: LexerTestUtilities.assertNextTokenEquals(ts,
1058: CppTokenId.IDENTIFIER, "typeof"); // C++
1059: LexerTestUtilities.assertNextTokenEquals(ts,
1060: CppTokenId.WHITESPACE, " ");
1061: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.UNION,
1062: "union");
1063: LexerTestUtilities.assertNextTokenEquals(ts,
1064: CppTokenId.WHITESPACE, " ");
1065: LexerTestUtilities.assertNextTokenEquals(ts,
1066: CppTokenId.UNSIGNED, "unsigned");
1067: LexerTestUtilities.assertNextTokenEquals(ts,
1068: CppTokenId.WHITESPACE, " ");
1069: LexerTestUtilities.assertNextTokenEquals(ts,
1070: CppTokenId.IDENTIFIER, "using"); // C++
1071: LexerTestUtilities.assertNextTokenEquals(ts,
1072: CppTokenId.WHITESPACE, " ");
1073: LexerTestUtilities.assertNextTokenEquals(ts,
1074: CppTokenId.IDENTIFIER, "virtual"); // C++
1075: LexerTestUtilities.assertNextTokenEquals(ts,
1076: CppTokenId.WHITESPACE, " ");
1077: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.VOID,
1078: "void");
1079: LexerTestUtilities.assertNextTokenEquals(ts,
1080: CppTokenId.WHITESPACE, " ");
1081: LexerTestUtilities.assertNextTokenEquals(ts,
1082: CppTokenId.VOLATILE, "volatile");
1083: LexerTestUtilities.assertNextTokenEquals(ts,
1084: CppTokenId.WHITESPACE, " ");
1085: LexerTestUtilities.assertNextTokenEquals(ts,
1086: CppTokenId.IDENTIFIER, "wchar_t"); // C++
1087: LexerTestUtilities.assertNextTokenEquals(ts,
1088: CppTokenId.WHITESPACE, " ");
1089: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.WHILE,
1090: "while");
1091: LexerTestUtilities.assertNextTokenEquals(ts,
1092: CppTokenId.WHITESPACE, " ");
1093:
1094: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId._BOOL,
1095: "_Bool"); //C
1096: LexerTestUtilities.assertNextTokenEquals(ts,
1097: CppTokenId.WHITESPACE, " ");
1098: LexerTestUtilities.assertNextTokenEquals(ts,
1099: CppTokenId._COMPLEX, "_Complex"); //C
1100: LexerTestUtilities.assertNextTokenEquals(ts,
1101: CppTokenId.WHITESPACE, " ");
1102: LexerTestUtilities.assertNextTokenEquals(ts,
1103: CppTokenId._IMAGINARY, "_Imaginary"); //C
1104: LexerTestUtilities.assertNextTokenEquals(ts,
1105: CppTokenId.WHITESPACE, " ");
1106:
1107: LexerTestUtilities.assertNextTokenEquals(ts,
1108: CppTokenId.IDENTIFIER, "null");
1109: LexerTestUtilities.assertNextTokenEquals(ts,
1110: CppTokenId.WHITESPACE, " ");
1111: LexerTestUtilities.assertNextTokenEquals(ts,
1112: CppTokenId.IDENTIFIER, "true"); // C++
1113: LexerTestUtilities.assertNextTokenEquals(ts,
1114: CppTokenId.WHITESPACE, " ");
1115: LexerTestUtilities.assertNextTokenEquals(ts,
1116: CppTokenId.IDENTIFIER, "false"); // C++
1117: }
1118:
1119: public void testNonKeywords() {
1120: String text = "asma autos b br car dou doubl finall im i ifa inti throwx ";
1121:
1122: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
1123: .languageCpp());
1124: TokenSequence<?> ts = hi.tokenSequence();
1125: LexerTestUtilities.assertNextTokenEquals(ts,
1126: CppTokenId.IDENTIFIER, "asma");
1127: LexerTestUtilities.assertNextTokenEquals(ts,
1128: CppTokenId.WHITESPACE, " ");
1129: LexerTestUtilities.assertNextTokenEquals(ts,
1130: CppTokenId.IDENTIFIER, "autos");
1131: LexerTestUtilities.assertNextTokenEquals(ts,
1132: CppTokenId.WHITESPACE, " ");
1133: LexerTestUtilities.assertNextTokenEquals(ts,
1134: CppTokenId.IDENTIFIER, "b");
1135: LexerTestUtilities.assertNextTokenEquals(ts,
1136: CppTokenId.WHITESPACE, " ");
1137: LexerTestUtilities.assertNextTokenEquals(ts,
1138: CppTokenId.IDENTIFIER, "br");
1139: LexerTestUtilities.assertNextTokenEquals(ts,
1140: CppTokenId.WHITESPACE, " ");
1141: LexerTestUtilities.assertNextTokenEquals(ts,
1142: CppTokenId.IDENTIFIER, "car");
1143: LexerTestUtilities.assertNextTokenEquals(ts,
1144: CppTokenId.WHITESPACE, " ");
1145: LexerTestUtilities.assertNextTokenEquals(ts,
1146: CppTokenId.IDENTIFIER, "dou");
1147: LexerTestUtilities.assertNextTokenEquals(ts,
1148: CppTokenId.WHITESPACE, " ");
1149: LexerTestUtilities.assertNextTokenEquals(ts,
1150: CppTokenId.IDENTIFIER, "doubl");
1151: LexerTestUtilities.assertNextTokenEquals(ts,
1152: CppTokenId.WHITESPACE, " ");
1153: LexerTestUtilities.assertNextTokenEquals(ts,
1154: CppTokenId.IDENTIFIER, "finall");
1155: LexerTestUtilities.assertNextTokenEquals(ts,
1156: CppTokenId.WHITESPACE, " ");
1157: LexerTestUtilities.assertNextTokenEquals(ts,
1158: CppTokenId.IDENTIFIER, "im");
1159: LexerTestUtilities.assertNextTokenEquals(ts,
1160: CppTokenId.WHITESPACE, " ");
1161: LexerTestUtilities.assertNextTokenEquals(ts,
1162: CppTokenId.IDENTIFIER, "i");
1163: LexerTestUtilities.assertNextTokenEquals(ts,
1164: CppTokenId.WHITESPACE, " ");
1165: LexerTestUtilities.assertNextTokenEquals(ts,
1166: CppTokenId.IDENTIFIER, "ifa");
1167: LexerTestUtilities.assertNextTokenEquals(ts,
1168: CppTokenId.WHITESPACE, " ");
1169: LexerTestUtilities.assertNextTokenEquals(ts,
1170: CppTokenId.IDENTIFIER, "inti");
1171: LexerTestUtilities.assertNextTokenEquals(ts,
1172: CppTokenId.WHITESPACE, " ");
1173: LexerTestUtilities.assertNextTokenEquals(ts,
1174: CppTokenId.IDENTIFIER, "throwx");
1175: }
1176:
1177: public void testEmbedding() {
1178: String text = "ddx \"d\\t\\br\" /** @see X */";
1179:
1180: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
1181: .languageCpp());
1182: TokenSequence<?> ts = hi.tokenSequence();
1183:
1184: LexerTestUtilities.assertNextTokenEquals(ts,
1185: CppTokenId.IDENTIFIER, "ddx");
1186: assertEquals(0, ts.offset());
1187: LexerTestUtilities.assertNextTokenEquals(ts,
1188: CppTokenId.WHITESPACE, " ");
1189: assertEquals(3, ts.offset());
1190: LexerTestUtilities.assertNextTokenEquals(ts,
1191: CppTokenId.STRING_LITERAL, "\"d\\t\\br\"");
1192: assertEquals(4, ts.offset());
1193:
1194: TokenSequence<?> es = ts.embedded();
1195:
1196: LexerTestUtilities.assertNextTokenEquals(es,
1197: CppStringTokenId.TEXT, "d");
1198: assertEquals(5, es.offset());
1199: LexerTestUtilities.assertNextTokenEquals(es,
1200: CppStringTokenId.TAB, "\\t");
1201: assertEquals(6, es.offset());
1202: LexerTestUtilities.assertNextTokenEquals(es,
1203: CppStringTokenId.BACKSPACE, "\\b");
1204: assertEquals(8, es.offset());
1205: LexerTestUtilities.assertNextTokenEquals(es,
1206: CppStringTokenId.TEXT, "r");
1207: assertEquals(10, es.offset());
1208:
1209: assertFalse(es.moveNext());
1210:
1211: LexerTestUtilities.assertNextTokenEquals(ts,
1212: CppTokenId.WHITESPACE, " ");
1213: assertEquals(12, ts.offset());
1214: LexerTestUtilities.assertNextTokenEquals(ts,
1215: CppTokenId.DOXYGEN_COMMENT, "/** @see X */");
1216: assertEquals(13, ts.offset());
1217:
1218: TokenSequence<?> ds = ts.embedded();
1219:
1220: LexerTestUtilities.assertNextTokenEquals(ds,
1221: DoxygenTokenId.OTHER_TEXT, " ");
1222: assertEquals(16, ds.offset());
1223: LexerTestUtilities.assertNextTokenEquals(ds,
1224: DoxygenTokenId.TAG, "@see");
1225: assertEquals(17, ds.offset());
1226: LexerTestUtilities.assertNextTokenEquals(ds,
1227: DoxygenTokenId.OTHER_TEXT, " ");
1228: assertEquals(21, ds.offset());
1229: LexerTestUtilities.assertNextTokenEquals(ds,
1230: DoxygenTokenId.IDENT, "X");
1231: assertEquals(22, ds.offset());
1232: LexerTestUtilities.assertNextTokenEquals(ds,
1233: DoxygenTokenId.OTHER_TEXT, " ");
1234: assertEquals(23, ds.offset());
1235:
1236: assertFalse(ds.moveNext());
1237:
1238: assertFalse(ts.moveNext());
1239: }
1240:
1241: public void testSpecial() {
1242: String text = "\\ ... $ @";
1243: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
1244: .languageCpp());
1245: TokenSequence<?> ts = hi.tokenSequence();
1246: LexerTestUtilities.assertNextTokenEquals(ts,
1247: CppTokenId.BACK_SLASH, "\\");
1248: LexerTestUtilities.assertNextTokenEquals(ts,
1249: CppTokenId.WHITESPACE, " ");
1250: LexerTestUtilities.assertNextTokenEquals(ts,
1251: CppTokenId.ELLIPSIS, "...");
1252: LexerTestUtilities.assertNextTokenEquals(ts,
1253: CppTokenId.WHITESPACE, " ");
1254: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DOLLAR,
1255: "$");
1256: LexerTestUtilities.assertNextTokenEquals(ts,
1257: CppTokenId.WHITESPACE, " ");
1258: LexerTestUtilities
1259: .assertNextTokenEquals(ts, CppTokenId.AT, "@");
1260:
1261: assertFalse("No more tokens", ts.moveNext());
1262: }
1263:
1264: public void testSeparators() {
1265: String text = "( ) { } [ ] ; , . .* :: -> ->*";
1266: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
1267: .languageCpp());
1268: TokenSequence<?> ts = hi.tokenSequence();
1269: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LPAREN,
1270: "(");
1271: LexerTestUtilities.assertNextTokenEquals(ts,
1272: CppTokenId.WHITESPACE, " ");
1273: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.RPAREN,
1274: ")");
1275: LexerTestUtilities.assertNextTokenEquals(ts,
1276: CppTokenId.WHITESPACE, " ");
1277: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.LBRACE,
1278: "{");
1279: LexerTestUtilities.assertNextTokenEquals(ts,
1280: CppTokenId.WHITESPACE, " ");
1281: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.RBRACE,
1282: "}");
1283: LexerTestUtilities.assertNextTokenEquals(ts,
1284: CppTokenId.WHITESPACE, " ");
1285: LexerTestUtilities.assertNextTokenEquals(ts,
1286: CppTokenId.LBRACKET, "[");
1287: LexerTestUtilities.assertNextTokenEquals(ts,
1288: CppTokenId.WHITESPACE, " ");
1289: LexerTestUtilities.assertNextTokenEquals(ts,
1290: CppTokenId.RBRACKET, "]");
1291: LexerTestUtilities.assertNextTokenEquals(ts,
1292: CppTokenId.WHITESPACE, " ");
1293: LexerTestUtilities.assertNextTokenEquals(ts,
1294: CppTokenId.SEMICOLON, ";");
1295: LexerTestUtilities.assertNextTokenEquals(ts,
1296: CppTokenId.WHITESPACE, " ");
1297: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.COMMA,
1298: ",");
1299: LexerTestUtilities.assertNextTokenEquals(ts,
1300: CppTokenId.WHITESPACE, " ");
1301: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DOT,
1302: ".");
1303: LexerTestUtilities.assertNextTokenEquals(ts,
1304: CppTokenId.WHITESPACE, " ");
1305: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.DOTMBR,
1306: ".*");
1307: LexerTestUtilities.assertNextTokenEquals(ts,
1308: CppTokenId.WHITESPACE, " ");
1309: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.SCOPE,
1310: "::");
1311: LexerTestUtilities.assertNextTokenEquals(ts,
1312: CppTokenId.WHITESPACE, " ");
1313: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.ARROW,
1314: "->");
1315: LexerTestUtilities.assertNextTokenEquals(ts,
1316: CppTokenId.WHITESPACE, " ");
1317: LexerTestUtilities.assertNextTokenEquals(ts,
1318: CppTokenId.ARROWMBR, "->*");
1319:
1320: assertFalse("No more tokens", ts.moveNext());
1321: }
1322:
1323: public void testComment2() {
1324: // IZ#83566: "*/" string is highlighted as error
1325: String text = "const/* */int*/* */i = 0; */";
1326: TokenHierarchy<?> hi = TokenHierarchy.create(text, CppTokenId
1327: .languageCpp());
1328: TokenSequence<?> ts = hi.tokenSequence();
1329:
1330: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.CONST,
1331: "const");
1332: LexerTestUtilities.assertNextTokenEquals(ts,
1333: CppTokenId.BLOCK_COMMENT, "/* */");
1334: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.INT,
1335: "int");
1336: LexerTestUtilities.assertNextTokenEquals(ts, CppTokenId.STAR,
1337: "*");
1338: LexerTestUtilities.assertNextTokenEquals(ts,
1339: CppTokenId.BLOCK_COMMENT, "/* */");
1340: LexerTestUtilities.assertNextTokenEquals(ts,
1341: CppTokenId.IDENTIFIER, "i");
1342: LexerTestUtilities.assertNextTokenEquals(ts,
1343: CppTokenId.WHITESPACE, " ");
1344: LexerTestUtilities
1345: .assertNextTokenEquals(ts, CppTokenId.EQ, "=");
1346: LexerTestUtilities.assertNextTokenEquals(ts,
1347: CppTokenId.WHITESPACE, " ");
1348: LexerTestUtilities.assertNextTokenEquals(ts,
1349: CppTokenId.INT_LITERAL, "0");
1350: LexerTestUtilities.assertNextTokenEquals(ts,
1351: CppTokenId.SEMICOLON, ";");
1352: LexerTestUtilities.assertNextTokenEquals(ts,
1353: CppTokenId.WHITESPACE, " ");
1354: LexerTestUtilities.assertNextTokenEquals(ts,
1355: CppTokenId.INVALID_COMMENT_END, "*/");
1356:
1357: assertFalse("No more tokens", ts.moveNext());
1358:
1359: }
1360: }
|