fixing the lexer, added lexer unit test
authorMarek Fukala <mfukala@netbeans.org>
Sat, 12 Jan 2013 13:04:21 +0100
changeset 1792025a8acfe62f4
parent 17919 6ac43eb330ca
child 17922 16184ed8f1e0
child 17923 51e93e0cfb7a
fixing the lexer, added lexer unit test
o.n.antlr.editor/nbproject/project.xml
o.n.antlr.editor/src/org/netbeans/modules/antlr/editor/AntlrTokenId.java
o.n.antlr.editor/test/unit/src/org/netbeans/modules/antlr/editor/AntlrTestBase.java
o.n.antlr.editor/test/unit/src/org/netbeans/modules/antlr/editor/NbAntlrLexerTest.java
     1.1 --- a/o.n.antlr.editor/nbproject/project.xml	Sat Jan 12 08:46:08 2013 +0100
     1.2 +++ b/o.n.antlr.editor/nbproject/project.xml	Sat Jan 12 13:04:21 2013 +0100
     1.3 @@ -126,9 +126,29 @@
     1.4                  <test-type>
     1.5                      <name>unit</name>
     1.6                      <test-dependency>
     1.7 +                        <code-name-base>org.netbeans.insane</code-name-base>
     1.8 +                        <compile-dependency/>
     1.9 +                    </test-dependency>
    1.10 +                    <test-dependency>
    1.11                          <code-name-base>org.netbeans.libs.junit4</code-name-base>
    1.12                          <compile-dependency/>
    1.13                      </test-dependency>
    1.14 +                    <test-dependency>
    1.15 +                        <code-name-base>org.netbeans.modules.csl.api</code-name-base>
    1.16 +                        <recursive/>
    1.17 +                        <compile-dependency/>
    1.18 +                        <test/>
    1.19 +                    </test-dependency>
    1.20 +                    <test-dependency>
    1.21 +                        <code-name-base>org.netbeans.modules.nbjunit</code-name-base>
    1.22 +                        <compile-dependency/>
    1.23 +                        <test/>
    1.24 +                    </test-dependency>
    1.25 +                    <test-dependency>
    1.26 +                        <code-name-base>org.openide.util.lookup</code-name-base>
    1.27 +                        <compile-dependency/>
    1.28 +                        <test/>
    1.29 +                    </test-dependency>
    1.30                  </test-type>
    1.31              </test-dependencies>
    1.32              <public-packages/>
     2.1 --- a/o.n.antlr.editor/src/org/netbeans/modules/antlr/editor/AntlrTokenId.java	Sat Jan 12 08:46:08 2013 +0100
     2.2 +++ b/o.n.antlr.editor/src/org/netbeans/modules/antlr/editor/AntlrTokenId.java	Sat Jan 12 13:04:21 2013 +0100
     2.3 @@ -120,8 +120,38 @@
     2.4      NESTED_ACTION(ANTLRv3Lexer.NESTED_ACTION, OTHERS),
     2.5      ACTION_ESC(ANTLRv3Lexer.ACTION_ESC, OTHERS),
     2.6      WS_LOOP(ANTLRv3Lexer.WS_LOOP, OTHERS),
     2.7 -    WS(ANTLRv3Lexer.WS, OTHERS);
     2.8 +    WS(ANTLRv3Lexer.WS, OTHERS),
     2.9      
    2.10 +    
    2.11 +    T__65(ANTLRv3Lexer.T__65, OTHERS),
    2.12 +    T__66(ANTLRv3Lexer.T__66, OTHERS),
    2.13 +    T__67(ANTLRv3Lexer.T__67, OTHERS),
    2.14 +    T__68(ANTLRv3Lexer.T__68, OTHERS),
    2.15 +    T__69(ANTLRv3Lexer.T__69, OTHERS),
    2.16 +    T__70(ANTLRv3Lexer.T__70, OTHERS),
    2.17 +    T__71(ANTLRv3Lexer.T__71, OTHERS),
    2.18 +    T__72(ANTLRv3Lexer.T__72, OTHERS),
    2.19 +    T__73(ANTLRv3Lexer.T__73, OTHERS),
    2.20 +    T__74(ANTLRv3Lexer.T__74, OTHERS),
    2.21 +    T__75(ANTLRv3Lexer.T__75, OTHERS),
    2.22 +    T__76(ANTLRv3Lexer.T__76, OTHERS),
    2.23 +    T__77(ANTLRv3Lexer.T__77, OTHERS),
    2.24 +    T__78(ANTLRv3Lexer.T__78, OTHERS),
    2.25 +    T__79(ANTLRv3Lexer.T__79, OTHERS),
    2.26 +    T__80(ANTLRv3Lexer.T__80, OTHERS),
    2.27 +    T__81(ANTLRv3Lexer.T__81, OTHERS),
    2.28 +    T__82(ANTLRv3Lexer.T__82, OTHERS),
    2.29 +    T__83(ANTLRv3Lexer.T__83, OTHERS),
    2.30 +    T__84(ANTLRv3Lexer.T__84, OTHERS),
    2.31 +    T__85(ANTLRv3Lexer.T__85, OTHERS),
    2.32 +    T__86(ANTLRv3Lexer.T__86, OTHERS),
    2.33 +    T__87(ANTLRv3Lexer.T__87, OTHERS),
    2.34 +    T__88(ANTLRv3Lexer.T__88, OTHERS),
    2.35 +    T__89(ANTLRv3Lexer.T__89, OTHERS),
    2.36 +    T__90(ANTLRv3Lexer.T__90, OTHERS),
    2.37 +    T__91(ANTLRv3Lexer.T__91, OTHERS),
    2.38 +    T__92(ANTLRv3Lexer.T__92, OTHERS),
    2.39 +    T__93(ANTLRv3Lexer.T__93, OTHERS);
    2.40      private static final Map<Integer, AntlrTokenId> codesMap = new HashMap<Integer, AntlrTokenId>();
    2.41  
    2.42      static {
    2.43 @@ -131,7 +161,9 @@
    2.44      }
    2.45  
    2.46      public static AntlrTokenId forTokenTypeCode(int tokenTypeCode) {
    2.47 -        return codesMap.get(tokenTypeCode);
    2.48 +        AntlrTokenId tid = codesMap.get(tokenTypeCode);
    2.49 +        assert tid != null : "No AntlrTokenId found for ANLR token code " + tokenTypeCode;
    2.50 +        return tid;
    2.51      }
    2.52      private final AntlrTokenIdCategory primaryCategory;
    2.53      private final int code;
     3.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     3.2 +++ b/o.n.antlr.editor/test/unit/src/org/netbeans/modules/antlr/editor/AntlrTestBase.java	Sat Jan 12 13:04:21 2013 +0100
     3.3 @@ -0,0 +1,71 @@
     3.4 +/*
     3.5 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
     3.6 + *
     3.7 + * Copyright 2013 Oracle and/or its affiliates. All rights reserved.
     3.8 + *
     3.9 + * Oracle and Java are registered trademarks of Oracle and/or its affiliates.
    3.10 + * Other names may be trademarks of their respective owners.
    3.11 + *
    3.12 + * The contents of this file are subject to the terms of either the GNU
    3.13 + * General Public License Version 2 only ("GPL") or the Common
    3.14 + * Development and Distribution License("CDDL") (collectively, the
    3.15 + * "License"). You may not use this file except in compliance with the
    3.16 + * License. You can obtain a copy of the License at
    3.17 + * http://www.netbeans.org/cddl-gplv2.html
    3.18 + * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
    3.19 + * specific language governing permissions and limitations under the
    3.20 + * License.  When distributing the software, include this License Header
    3.21 + * Notice in each file and include the License file at
    3.22 + * nbbuild/licenses/CDDL-GPL-2-CP.  Oracle designates this
    3.23 + * particular file as subject to the "Classpath" exception as provided
    3.24 + * by Oracle in the GPL Version 2 section of the License file that
    3.25 + * accompanied this code. If applicable, add the following below the
    3.26 + * License Header, with the fields enclosed by brackets [] replaced by
    3.27 + * your own identifying information:
    3.28 + * "Portions Copyrighted [year] [name of copyright owner]"
    3.29 + *
    3.30 + * If you wish your version of this file to be governed by only the CDDL
    3.31 + * or only the GPL Version 2, indicate your decision by adding
    3.32 + * "[Contributor] elects to include this software in this distribution
    3.33 + * under the [CDDL or GPL Version 2] license." If you do not indicate a
    3.34 + * single choice of license, a recipient has the option to distribute
    3.35 + * your version of this file under either the CDDL, the GPL Version 2 or
    3.36 + * to extend the choice of license to its licensees as provided above.
    3.37 + * However, if you add GPL Version 2 code and therefore, elected the GPL
    3.38 + * Version 2 license, then the option applies only if the new code is
    3.39 + * made subject to such option by the copyright holder.
    3.40 + *
    3.41 + * Contributor(s):
    3.42 + *
    3.43 + * Portions Copyrighted 2013 Sun Microsystems, Inc.
    3.44 + */
    3.45 +package org.netbeans.modules.antlr.editor;
    3.46 +
    3.47 +import org.netbeans.api.lexer.TokenHierarchy;
    3.48 +import org.netbeans.api.lexer.TokenSequence;
    3.49 +import org.netbeans.editor.BaseDocument;
    3.50 +import org.netbeans.modules.csl.api.test.CslTestBase;
    3.51 +import org.netbeans.modules.csl.spi.DefaultLanguageConfig;
    3.52 +import org.openide.filesystems.FileObject;
    3.53 +
    3.54 +/**
    3.55 + *
    3.56 + * @author marekfukala
    3.57 + */
    3.58 +public class AntlrTestBase extends CslTestBase {
    3.59 +
    3.60 +    public AntlrTestBase(String testName) {
    3.61 +        super(testName);
    3.62 +    }
    3.63 +
    3.64 +    @Override
    3.65 +    protected DefaultLanguageConfig getPreferredLanguage() {
    3.66 +        return new AntlrCslLanguage();
    3.67 +    }
    3.68 +
    3.69 +    @Override
    3.70 +    protected String getPreferredMimeType() {
    3.71 +        return "text/antlr";
    3.72 +    }
    3.73 +    
    3.74 +}
     4.1 --- a/o.n.antlr.editor/test/unit/src/org/netbeans/modules/antlr/editor/NbAntlrLexerTest.java	Sat Jan 12 08:46:08 2013 +0100
     4.2 +++ b/o.n.antlr.editor/test/unit/src/org/netbeans/modules/antlr/editor/NbAntlrLexerTest.java	Sat Jan 12 13:04:21 2013 +0100
     4.3 @@ -41,45 +41,43 @@
     4.4   */
     4.5  package org.netbeans.modules.antlr.editor;
     4.6  
     4.7 -import static junit.framework.Assert.assertEquals;
     4.8 -import static junit.framework.Assert.assertFalse;
     4.9 -import static junit.framework.Assert.assertTrue;
    4.10 -import junit.framework.TestCase;
    4.11 +import org.netbeans.api.lexer.Token;
    4.12  import org.netbeans.api.lexer.TokenHierarchy;
    4.13  import org.netbeans.api.lexer.TokenSequence;
    4.14 +import org.netbeans.editor.BaseDocument;
    4.15 +import org.openide.filesystems.FileObject;
    4.16  
    4.17  /**
    4.18   *
    4.19   * @author marekfukala
    4.20   */
    4.21 -public class NbAntlrLexerTest extends TestCase {
    4.22 -    
    4.23 +public class NbAntlrLexerTest extends AntlrTestBase {
    4.24 +
    4.25      public NbAntlrLexerTest(String testName) {
    4.26          super(testName);
    4.27      }
    4.28  
    4.29 -     public void test146930() {
    4.30 -//        TokenHierarchy th = TokenHierarchy.create("<<body>", HTMLTokenId.language());
    4.31 -//        TokenSequence ts = th.tokenSequence();
    4.32 -//        ts.moveStart();
    4.33 -//
    4.34 -//        assertTrue(ts.moveNext());
    4.35 -//        assertEquals("<", ts.token().text().toString());
    4.36 -//        assertEquals(HTMLTokenId.TEXT, ts.token().id());
    4.37 -//
    4.38 -//        assertTrue(ts.moveNext());
    4.39 -//        assertEquals("<", ts.token().text().toString());
    4.40 -//        assertEquals(HTMLTokenId.TAG_OPEN_SYMBOL, ts.token().id());
    4.41 -//
    4.42 -//        assertTrue(ts.moveNext());
    4.43 -//        assertEquals("body", ts.token().text().toString());
    4.44 -//        assertEquals(HTMLTokenId.TAG_OPEN, ts.token().id());
    4.45 -//
    4.46 -//        assertTrue(ts.moveNext());
    4.47 -//        assertEquals(">", ts.token().text().toString());
    4.48 -//        assertEquals(HTMLTokenId.TAG_CLOSE_SYMBOL, ts.token().id());
    4.49 -//
    4.50 -//        assertFalse(ts.moveNext());
    4.51 +    public void testLexAntlrGrammar() {
    4.52 +        FileObject testFile = getTestFile("testfiles/ANTLRv3.g");
    4.53 +        final BaseDocument document = getDocument(testFile);
    4.54 +
    4.55 +        document.render(new Runnable() {
    4.56 +
    4.57 +            @Override
    4.58 +            public void run() {
    4.59 +                TokenHierarchy th = TokenHierarchy.get(document);
    4.60 +                assertNotNull(th);
    4.61 +                TokenSequence<AntlrTokenId> ts = th.tokenSequence(AntlrTokenId.language());
    4.62 +                ts.moveStart();
    4.63 +                
    4.64 +                while(ts.moveNext()) {
    4.65 +                    Token<AntlrTokenId> t = ts.token();
    4.66 +                    System.out.println("'" + t.text() + "', " + t.id());
    4.67 +                }
    4.68 +            }
    4.69 +            
    4.70 +        });
    4.71 +
    4.72      }
    4.73      
    4.74  }