Adding ability to run (custom) tests.
authorjlahoda
Sun, 12 Feb 2017 10:11:52 +0100
changeset 1041b03a880d538e
parent 1040 f7b6892fd754
child 1042 06b3af1708ef
Adding ability to run (custom) tests.
cmdline/tool/nbproject/genfiles.properties
cmdline/tool/nbproject/project.xml
cmdline/tool/src/org/netbeans/modules/jackpot30/cmdline/Main.java
cmdline/tool/test/unit/src/org/netbeans/modules/jackpot30/cmdline/MainTest.java
     1.1 --- a/cmdline/tool/nbproject/genfiles.properties	Sun Jan 08 22:32:39 2017 +0100
     1.2 +++ b/cmdline/tool/nbproject/genfiles.properties	Sun Feb 12 10:11:52 2017 +0100
     1.3 @@ -44,6 +44,6 @@
     1.4  build.xml.stylesheet.CRC32=a56c6a5b@1.44
     1.5  # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml.
     1.6  # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you.
     1.7 -nbproject/build-impl.xml.data.CRC32=c79833a9
     1.8 +nbproject/build-impl.xml.data.CRC32=8514a197
     1.9  nbproject/build-impl.xml.script.CRC32=08f1fb11
    1.10 -nbproject/build-impl.xml.stylesheet.CRC32=238281d1@2.70
    1.11 +nbproject/build-impl.xml.stylesheet.CRC32=238281d1@2.73
     2.1 --- a/cmdline/tool/nbproject/project.xml	Sun Jan 08 22:32:39 2017 +0100
     2.2 +++ b/cmdline/tool/nbproject/project.xml	Sun Feb 12 10:11:52 2017 +0100
     2.3 @@ -148,6 +148,15 @@
     2.4                      </run-dependency>
     2.5                  </dependency>
     2.6                  <dependency>
     2.7 +                    <code-name-base>org.netbeans.modules.java.hints.declarative</code-name-base>
     2.8 +                    <build-prerequisite/>
     2.9 +                    <compile-dependency/>
    2.10 +                    <run-dependency>
    2.11 +                        <release-version>1</release-version>
    2.12 +                        <implementation-version/>
    2.13 +                    </run-dependency>
    2.14 +                </dependency>
    2.15 +                <dependency>
    2.16                      <code-name-base>org.netbeans.modules.java.hints.ui</code-name-base>
    2.17                      <build-prerequisite/>
    2.18                      <compile-dependency/>
    2.19 @@ -387,10 +396,6 @@
    2.20                          <compile-dependency/>
    2.21                      </test-dependency>
    2.22                      <test-dependency>
    2.23 -                        <code-name-base>org.netbeans.modules.java.hints.declarative</code-name-base>
    2.24 -                        <compile-dependency/>
    2.25 -                    </test-dependency>
    2.26 -                    <test-dependency>
    2.27                          <code-name-base>org.netbeans.modules.java.hints.declarative.test</code-name-base>
    2.28                          <compile-dependency/>
    2.29                      </test-dependency>
     3.1 --- a/cmdline/tool/src/org/netbeans/modules/jackpot30/cmdline/Main.java	Sun Jan 08 22:32:39 2017 +0100
     3.2 +++ b/cmdline/tool/src/org/netbeans/modules/jackpot30/cmdline/Main.java	Sun Feb 12 10:11:52 2017 +0100
     3.3 @@ -54,6 +54,7 @@
     3.4  import java.util.ArrayList;
     3.5  import java.util.Arrays;
     3.6  import java.util.Collection;
     3.7 +import java.util.Collections;
     3.8  import java.util.HashMap;
     3.9  import java.util.HashSet;
    3.10  import java.util.Iterator;
    3.11 @@ -87,6 +88,10 @@
    3.12  import org.netbeans.core.startup.MainLookup;
    3.13  import org.netbeans.modules.jackpot30.cmdline.lib.Utils;
    3.14  import org.netbeans.modules.jackpot30.ui.settings.XMLHintPreferences;
    3.15 +import org.netbeans.modules.java.hints.declarative.DeclarativeHintRegistry;
    3.16 +import org.netbeans.modules.java.hints.declarative.test.TestParser;
    3.17 +import org.netbeans.modules.java.hints.declarative.test.TestParser.TestCase;
    3.18 +import org.netbeans.modules.java.hints.declarative.test.TestPerformer;
    3.19  import org.netbeans.modules.java.hints.jackpot.spi.PatternConvertor;
    3.20  import org.netbeans.modules.java.hints.providers.spi.HintDescription;
    3.21  import org.netbeans.modules.java.hints.providers.spi.HintMetadata;
    3.22 @@ -108,6 +113,8 @@
    3.23  import org.netbeans.modules.parsing.impl.indexing.CacheFolder;
    3.24  import org.netbeans.modules.parsing.impl.indexing.RepositoryUpdater;
    3.25  import org.netbeans.spi.editor.hints.ErrorDescription;
    3.26 +import org.netbeans.spi.editor.hints.ErrorDescriptionFactory;
    3.27 +import org.netbeans.spi.editor.hints.Fix;
    3.28  import org.netbeans.spi.editor.hints.Severity;
    3.29  import org.netbeans.spi.java.classpath.ClassPathProvider;
    3.30  import org.netbeans.spi.java.classpath.support.ClassPathSupport;
    3.31 @@ -133,6 +140,7 @@
    3.32      private static final String OPTION_APPLY = "apply";
    3.33      private static final String OPTION_NO_APPLY = "no-apply";
    3.34      private static final String OPTION_FAIL_ON_WARNINGS = "fail-on-warnings";
    3.35 +    private static final String RUN_TESTS = "run-tests";
    3.36      private static final String SOURCE_LEVEL_DEFAULT = "1.7";
    3.37      private static final String ACCEPTABLE_SOURCE_LEVEL_PATTERN = "(1\\.)?[2-9][0-9]*";
    3.38      
    3.39 @@ -162,6 +170,7 @@
    3.40          parser.accepts(OPTION_APPLY, "apply changes");
    3.41          parser.accepts("show-gui", "show configuration dialog");
    3.42          parser.accepts(OPTION_FAIL_ON_WARNINGS, "fail when warnings are detected");
    3.43 +        parser.accepts(RUN_TESTS, "run tests for declarative rules that were used");
    3.44  
    3.45          OptionSet parsed;
    3.46  
    3.47 @@ -286,6 +295,7 @@
    3.48              Preferences hintSettingsPreferences;
    3.49              boolean apply;
    3.50              boolean runDeclarative;
    3.51 +            boolean runDeclarativeTests;
    3.52  
    3.53              if (parsed.has(configFile)) {
    3.54                  Preferences settingsFromConfigFile;
    3.55 @@ -293,6 +303,7 @@
    3.56                  hintSettingsPreferences = settingsFromConfigFile.node("settings");
    3.57                  apply = settingsFromConfigFile.getBoolean("apply", false);
    3.58                  runDeclarative = settingsFromConfigFile.getBoolean("runDeclarative", true);
    3.59 +                runDeclarativeTests = settingsFromConfigFile.getBoolean("runDeclarativeTests", false);
    3.60                  if (parsed.has(hint)) {
    3.61                      System.err.println("cannot specify --hint and --config-file together");
    3.62                      return 1;
    3.63 @@ -304,6 +315,7 @@
    3.64                  hintSettingsPreferences = null;
    3.65                  apply = false;
    3.66                  runDeclarative = true;
    3.67 +                runDeclarativeTests = parsed.has(RUN_TESTS);
    3.68              }
    3.69  
    3.70              if (parsed.has(config) && !parsed.has(hint)) {
    3.71 @@ -320,7 +332,7 @@
    3.72              GroupResult result = GroupResult.NOTHING_TO_DO;
    3.73  
    3.74              try (Writer outS = parsed.has(out) ? new BufferedWriter(new OutputStreamWriter(new FileOutputStream(parsed.valueOf(out)))) : null) {
    3.75 -                GlobalConfiguration globalConfig = new GlobalConfiguration(hintSettingsPreferences, apply, runDeclarative, parsed.valueOf(hint), parsed.valueOf(hintFile), outS, parsed.has(OPTION_FAIL_ON_WARNINGS));
    3.76 +                GlobalConfiguration globalConfig = new GlobalConfiguration(hintSettingsPreferences, apply, runDeclarative, runDeclarativeTests, parsed.valueOf(hint), parsed.valueOf(hintFile), outS, parsed.has(OPTION_FAIL_ON_WARNINGS));
    3.77  
    3.78                  for (RootConfiguration groupConfig : groups) {
    3.79                      result = result.join(handleGroup(groupConfig, progress, globalConfig, parsed.valuesOf(config)));
    3.80 @@ -405,6 +417,8 @@
    3.81              return GroupResult.NOTHING_TO_DO;
    3.82          }
    3.83  
    3.84 +        WarningsAndErrors wae = new WarningsAndErrors();
    3.85 +
    3.86          ProgressHandleWrapper progress = w.startNextPartWithEmbedding(1);
    3.87          Preferences settings = globalConfig.configurationPreferences != null ? globalConfig.configurationPreferences : new MemoryPreferences();
    3.88          HintsSettings hintSettings = HintsSettings.createPreferencesBasedHintsSettings(settings, false, null);
    3.89 @@ -420,6 +434,46 @@
    3.90              }
    3.91          } else {
    3.92              hints = readHints(rootConfiguration.sourceCP, rootConfiguration.binaryCP, hintSettings, settings, globalConfig.runDeclarative);
    3.93 +            if (globalConfig.runDeclarativeTests) {
    3.94 +                Set<String> enabledHints = new HashSet<>();
    3.95 +                for (HintDescription desc : hints) {
    3.96 +                    enabledHints.add(desc.getMetadata().id);
    3.97 +                }
    3.98 +                ClassPath combined = ClassPathSupport.createProxyClassPath(rootConfiguration.sourceCP, rootConfiguration.binaryCP);
    3.99 +                Map<FileObject, FileObject> testFiles = new HashMap<>();
   3.100 +                for (FileObject upgrade : combined.findAllResources("META-INF/upgrade")) {
   3.101 +                    for (FileObject c : upgrade.getChildren()) {
   3.102 +                        if (c.getExt().equals("test")) {
   3.103 +                            FileObject hintFile = FileUtil.findBrother(c, "hint");
   3.104 +
   3.105 +                            for (HintMetadata hm : DeclarativeHintRegistry.parseHintFile(hintFile).keySet()) {
   3.106 +                                if (enabledHints.contains(hm.id)) {
   3.107 +                                    testFiles.put(c, hintFile);
   3.108 +                                    break;
   3.109 +                                }
   3.110 +                            }
   3.111 +                        }
   3.112 +                    }
   3.113 +                }
   3.114 +                for (Entry<FileObject, FileObject> e : testFiles.entrySet()) {
   3.115 +                    TestCase[] testCases = TestParser.parse(e.getKey().asText()); //XXX: encoding
   3.116 +                    try {
   3.117 +                        Map<TestCase, Collection<String>> testResult = TestPerformer.performTest(e.getValue(), e.getKey(), testCases, new AtomicBoolean());
   3.118 +                        for (TestCase tc : testCases) {
   3.119 +                            List<String> expected = Arrays.asList(tc.getResults());
   3.120 +                            List<String> actual = new ArrayList<>(testResult.get(tc));
   3.121 +                            if (!expected.equals(actual)) {
   3.122 +                                int pos = tc.getTestCaseStart();
   3.123 +                                String id = "test-failure";
   3.124 +                                ErrorDescription ed = ErrorDescriptionFactory.createErrorDescription(id, Severity.ERROR, "Actual results did not match the expected test results. Actual results: " + expected, null, ErrorDescriptionFactory.lazyListForFixes(Collections.<Fix>emptyList()), e.getKey(), pos, pos);
   3.125 +                                print(ed, wae, Collections.singletonMap(id, id));
   3.126 +                            }
   3.127 +                        }
   3.128 +                    } catch (Exception ex) {
   3.129 +                        ex.printStackTrace();
   3.130 +                    }
   3.131 +                }
   3.132 +            }
   3.133          }
   3.134  
   3.135          if (config != null && !config.isEmpty()) {
   3.136 @@ -478,10 +532,8 @@
   3.137              if (globalConfig.apply) {
   3.138                  apply(hints, rootConfiguration.rootFolders.toArray(new Folder[0]), progress, hintSettings, globalConfig.out);
   3.139  
   3.140 -                return GroupResult.SUCCESS;
   3.141 +                return GroupResult.SUCCESS; //TODO: WarningsAndErrors?
   3.142              } else {
   3.143 -                WarningsAndErrors wae = new WarningsAndErrors();
   3.144 -
   3.145                  findOccurrences(hints, rootConfiguration.rootFolders.toArray(new Folder[0]), progress, hintSettings, wae);
   3.146  
   3.147                  if (wae.errors != 0 || (wae.warnings != 0 && globalConfig.failOnWarnings)) {
   3.148 @@ -870,15 +922,17 @@
   3.149          private final Preferences configurationPreferences;
   3.150          private final boolean apply;
   3.151          private final boolean runDeclarative;
   3.152 +        private final boolean runDeclarativeTests;
   3.153          private final String hint;
   3.154          private final File hintFile;
   3.155          private final Writer out;
   3.156          private final boolean failOnWarnings;
   3.157  
   3.158 -        public GlobalConfiguration(Preferences configurationPreferences, boolean apply, boolean runDeclarative, String hint, File hintFile, Writer out, boolean failOnWarnings) {
   3.159 +        public GlobalConfiguration(Preferences configurationPreferences, boolean apply, boolean runDeclarative, boolean runDeclarativeTests, String hint, File hintFile, Writer out, boolean failOnWarnings) {
   3.160              this.configurationPreferences = configurationPreferences;
   3.161              this.apply = apply;
   3.162              this.runDeclarative = runDeclarative;
   3.163 +            this.runDeclarativeTests = runDeclarativeTests;
   3.164              this.hint = hint;
   3.165              this.hintFile = hintFile;
   3.166              this.out = out;
     4.1 --- a/cmdline/tool/test/unit/src/org/netbeans/modules/jackpot30/cmdline/MainTest.java	Sun Jan 08 22:32:39 2017 +0100
     4.2 +++ b/cmdline/tool/test/unit/src/org/netbeans/modules/jackpot30/cmdline/MainTest.java	Sun Feb 12 10:11:52 2017 +0100
     4.3 @@ -536,12 +536,120 @@
     4.4                        "--apply");
     4.5      }
     4.6  
     4.7 +    public void testAutomaticTestRun() throws Exception {
     4.8 +        class Config {
     4.9 +            private final String commandLineOption;
    4.10 +            private final int result;
    4.11 +            private final Validator validator;
    4.12 +
    4.13 +            public Config(String commandLineOption, int result, Validator validator) {
    4.14 +                this.commandLineOption = commandLineOption;
    4.15 +                this.result = result;
    4.16 +                this.validator = validator;
    4.17 +            }
    4.18 +
    4.19 +        }
    4.20 +        Validator testValidator =
    4.21 +                equivalentValidator("${workdir}/src/META-INF/upgrade/test.test:15: error: [test_failure] Actual results did not match the expected test results. Actual results: []\n" +
    4.22 +                                    "%%TestCase pos-neg\n" +
    4.23 +                                    "^\n" +
    4.24 +                                    "${workdir}/src/META-INF/upgrade/test.test:29: error: [test_failure] Actual results did not match the expected test results. Actual results: [package test;\n" +
    4.25 +                                    "public class Test {{\n" +
    4.26 +                                    "    private void test(java.util.Collection c) {\n" +
    4.27 +                                    "        boolean b1 = c.isEmpty();\n" +
    4.28 +                                    "    }\n" +
    4.29 +                                    "}}\n" +
    4.30 +                                    "]\n" +
    4.31 +                                    "%%TestCase neg-neg\n" +
    4.32 +                                    "^\n" +
    4.33 +                                    "${workdir}/src/test/Test.java:4: warning: [test] test\n" +
    4.34 +                                    "        boolean b1 = c.size() == 0;\n" +
    4.35 +                                    "                     ^\n");
    4.36 +        List<? extends Config> options =
    4.37 +                Arrays.asList(new Config("--run-tests", 1, testValidator),
    4.38 +                              new Config(IGNORE, 0, equivalentValidator("${workdir}/src/test/Test.java:4: warning: [test] test\n" +
    4.39 +                                                                        "        boolean b1 = c.size() == 0;\n" +
    4.40 +                                                                        "                     ^\n")));
    4.41 +        for (Config testConfig : options) {
    4.42 +            doRunCompiler(equivalentValidator("package test;\n" +
    4.43 +                                              "public class Test {\n" +
    4.44 +                                              "    private void test(java.util.Collection c) {\n" +
    4.45 +                                              "        boolean b1 = c.size() == 0;\n" +
    4.46 +                                              "        boolean b2 = c.size() != 0;\n" +
    4.47 +                                              "    }\n" +
    4.48 +                                              "}\n"),
    4.49 +                          testConfig.validator,
    4.50 +                          equivalentValidator(""),
    4.51 +                          testConfig.result,
    4.52 +                          "src/META-INF/upgrade/test.hint",
    4.53 +                          "$coll.size() == 0 :: $coll instanceof java.util.Collection\n" +
    4.54 +                          "=>\n" +
    4.55 +                          "$coll.isEmpty()\n" +
    4.56 +                          ";;",
    4.57 +                          "src/META-INF/upgrade/test.test",
    4.58 +                          "%%TestCase pos-pos\n" +
    4.59 +                          "package test;\n" +
    4.60 +                          "public class Test {{\n" +
    4.61 +                          "    private void test(java.util.Collection c) {\n" +
    4.62 +                          "        boolean b1 = c.size() == 0;\n" +
    4.63 +                          "    }\n" +
    4.64 +                          "}}\n" +
    4.65 +                          "%%=>\n" +
    4.66 +                          "package test;\n" +
    4.67 +                          "public class Test {{\n" +
    4.68 +                          "    private void test(java.util.Collection c) {\n" +
    4.69 +                          "        boolean b1 = c.isEmpty();\n" +
    4.70 +                          "    }\n" +
    4.71 +                          "}}\n" +
    4.72 +                          "%%TestCase pos-neg\n" +
    4.73 +                          "package test;\n" +
    4.74 +                          "public class Test {{\n" +
    4.75 +                          "    private void test(java.util.Collection c) {\n" +
    4.76 +                          "        boolean b1 = c.size() == 0;\n" +
    4.77 +                          "    }\n" +
    4.78 +                          "}}\n" +
    4.79 +                          "%%TestCase neg-pos\n" +
    4.80 +                          "package test;\n" +
    4.81 +                          "public class Test {{\n" +
    4.82 +                          "    private void test(java.util.Collection c) {\n" +
    4.83 +                          "        boolean b1 = c.size() != 0;\n" +
    4.84 +                          "    }\n" +
    4.85 +                          "}}\n" +
    4.86 +                          "%%TestCase neg-neg\n" +
    4.87 +                          "package test;\n" +
    4.88 +                          "public class Test {{\n" +
    4.89 +                          "    private void test(java.util.Collection c) {\n" +
    4.90 +                          "        boolean b1 = c.size() != 0;\n" +
    4.91 +                          "    }\n" +
    4.92 +                          "}}\n" +
    4.93 +                          "%%=>\n" +
    4.94 +                          "package test;\n" +
    4.95 +                          "public class Test {{\n" +
    4.96 +                          "    private void test(java.util.Collection c) {\n" +
    4.97 +                          "        boolean b1 = c.isEmpty();\n" +
    4.98 +                          "    }\n" +
    4.99 +                          "}}\n",
   4.100 +                          "src/test/Test.java",
   4.101 +                          "package test;\n" +
   4.102 +                          "public class Test {\n" +
   4.103 +                          "    private void test(java.util.Collection c) {\n" +
   4.104 +                          "        boolean b1 = c.size() == 0;\n" +
   4.105 +                          "        boolean b2 = c.size() != 0;\n" +
   4.106 +                          "    }\n" +
   4.107 +                          "}\n",
   4.108 +                          null,
   4.109 +                          testConfig.commandLineOption,
   4.110 +                          "--sourcepath", "${workdir}/src");
   4.111 +        }
   4.112 +    }
   4.113 +
   4.114      public void testGroupsParamEscape() throws Exception {
   4.115          assertEquals(Arrays.asList("a b", "a\\b"),
   4.116                       Arrays.asList(Main.splitGroupArg("a\\ b a\\\\b")));
   4.117      }
   4.118  
   4.119      private static final String DONT_APPEND_PATH = new String("DONT_APPEND_PATH");
   4.120 +    private static final String IGNORE = new String("IGNORE");
   4.121  
   4.122      private void doRunCompiler(String golden, String stdOut, String stdErr, String... fileContentAndExtraOptions) throws Exception {
   4.123          doRunCompiler(equivalentValidator(golden), equivalentValidator(stdOut), equivalentValidator(stdErr), fileContentAndExtraOptions);
   4.124 @@ -590,6 +698,9 @@
   4.125                  appendPath = false;
   4.126                  continue;
   4.127              }
   4.128 +            if (extraOption == IGNORE) {
   4.129 +                continue;
   4.130 +            }
   4.131              options.add(extraOption.replace("${workdir}", wd.getAbsolutePath()));
   4.132          }
   4.133