1.1 --- a/api/src/org/netbeans/modules/jackpot30/impl/indexing/AbstractLuceneIndex.java	Tue Feb 08 22:08:44 2011 +0100
     1.2 +++ b/api/src/org/netbeans/modules/jackpot30/impl/indexing/AbstractLuceneIndex.java	Tue Feb 08 23:13:13 2011 +0100
     1.3 @@ -50,9 +50,11 @@
     1.4  import java.util.BitSet;
     1.5  import java.util.Collection;
     1.6  import java.util.Collections;
     1.7 +import java.util.HashMap;
     1.8  import java.util.HashSet;
     1.9  import java.util.Iterator;
    1.10  import java.util.List;
    1.11 +import java.util.Map;
    1.12  import java.util.Set;
    1.13  import java.util.logging.Level;
    1.14  import java.util.logging.Logger;
    1.15 @@ -60,7 +62,6 @@
    1.16  import javax.lang.model.type.ArrayType;
    1.17  import javax.lang.model.type.TypeKind;
    1.18  import javax.lang.model.type.TypeMirror;
    1.19 -import org.apache.lucene.analysis.Token;
    1.20  import org.apache.lucene.analysis.TokenStream;
    1.21  import org.apache.lucene.analysis.tokenattributes.TermAttribute;
    1.22  import org.apache.lucene.document.CompressionTools;
    1.23 @@ -119,10 +120,19 @@
    1.24  
    1.25      @Override
    1.26      public Collection<? extends String> findCandidates(BulkPattern pattern) throws IOException {
    1.27 +        return findCandidates(pattern, false).keySet();
    1.28 +    }
    1.29 +
    1.30 +    @Override
    1.31 +    public Map<String, Map<String, Integer>> findCandidatesWithFrequencies(BulkPattern pattern) throws IOException {
    1.32 +        return findCandidates(pattern, true);
    1.33 +    }
    1.34 +
    1.35 +    private Map<String, Map<String, Integer>> findCandidates(BulkPattern pattern, boolean withFrequencies) throws IOException {
    1.36          IndexReader reader = createReader();
    1.37  
    1.38          if (reader == null) {
    1.39 -             return Collections.emptyList();
    1.40 +             return Collections.emptyMap();
    1.41           }
    1.42  
    1.43          Searcher s = new IndexSearcher(reader);
    1.44 @@ -135,7 +145,7 @@
    1.45              throw new IOException(ex);
    1.46          }
    1.47  
    1.48 -        Collection<String> result = new HashSet<String>();
    1.49 +        Map<String, Map<String, Integer>> result = new HashMap<String, Map<String, Integer>>();
    1.50  
    1.51          for (int docNum = matchingDocuments.nextSetBit(0); docNum >= 0; docNum = matchingDocuments.nextSetBit(docNum+1)) {
    1.52              try {
    1.53 @@ -148,14 +158,24 @@
    1.54                  ByteArrayInputStream in = new ByteArrayInputStream(CompressionTools.decompress(doc.getField("encoded").getBinaryValue()));
    1.55  
    1.56                  try {
    1.57 -                    if (!BulkSearch.getDefault().matches(in, pattern)) {
    1.58 +                    Map<String, Integer> freqs;
    1.59 +                    boolean matches;
    1.60 +
    1.61 +                    if (withFrequencies) {
    1.62 +                        freqs = BulkSearch.getDefault().matchesWithFrequencies(in, pattern);
    1.63 +                        matches = !freqs.isEmpty();
    1.64 +                    } else {
    1.65 +                        freqs = null;
    1.66 +                        matches = BulkSearch.getDefault().matches(in, pattern);
    1.67 +                    }
    1.68 +
    1.69 +                    if (matches) {
    1.70 +                        result.put(doc.getField("path").stringValue(), freqs);
    1.71                          continue;
    1.72                      }
    1.73                  } finally {
    1.74                      in.close();
    1.75                  }
    1.76 -
    1.77 -                result.add(doc.getField("path").stringValue());
    1.78              } catch (DataFormatException ex) {
    1.79                  throw new IOException(ex);
    1.80              }
    1.81 @@ -226,7 +246,7 @@
    1.82                  }
    1.83              });
    1.84              
    1.85 -            return CompressionTools.decompressString(doc.getField("sourceCode").getBinaryValue());
    1.86 +            return CompressionTools.decompressString(doc.getField("sourceCode").getBinaryValue()).replaceAll("\r\n", "\n")/*XXX*/;
    1.87          } catch (DataFormatException ex) {
    1.88              Exceptions.printStackTrace(ex);
    1.89              return "";

     2.1 --- a/api/src/org/netbeans/modules/jackpot30/impl/indexing/Index.java	Tue Feb 08 22:08:44 2011 +0100
     2.2 +++ b/api/src/org/netbeans/modules/jackpot30/impl/indexing/Index.java	Tue Feb 08 23:13:13 2011 +0100
     2.3 @@ -39,7 +39,7 @@
     2.4  
     2.5  package org.netbeans.modules.jackpot30.impl.indexing;
     2.6  
     2.7 -import org.netbeans.modules.jackpot30.spi.HintDescription.AdditionalQueryConstraints;
     2.8 +import java.util.Map;
     2.9  import com.sun.source.util.Trees;
    2.10  import javax.lang.model.util.Types;
    2.11  import org.netbeans.modules.jackpot30.impl.WebUtilities;
    2.12 @@ -93,6 +93,10 @@
    2.13                  }
    2.14              }
    2.15              @Override
    2.16 +            public Map<String, Map<String, Integer>> findCandidatesWithFrequencies(BulkPattern pattern) throws IOException {
    2.17 +                throw new UnsupportedOperationException("Not supported yet.");
    2.18 +            }
    2.19 +            @Override
    2.20              public @NonNull IndexInfo getIndexInfo() {
    2.21                  IndexInfo result = IndexInfo.empty();
    2.22  
    2.23 @@ -110,7 +114,7 @@
    2.24              @Override
    2.25              public CharSequence getSourceCode(String relativePath) {
    2.26                  try {
    2.27 -                    URI u = new URI(indexURL + "?path=" + escapeForQuery(subIndex) + "&relative=" + escapeForQuery(relativePath));
    2.28 +                    URI u = new URI(indexURL + "/cat?path=" + escapeForQuery(subIndex) + "&relative=" + escapeForQuery(relativePath));
    2.29  
    2.30                      return WebUtilities.requestStringResponse(u);
    2.31                  } catch (URISyntaxException ex) {
    2.32 @@ -125,6 +129,7 @@
    2.33      public abstract IndexWriter openForWriting() throws IOException;
    2.34  
    2.35      public abstract Collection<? extends String> findCandidates(BulkPattern pattern) throws IOException;
    2.36 +    public abstract Map<String, Map<String, Integer>> findCandidatesWithFrequencies(BulkPattern pattern) throws IOException;
    2.37  
    2.38      public abstract CharSequence getSourceCode(String relativePath);
    2.39  

     3.1 --- a/api/src/org/netbeans/modules/jackpot30/impl/pm/BulkSearch.java	Tue Feb 08 22:08:44 2011 +0100
     3.2 +++ b/api/src/org/netbeans/modules/jackpot30/impl/pm/BulkSearch.java	Tue Feb 08 23:13:13 2011 +0100
     3.3 @@ -83,6 +83,7 @@
     3.4      public abstract Map<String, Collection<TreePath>> match(CompilationInfo info, TreePath toSearch, BulkPattern pattern, Map<String, Long> timeLog);
     3.5  
     3.6      public abstract boolean matches(InputStream encoded, BulkPattern pattern);
     3.7 +    public abstract Map<String, Integer> matchesWithFrequencies(InputStream encoded, BulkPattern pattern);
     3.8      
     3.9      public abstract boolean matches(CompilationInfo info, TreePath toSearch, BulkPattern pattern);
    3.10  

     4.1 --- a/api/src/org/netbeans/modules/jackpot30/impl/pm/CopyFinderBasedBulkSearch.java	Tue Feb 08 22:08:44 2011 +0100
     4.2 +++ b/api/src/org/netbeans/modules/jackpot30/impl/pm/CopyFinderBasedBulkSearch.java	Tue Feb 08 23:13:13 2011 +0100
     4.3 @@ -117,6 +117,11 @@
     4.4          throw new UnsupportedOperationException("Not supported yet.");
     4.5      }
     4.6  
     4.7 +    @Override
     4.8 +    public Map<String, Integer> matchesWithFrequencies(InputStream encoded, BulkPattern pattern) {
     4.9 +        throw new UnsupportedOperationException("Not supported yet.");
    4.10 +    }
    4.11 +
    4.12      private static final class BulkPatternImpl extends BulkPattern {
    4.13  
    4.14          private final Map<Tree, String> pattern2Code;

     5.1 --- a/api/src/org/netbeans/modules/jackpot30/impl/pm/NFABasedBulkSearch.java	Tue Feb 08 22:08:44 2011 +0100
     5.2 +++ b/api/src/org/netbeans/modules/jackpot30/impl/pm/NFABasedBulkSearch.java	Tue Feb 08 23:13:13 2011 +0100
     5.3 @@ -56,6 +56,7 @@
     5.4  import java.io.UnsupportedEncodingException;
     5.5  import java.util.ArrayList;
     5.6  import java.util.Collection;
     5.7 +import java.util.Collections;
     5.8  import java.util.EnumMap;
     5.9  import java.util.EnumSet;
    5.10  import java.util.HashMap;
    5.11 @@ -473,14 +474,23 @@
    5.12      @Override
    5.13      public boolean matches(InputStream encoded, BulkPattern patternIn) {
    5.14          try {
    5.15 -            return matchesImpl(encoded, patternIn);
    5.16 +            return !matchesImpl(encoded, patternIn, false).isEmpty();
    5.17          } catch (IOException ex) {
    5.18              Exceptions.printStackTrace(ex);
    5.19              return false;
    5.20          }
    5.21      }
    5.22  
    5.23 -    private boolean matchesImpl(InputStream encoded, BulkPattern patternIn) throws IOException {
    5.24 +    public Map<String, Integer> matchesWithFrequencies(InputStream encoded, BulkPattern patternIn) {
    5.25 +        try {
    5.26 +            return matchesImpl(encoded, patternIn, true);
    5.27 +        } catch (IOException ex) {
    5.28 +            Exceptions.printStackTrace(ex);
    5.29 +            return Collections.emptyMap();
    5.30 +        }
    5.31 +    }
    5.32 +
    5.33 +    public Map<String, Integer> matchesImpl(InputStream encoded, BulkPattern patternIn, boolean withFrequencies) throws IOException {
    5.34          BulkPatternImpl pattern = (BulkPatternImpl) patternIn;
    5.35          final NFA<Input, Res> nfa = pattern.toNFA();
    5.36          Stack<NFA.State> skips = new Stack<NFA.State>();
    5.37 @@ -508,6 +518,7 @@
    5.38              identifiers.add(new String(baos.toByteArray(), "UTF-8"));
    5.39          }
    5.40  
    5.41 +        Map<String, Integer> patternsAndFrequencies = new HashMap<String, Integer>();
    5.42          int read = encoded.read();
    5.43          
    5.44          while (read != (-1)) {
    5.45 @@ -560,7 +571,16 @@
    5.46                  
    5.47                  for (Res res : nfa.getResults(active)) {
    5.48                      if (identifiers.containsAll(pattern.getIdentifiers().get(res.patternIndex))) {
    5.49 -                        return true;
    5.50 +                        if (!withFrequencies) {
    5.51 +                            patternsAndFrequencies.put(res.pattern, 1);
    5.52 +                            return patternsAndFrequencies;
    5.53 +                        }
    5.54 +                        
    5.55 +                        Integer freqs = patternsAndFrequencies.get(res.pattern);
    5.56 +
    5.57 +                        if (freqs == null) freqs = 0;
    5.58 +
    5.59 +                        patternsAndFrequencies.put(res.pattern, freqs + 1);
    5.60                      }
    5.61                  }
    5.62  
    5.63 @@ -568,7 +588,7 @@
    5.64              }
    5.65          }
    5.66  
    5.67 -        return false;
    5.68 +        return patternsAndFrequencies;
    5.69      }
    5.70  
    5.71      private static final Map<Kind, byte[]> kind2Encoded;

     6.1 --- a/api/test/unit/src/org/netbeans/modules/jackpot30/impl/indexing/IndexTest.java	Tue Feb 08 22:08:44 2011 +0100
     6.2 +++ b/api/test/unit/src/org/netbeans/modules/jackpot30/impl/indexing/IndexTest.java	Tue Feb 08 23:13:13 2011 +0100
     6.3 @@ -39,10 +39,12 @@
     6.4  
     6.5  package org.netbeans.modules.jackpot30.impl.indexing;
     6.6  
     6.7 +import java.util.Map;
     6.8  import org.netbeans.modules.jackpot30.impl.Utilities;
     6.9  import java.util.Collections;
    6.10  import java.net.URL;
    6.11  import java.util.Arrays;
    6.12 +import java.util.HashMap;
    6.13  import java.util.HashSet;
    6.14  import java.util.Set;
    6.15  import org.netbeans.api.java.classpath.ClassPath;
    6.16 @@ -149,6 +151,33 @@
    6.17          verifyIndex("$1.length()", new AdditionalQueryConstraints(Collections.singleton("java.lang.CharSequence")), "test/Test1.java");
    6.18      }
    6.19  
    6.20 +    public void testFrequencies() throws Exception {
    6.21 +        writeFilesAndWaitForScan(src,
    6.22 +                                 new File("test/Test1.java", "package test; public class Test1 { private void test() { java.io.File f = null; f.isDirectory(); f.isDirectory(); } }"),
    6.23 +                                 new File("test/Test2.java", "package test; public class Test2 { private void test() { new javax.swing.ImageIcon(null); java.io.File f = null; f.isDirectory(); } }"),
    6.24 +                                 new File("test/Test3.java", "package test; public class Test3 { private void test() { new javax.swing.ImageIcon(null); new javax.swing.ImageIcon(null); } }")
    6.25 +                                );
    6.26 +
    6.27 +        String[] patterns = new String[] {
    6.28 +            "$1.isDirectory()",
    6.29 +            "new ImageIcon($1)"
    6.30 +        };
    6.31 +
    6.32 +        Map<String, Map<String, Integer>> golden = new HashMap<String, Map<String, Integer>>();
    6.33 +        
    6.34 +        golden.put("test/Test3.java", Collections.singletonMap("new ImageIcon($1)", 2));
    6.35 +        golden.put("test/Test1.java", Collections.singletonMap("$1.isDirectory()", 2));
    6.36 +        
    6.37 +        Map<String, Integer> freqsTest2 = new HashMap<String, Integer>();
    6.38 +        
    6.39 +        freqsTest2.put("$1.isDirectory()", 1);
    6.40 +        freqsTest2.put("new ImageIcon($1)", 1);
    6.41 +        
    6.42 +        golden.put("test/Test2.java", freqsTest2);
    6.43 +        
    6.44 +        verifyIndexWithFrequencies(patterns, golden);
    6.45 +    }
    6.46 +
    6.47      private void verifyIndex(final String[] patterns, String... containedIn) throws Exception {
    6.48          ClassPath EMPTY = ClassPathSupport.createClassPath(new FileObject[0]);
    6.49          ClasspathInfo cpInfo = ClasspathInfo.create(ClassPathSupport.createClassPath(SourceUtilsTestUtil.getBootClassPath().toArray(new URL[0])),
    6.50 @@ -191,4 +220,21 @@
    6.51          assertEquals(golden, real);
    6.52      }
    6.53  
    6.54 +    private void verifyIndexWithFrequencies(final String[] patterns, Map<String, Map<String, Integer>> golden) throws Exception {
    6.55 +        ClassPath EMPTY = ClassPathSupport.createClassPath(new FileObject[0]);
    6.56 +        ClasspathInfo cpInfo = ClasspathInfo.create(ClassPathSupport.createClassPath(SourceUtilsTestUtil.getBootClassPath().toArray(new URL[0])),
    6.57 +                                                    EMPTY,
    6.58 +                                                    EMPTY);
    6.59 +
    6.60 +        final Map<String, Map<String, Integer>> real = new HashMap<String, Map<String, Integer>>();
    6.61 +
    6.62 +        JavaSource.create(cpInfo).runUserActionTask(new Task<CompilationController>() {
    6.63 +            public void run(CompilationController parameter) throws Exception {
    6.64 +                real.putAll(FileBasedIndex.get(src.getURL()).findCandidatesWithFrequencies(BulkSearch.getDefault().create(parameter, patterns)));
    6.65 +            }
    6.66 +        }, true);
    6.67 +
    6.68 +        assertEquals(golden, real);
    6.69 +    }
    6.70 +
    6.71  }
    6.72 \ No newline at end of file

     7.1 --- a/api/test/unit/src/org/netbeans/modules/jackpot30/impl/pm/BulkSearchTestPerformer.java	Tue Feb 08 22:08:44 2011 +0100
     7.2 +++ b/api/test/unit/src/org/netbeans/modules/jackpot30/impl/pm/BulkSearchTestPerformer.java	Tue Feb 08 23:13:13 2011 +0100
     7.3 @@ -505,6 +505,25 @@
     7.4          assertTrue(matches);
     7.5      }
     7.6  
     7.7 +    public void testFrequencies() throws Exception {
     7.8 +        String text = "package test; public class Test { public void test1(boolean b) { java.io.File f = null; f.isDirectory(); f.isDirectory(); new javax.swing.ImageIcon(null); } }";
     7.9 +
    7.10 +        prepareTest("test/Test.java", text);
    7.11 +
    7.12 +        ByteArrayOutputStream out = new ByteArrayOutputStream();
    7.13 +        EncodingContext ec = new EncodingContext(out, false);
    7.14 +
    7.15 +        createSearch().encode(info.getCompilationUnit(), ec);
    7.16 +        
    7.17 +        Map<String, Integer> actual = createSearch().matchesWithFrequencies(new ByteArrayInputStream(out.toByteArray()), createSearch().create(info, "$1.isDirectory()", "new ImageIcon($1)"));
    7.18 +        Map<String, Integer> golden = new HashMap<String, Integer>();
    7.19 +
    7.20 +        golden.put("$1.isDirectory()", 2);
    7.21 +        golden.put("new ImageIcon($1)", 1);
    7.22 +
    7.23 +        assertEquals(golden, actual);
    7.24 +    }
    7.25 +
    7.26      public void testPatternEncodingAndIdentifiers() throws Exception {
    7.27          String text = "package test; public class Test { }";
    7.28  

     8.1 --- a/api/test/unit/src/org/netbeans/modules/jackpot30/impl/pm/CopyFinderBasedBulkSearchTest.java	Tue Feb 08 22:08:44 2011 +0100
     8.2 +++ b/api/test/unit/src/org/netbeans/modules/jackpot30/impl/pm/CopyFinderBasedBulkSearchTest.java	Tue Feb 08 23:13:13 2011 +0100
     8.3 @@ -65,6 +65,11 @@
     8.4      }
     8.5  
     8.6      @Override
     8.7 +    public void testFrequencies() throws Exception {
     8.8 +        //XXX: serialization is a prerequisite
     8.9 +    }
    8.10 +
    8.11 +    @Override
    8.12      public void testPatternEncodingAndIdentifiers() throws Exception {
    8.13          //XXX
    8.14      }

     9.1 --- a/api/test/unit/src/org/netbeans/modules/jackpot30/impl/pm/NFABasedBulkSearchTest.java	Tue Feb 08 22:08:44 2011 +0100
     9.2 +++ b/api/test/unit/src/org/netbeans/modules/jackpot30/impl/pm/NFABasedBulkSearchTest.java	Tue Feb 08 23:13:13 2011 +0100
     9.3 @@ -128,6 +128,11 @@
     9.4          public BulkPattern create(Collection<? extends String> code, Collection<? extends Tree> patterns, Collection<? extends AdditionalQueryConstraints> additionalConstraints) {
     9.5              return new NFABasedBulkSearch().create(code, patterns, additionalConstraints);
     9.6          }
     9.7 +
     9.8 +        @Override
     9.9 +        public Map<String, Integer> matchesWithFrequencies(InputStream encoded, BulkPattern pattern) {
    9.10 +            return new NFABasedBulkSearch().matchesWithFrequencies(encoded, pattern);
    9.11 +        }
    9.12      }
    9.13  
    9.14      private static final class IndexImpl extends AbstractLuceneIndex {