Compress the fulltext index, to make the index slightly smaller and easier to fit into caches
authorJan Lahoda <jlahoda@netbeans.org>
Wed, 06 Jul 2011 11:38:07 +0200
changeset 624627d12658781
parent 623 2b29633e523d
child 625 f7e813947ce3
Compress the fulltext index, to make the index slightly smaller and easier to fit into caches
remoting/server/indexer/source/src/org/netbeans/modules/jackpot30/indexer/source/SourceIndexer.java
remoting/server/tests/testcases/sources/simple.tc/response
remoting/server/web/source.web.api/src/org/netbeans/modules/jackpot30/source/api/API.java
     1.1 --- a/remoting/server/indexer/source/src/org/netbeans/modules/jackpot30/indexer/source/SourceIndexer.java	Wed Jul 06 11:31:54 2011 +0200
     1.2 +++ b/remoting/server/indexer/source/src/org/netbeans/modules/jackpot30/indexer/source/SourceIndexer.java	Wed Jul 06 11:38:07 2011 +0200
     1.3 @@ -48,6 +48,7 @@
     1.4  import java.net.URL;
     1.5  import java.util.logging.Level;
     1.6  import java.util.logging.Logger;
     1.7 +import org.apache.lucene.document.CompressionTools;
     1.8  import org.apache.lucene.document.Document;
     1.9  import org.apache.lucene.document.Field;
    1.10  import org.apache.lucene.document.Field.Index;
    1.11 @@ -87,7 +88,7 @@
    1.12                  Document doc = new Document();
    1.13  
    1.14                  doc.add(new Field("relativePath", relPath, Store.YES, Index.NOT_ANALYZED));
    1.15 -                doc.add(new Field(KEY_CONTENT, readFully(i.getURL()), Store.YES, Index.NO));
    1.16 +                doc.add(new Field(KEY_CONTENT, CompressionTools.compressString(readFully(i.getURL())), Store.YES));
    1.17  
    1.18                  IndexAccessor.getCurrent().getIndexWriter().addDocument(doc);
    1.19              }
     2.1 --- a/remoting/server/tests/testcases/sources/simple.tc/response	Wed Jul 06 11:31:54 2011 +0200
     2.2 +++ b/remoting/server/tests/testcases/sources/simple.tc/response	Wed Jul 06 11:38:07 2011 +0200
     2.3 @@ -1,1 +1,8 @@
     2.4 -prj1/src/project1/A.java
     2.5 +package project1;
     2.6 +
     2.7 +public class A {
     2.8 +
     2.9 +    {
    2.10 +        B.test();
    2.11 +    }
    2.12 +}
     3.1 --- a/remoting/server/web/source.web.api/src/org/netbeans/modules/jackpot30/source/api/API.java	Wed Jul 06 11:31:54 2011 +0200
     3.2 +++ b/remoting/server/web/source.web.api/src/org/netbeans/modules/jackpot30/source/api/API.java	Wed Jul 06 11:38:07 2011 +0200
     3.3 @@ -45,10 +45,14 @@
     3.4  import java.util.ArrayList;
     3.5  import java.util.List;
     3.6  import java.util.concurrent.atomic.AtomicBoolean;
     3.7 +import java.util.logging.Level;
     3.8 +import java.util.logging.Logger;
     3.9 +import java.util.zip.DataFormatException;
    3.10  import javax.ws.rs.GET;
    3.11  import javax.ws.rs.Path;
    3.12  import javax.ws.rs.Produces;
    3.13  import javax.ws.rs.QueryParam;
    3.14 +import org.apache.lucene.document.CompressionTools;
    3.15  import org.apache.lucene.document.Document;
    3.16  import org.apache.lucene.search.Query;
    3.17  import org.netbeans.modules.jackpot30.backend.base.CategoryStorage;
    3.18 @@ -83,7 +87,12 @@
    3.19  
    3.20      private static class ConvertorImpl implements Convertor<Document, String> {
    3.21          @Override public String convert(Document p) {
    3.22 -            return p.get(KEY_CONTENT);
    3.23 +            try {
    3.24 +                return CompressionTools.decompressString(p.getBinaryValue(KEY_CONTENT));
    3.25 +            } catch (DataFormatException ex) {
    3.26 +                Logger.getLogger(API.class.getName()).log(Level.SEVERE, null, ex);
    3.27 +                return "";
    3.28 +            }
    3.29          }
    3.30      }
    3.31  }