Index: src/org/apache/lucene/luke/core/HighFreqTerms.java =================================================================== --- src/org/apache/lucene/luke/core/HighFreqTerms.java (revision 1655665) +++ src/org/apache/lucene/luke/core/HighFreqTerms.java (working copy) @@ -100,10 +100,10 @@ } /** - * - * @param reader - * @param numTerms - * @param field + * // TODO move this method to org.apache.lucene.misc.HighFreqTerms + * @param reader IndexReader + * @param numTerms the max number of terms + * @param fieldNames tye array of field names * @return TermStats[] ordered by terms with highest docFreq first. * @throws Exception */ Index: src/org/apache/lucene/luke/core/IndexInfo.java =================================================================== --- src/org/apache/lucene/luke/core/IndexInfo.java (revision 1655665) +++ src/org/apache/lucene/luke/core/IndexInfo.java (working copy) @@ -177,6 +177,7 @@ } } } + /** * @return the reader Index: src/org/apache/lucene/luke/core/TableComparator.java =================================================================== --- src/org/apache/lucene/luke/core/TableComparator.java (revision 1655665) +++ src/org/apache/lucene/luke/core/TableComparator.java (working copy) @@ -1,63 +0,0 @@ -package org.apache.lucene.luke.core; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in - * compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Comparator; - -import org.apache.pivot.collections.Dictionary; -import org.apache.pivot.collections.Map; -import org.apache.pivot.wtk.SortDirection; -import org.apache.pivot.wtk.TableView; - -public class TableComparator implements Comparator> { - private TableView tableView; - - public TableComparator(TableView fieldsTable) { - if (fieldsTable == null) { - throw new IllegalArgumentException(); - } - - this.tableView = fieldsTable; - } - - @Override - public int compare(Map o1, Map o2) { - Dictionary.Pair sort = tableView.getSort().get(0); - - int result; - if (sort.key.equals("name")) { - // sort by name - result = o1.get(sort.key).compareTo(o2.get(sort.key)); - } else if (sort.key.equals("termCount")) { - // sort by termCount - Integer c1 = Integer.parseInt(o1.get(sort.key)); - Integer c2 = Integer.parseInt(o2.get(sort.key)); - result = c1.compareTo(c2); - } else { - // other (ignored) - result = 0; - } - //int result = o1.get("name").compareTo(o2.get("name")); - //SortDirection sortDirection = tableView.getSort().get("name"); - SortDirection sortDirection = sort.value; - result *= (sortDirection == SortDirection.DESCENDING ? 1 : -1); - - return result * -1; - } - -} \ No newline at end of file Index: src/org/apache/lucene/luke/core/TermStats.java =================================================================== --- src/org/apache/lucene/luke/core/TermStats.java (revision 1655665) +++ src/org/apache/lucene/luke/core/TermStats.java (working copy) @@ -26,13 +26,15 @@ public long totalTermFreq; TermStats(String field, BytesRef termtext, int df) { - this.termtext = (BytesRef)termtext.clone(); + //this.termtext = (BytesRef)termtext.clone(); + this.termtext = BytesRef.deepCopyOf(termtext); this.field = field; this.docFreq = df; } TermStats(String field, BytesRef termtext, int df, long tf) { - this.termtext = (BytesRef)termtext.clone(); + //this.termtext = (BytesRef)termtext.clone(); + this.termtext = BytesRef.deepCopyOf(termtext); this.field = field; this.docFreq = df; this.totalTermFreq = tf; Index: src/org/apache/lucene/luke/core/Util.java =================================================================== --- src/org/apache/lucene/luke/core/Util.java (revision 1655665) +++ src/org/apache/lucene/luke/core/Util.java (working copy) @@ -19,11 +19,19 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import org.apache.lucene.document.DateTools.Resolution; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.luke.core.decoders.*; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.MMapDirectory; @@ -161,18 +169,20 @@ return sb.toString(); } - public static String fieldFlags(IndexableField f) { - if (f == null) { - return "-----------"; - } + public static String fieldFlags(IndexableField f, FieldInfo info) { + //if (f == null) { + // return "-----------"; + //} StringBuffer flags = new StringBuffer(); - if (f != null && f.fieldType().indexed()) flags.append("I"); + //if (f != null && f.fieldType().indexed()) flags.append("I"); + if (info != null && info.isIndexed()) flags.append("I"); else flags.append("-"); if (f != null && f.fieldType().tokenized()) flags.append("T"); else flags.append("-"); if (f != null && f.fieldType().stored()) flags.append("S"); else flags.append("-"); - if (f != null && f.fieldType().storeTermVectors()) flags.append("V"); + //if (f != null && f.fieldType().storeTermVectors()) flags.append("V"); + if (info != null && info.hasVectors()) flags.append("V"); else flags.append("-"); if (f != null && f.fieldType().storeTermVectorOffsets()) flags.append("o"); else flags.append("-"); @@ -180,9 +190,13 @@ else flags.append("-"); if (f != null && f.fieldType().storeTermVectorPayloads()) flags.append("a"); else flags.append("-"); - IndexOptions opts = f.fieldType().indexOptions(); + if (info != null && info.hasPayloads()) flags.append("P"); + else flags.append("-"); + //IndexOptions opts = f.fieldType().indexOptions(); + IndexOptions opts = info.getIndexOptions(); // TODO: how to handle these codes - if (f.fieldType().indexed() && opts != null) { + //if (f.fieldType().indexed() && opts != null) { + if (info.isIndexed() && opts != null) { switch (opts) { case DOCS_ONLY: flags.append("1"); @@ -199,13 +213,32 @@ } else { flags.append("-"); } - if (f != null && f.fieldType().omitNorms()) flags.append("O"); + //if (f != null && f.fieldType().omitNorms()) flags.append("O"); + if (info != null && !info.hasNorms()) flags.append("O"); else flags.append("-"); + // TODO lazy + flags.append("-"); + if (f != null && f.binaryValue() != null) flags.append("B"); + else flags.append("-"); return flags.toString(); } - + + public static String docValuesType(FieldInfo info) { + if (info == null || !info.hasDocValues() || info.getDocValuesType() == null) { + return "---"; + } + return info.getDocValuesType().name(); + } + + public static String normType(FieldInfo info) { + if (info == null || !info.hasNorms() || info.getNormType() == null) { + return "---"; + } + return info.getNormType().name(); + } + public static Resolution getResolution(String key) { if (key == null || key.trim().length() == 0) { return Resolution.MILLISECOND; @@ -270,4 +303,56 @@ return String.valueOf(len / 1048576); } } + + public static float decodeNormValue(long v, String fieldName, TFIDFSimilarity sim) throws Exception { + try { + return sim.decodeNormValue(v); + } catch (Exception e) { + throw new Exception("ERROR decoding norm for field " + fieldName + ":" + e.toString()); + } + } + + public static long encodeNormValue(float v, String fieldName, TFIDFSimilarity sim) throws Exception { + try { + return sim.encodeNormValue(v); + } catch (Exception e) { + throw new Exception("ERROR encoding norm for field " + fieldName + ":" + e.toString()); + } + } + + + public static List loadDecoders() { + List decoders = new ArrayList(); + // default decoders + decoders.add(new BinaryDecoder()); + decoders.add(new DateDecoder()); + decoders.add(new NumDoubleDecoder()); + decoders.add(new NumFloatDecoder()); + decoders.add(new NumIntDecoder()); + decoders.add(new NumLongDecoder()); + decoders.add(new StringDecoder()); + + // load external decoders + try { + String extLoaders = System.getProperty("luke.ext.decoder.loader"); + if (extLoaders != null) { + String[] classes = extLoaders.split(","); + for (String className : classes) { + Class clazz = Class.forName(className); + Class[] interfaces = clazz.getInterfaces(); + if (Arrays.asList(interfaces).indexOf(DecoderLoader.class) < 0) { + throw new Exception(className + " is not a DecoderLoader."); + } + DecoderLoader loader = (DecoderLoader)clazz.newInstance(); + List extDecoders = loader.loadDecoders(); + for (Decoder dec : extDecoders) { + decoders.add(dec); + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return decoders; + } } Index: src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/BinaryDecoder.java (working copy) @@ -19,23 +19,18 @@ import org.apache.lucene.document.Field; import org.apache.lucene.luke.core.Util; +import org.apache.lucene.util.BytesRef; public class BinaryDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) throws Exception { - byte[] data; - if (value instanceof byte[]) { - data = (byte[])value; - } else { - data = value.toString().getBytes(); - } - return Util.bytesToHex(data, 0, data.length, false); + public String decodeTerm(String fieldName, BytesRef value) throws Exception { + return Util.bytesToHex(value.bytes, 0, value.length, false); } @Override public String decodeStored(String fieldName, Field value) throws Exception { - return decodeTerm(fieldName, value); + return decodeTerm(fieldName, new BytesRef(value.stringValue())); } public String toString() { Index: src/org/apache/lucene/luke/core/decoders/DateDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/DateDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/DateDecoder.java (working copy) @@ -19,17 +19,18 @@ import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Field; +import org.apache.lucene.util.BytesRef; public class DateDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) throws Exception { + public String decodeTerm(String fieldName, BytesRef value) throws Exception { return DateTools.stringToDate(value.toString()).toString(); } @Override public String decodeStored(String fieldName, Field value) throws Exception { - return decodeTerm(fieldName, value.stringValue()); + return decodeTerm(fieldName, new BytesRef(value.stringValue())); } public String toString() { Index: src/org/apache/lucene/luke/core/decoders/Decoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/Decoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/Decoder.java (working copy) @@ -18,9 +18,10 @@ */ import org.apache.lucene.document.Field; +import org.apache.lucene.util.BytesRef; public interface Decoder { - public String decodeTerm(String fieldName, Object value) throws Exception; + public String decodeTerm(String fieldName, BytesRef value) throws Exception; public String decodeStored(String fieldName, Field value) throws Exception; } Index: src/org/apache/lucene/luke/core/decoders/DecoderLoader.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/DecoderLoader.java (revision 0) +++ src/org/apache/lucene/luke/core/decoders/DecoderLoader.java (working copy) @@ -0,0 +1,24 @@ +package org.apache.lucene.luke.core.decoders; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.List; + +public interface DecoderLoader { + public List loadDecoders(); +} Index: src/org/apache/lucene/luke/core/decoders/NumDoubleDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/NumDoubleDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/NumDoubleDecoder.java (working copy) @@ -1,5 +1,22 @@ package org.apache.lucene.luke.core.decoders; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import org.apache.lucene.document.Field; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; @@ -7,9 +24,8 @@ public class NumDoubleDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) { - BytesRef ref = new BytesRef(value.toString()); - return Double.toString(NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(ref))); + public String decodeTerm(String fieldName, BytesRef value) { + return Double.toString(NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(value))); } @Override @@ -18,7 +34,7 @@ } public String toString() { - return "numeric-double"; + return "numeric double"; } } Index: src/org/apache/lucene/luke/core/decoders/NumFloatDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/NumFloatDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/NumFloatDecoder.java (working copy) @@ -6,9 +6,9 @@ public class NumFloatDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) { - BytesRef ref = new BytesRef(value.toString()); - return Float.toString(NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(ref))); + public String decodeTerm(String fieldName, BytesRef value) { + //BytesRef ref = new BytesRef(value.toString()); + return Float.toString(NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(value))); } @Override @@ -17,7 +17,7 @@ } public String toString() { - return "numeric-float"; + return "numeric float"; } } Index: src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/NumIntDecoder.java (working copy) @@ -24,9 +24,8 @@ public class NumIntDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) { - BytesRef ref = new BytesRef(value.toString()); - return Integer.toString(NumericUtils.prefixCodedToInt(ref)); + public String decodeTerm(String fieldName, BytesRef value) { + return Integer.toString(NumericUtils.prefixCodedToInt(value)); } @Override @@ -35,7 +34,7 @@ } public String toString() { - return "numeric-int"; + return "numeric int"; } } Index: src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/NumLongDecoder.java (working copy) @@ -24,9 +24,8 @@ public class NumLongDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) { - BytesRef ref = new BytesRef(value.toString()); - return Long.toString(NumericUtils.prefixCodedToLong(ref)); + public String decodeTerm(String fieldName, BytesRef value) { + return Long.toString(NumericUtils.prefixCodedToLong(value)); } @Override @@ -35,7 +34,7 @@ } public String toString() { - return "numeric-long"; + return "numeric long"; } } Index: src/org/apache/lucene/luke/core/decoders/SolrDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/SolrDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/SolrDecoder.java (working copy) @@ -24,8 +24,14 @@ import org.apache.lucene.document.Field; import org.apache.lucene.luke.core.ClassFinder; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; import org.apache.solr.schema.FieldType; +/** + * NOT Used. + * The logic here has moved to org.apache.lucene.ext.SolrDecoderLoader. + */ public class SolrDecoder implements Decoder { private static final String solr_prefix = "org.apache.solr.schema."; @@ -86,8 +92,9 @@ name = type; } - public String decodeTerm(String fieldName, Object value) throws Exception { - return fieldType.indexedToReadable(value.toString()); + public String decodeTerm(String fieldName, BytesRef value) throws Exception { + CharsRef chars = fieldType.indexedToReadable(value, new CharsRef()); + return chars.toString(); } public String decodeStored(String fieldName, Field value) @@ -100,3 +107,4 @@ } } + Index: src/org/apache/lucene/luke/core/decoders/StringDecoder.java =================================================================== --- src/org/apache/lucene/luke/core/decoders/StringDecoder.java (revision 1655665) +++ src/org/apache/lucene/luke/core/decoders/StringDecoder.java (working copy) @@ -18,17 +18,18 @@ */ import org.apache.lucene.document.Field; +import org.apache.lucene.util.BytesRef; public class StringDecoder implements Decoder { @Override - public String decodeTerm(String fieldName, Object value) { - return value != null ? value.toString() : "(null)"; + public String decodeTerm(String fieldName, BytesRef value) { + return value != null ? value.utf8ToString() : "(null)"; } @Override public String decodeStored(String fieldName, Field value) { - return decodeTerm(fieldName, value.stringValue()); + return value.stringValue(); } public String toString() { Index: src/org/apache/lucene/luke/ext/SolrDecoderLoader.java =================================================================== --- src/org/apache/lucene/luke/ext/SolrDecoderLoader.java (revision 0) +++ src/org/apache/lucene/luke/ext/SolrDecoderLoader.java (working copy) @@ -0,0 +1,81 @@ +package org.apache.lucene.luke.ext; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.document.Field; +import org.apache.lucene.luke.core.ClassFinder; +import org.apache.lucene.luke.core.decoders.Decoder; +import org.apache.lucene.luke.core.decoders.DecoderLoader; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.solr.schema.FieldType; + +import java.util.ArrayList; +import java.util.List; + +public class SolrDecoderLoader implements DecoderLoader { + private static final String solr_prefix = "org.apache.solr.schema."; + + @Override + public List loadDecoders() { + List decoders = new ArrayList(); + try { + Class[] classes = ClassFinder.getInstantiableSubclasses(FieldType.class); + if (classes == null || classes.length == 0) { + throw new ClassNotFoundException("Missing Solr types???"); + } + for (Class cls : classes) { + FieldType ft = (FieldType) cls.newInstance(); + if (cls.getName().startsWith(solr_prefix)) { + String name = "solr." + cls.getName().substring(solr_prefix.length()); + decoders.add(new SolrDecoder(name, ft)); + } + } + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return decoders; + } +} + +class SolrDecoder implements Decoder { + private String name; + private FieldType fieldType; + + public SolrDecoder(String name, FieldType fieldType) { + this.name = name; + this.fieldType = fieldType; + } + + public String decodeTerm(String fieldName, BytesRef value) throws Exception { + CharsRef chars = fieldType.indexedToReadable(value, new CharsRef()); + return chars.toString(); + } + + public String decodeStored(String fieldName, Field value) + throws Exception { + return fieldType.storedToReadable(value); + } + + public String toString() { + return name; + } + +} + Index: src/org/apache/lucene/luke/ui/DocumentsTab.bxml =================================================================== --- src/org/apache/lucene/luke/ui/DocumentsTab.bxml (revision 1655665) +++ src/org/apache/lucene/luke/ui/DocumentsTab.bxml (working copy) @@ -1,169 +1,259 @@ + xmlns="org.apache.pivot.wtk" xmlns:luke="org.apache.lucene.luke.ui" + orientation="vertical" splitRatio="0.30"> + + + + + + + + + + + + - - - - - - - - + + +