Index: src/java/org/apache/nutch/searcher/Query.java =================================================================== --- src/java/org/apache/nutch/searcher/Query.java (revision 476852) +++ src/java/org/apache/nutch/searcher/Query.java (working copy) @@ -17,24 +17,22 @@ package org.apache.nutch.searcher; +import java.io.BufferedReader; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.io.BufferedReader; import java.io.InputStreamReader; +import java.util.ArrayList; import java.util.Arrays; -import java.util.ArrayList; -// Commons Logging imports import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Writable; import org.apache.nutch.analysis.AnalyzerFactory; - import org.apache.nutch.analysis.NutchAnalysis; +import org.apache.nutch.crawl.MapWritable; import org.apache.nutch.util.NutchConfiguration; /** A Nutch query. */ @@ -283,6 +281,10 @@ private ArrayList clauses = new ArrayList(); + + private MapWritable docFreqs = null; + + private int numDocs = 0; private Configuration conf; @@ -357,11 +359,31 @@ clauses.add(new Clause(new Phrase(terms), field, false, true, this.conf)); } } + + public MapWritable getDocFreqs() { + return docFreqs; + } + + public void setDocFreqs(MapWritable docFreqs) { + this.docFreqs = docFreqs; + } + + public int getNumDocs() { + return numDocs; + } + + public void setNumDocs(int totalIndexedDocs) { + this.numDocs = totalIndexedDocs; + } public void write(DataOutput out) throws IOException { out.writeByte(clauses.size()); for (int i = 0; i < clauses.size(); i++) ((Clause)clauses.get(i)).write(out); + out.writeBoolean(docFreqs != null); + if (docFreqs != null) + docFreqs.write(out); + out.writeInt(numDocs); } public static Query read(DataInput in, Configuration conf) throws IOException { @@ -375,6 +397,11 @@ int length = in.readByte(); for (int i = 0; i < length; i++) clauses.add(Clause.read(in, this.conf)); + if (in.readBoolean()) { + docFreqs = new MapWritable(); + docFreqs.readFields(in); + } + numDocs = in.readInt(); } public String toString() { Index: src/java/org/apache/nutch/searcher/NutchBean.java =================================================================== --- src/java/org/apache/nutch/searcher/NutchBean.java (revision 476852) +++ src/java/org/apache/nutch/searcher/NutchBean.java (working copy) @@ -26,10 +26,12 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.io.Closeable; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.conf.*; import org.apache.nutch.parse.*; import org.apache.nutch.indexer.*; import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.crawl.MapWritable; import org.apache.nutch.util.NutchConfiguration; /** @@ -367,7 +369,15 @@ public long getFetchDate(HitDetails hit) throws IOException { return content.getFetchDate(hit); } + + public MapWritable getDocFreqs(Query input) throws IOException { + return searcher.getDocFreqs(input); + } + public IntWritable getNumDocs() throws IOException { + return searcher.getNumDocs(); + } + public void close() throws IOException { if (content != null) { content.close(); } if (searcher != null) { searcher.close(); } Index: src/java/org/apache/nutch/searcher/DistributedSearch.java =================================================================== --- src/java/org/apache/nutch/searcher/DistributedSearch.java (revision 476852) +++ src/java/org/apache/nutch/searcher/DistributedSearch.java (working copy) @@ -28,14 +28,17 @@ import org.apache.nutch.parse.ParseData; import org.apache.nutch.parse.ParseText; import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.crawl.MapWritable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.VersionedProtocol; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.WritableStringPair; /** Implements the search API over IPC connnections. */ public class DistributedSearch { @@ -91,6 +94,8 @@ private boolean[] liveServer; private HashMap segmentToAddress = new HashMap(); + private int numDocs; + private boolean running = true; private Configuration conf; @@ -141,13 +146,18 @@ } private static final Method GET_SEGMENTS; + private static final Method NUM_DOCS; private static final Method SEARCH; private static final Method DETAILS; private static final Method SUMMARY; + private static final Method FREQS; static { try { GET_SEGMENTS = Protocol.class.getMethod ("getSegmentNames", new Class[] {}); + NUM_DOCS = Protocol.class.getMethod + ("getNumDocs", new Class[] {}); + SEARCH = Protocol.class.getMethod ("search", new Class[] { Query.class, Integer.TYPE, String.class, String.class, Boolean.TYPE}); @@ -155,6 +165,8 @@ ("getDetails", new Class[] { Hit.class}); SUMMARY = Protocol.class.getMethod ("getSummary", new Class[] { HitDetails.class, Query.class}); + FREQS = Protocol.class.getMethod + ("getDocFreqs", new Class[] { Query.class}); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } @@ -198,6 +210,14 @@ liveServers++; liveSegments+=segments.length; } + + IntWritable[] docNums = + (IntWritable[])RPC.call(NUM_DOCS, params, defaultAddresses, this.conf); + + numDocs = 0; + for (int i = 0; i < docNums.length; i++) { + numDocs += docNums[i].get(); + } // Now update live server flags. this.liveServer = updatedLiveServer; @@ -236,8 +256,33 @@ k++; } } + + Object[][] params = new Object[liveAddresses.length][1]; + for (int i = 0; i < params.length; i++) { + params[i][0] = query; + } + MapWritable[] docFreqs = + (MapWritable[])RPC.call(FREQS, params, liveAddresses, this.conf); - Object[][] params = new Object[liveAddresses.length][5]; + //combine the results in the first MapWritable + for (int i = 1; i < docFreqs.length; i++) { + Set keys = docFreqs[i].keySet(); + Iterator keyIterator = keys.iterator(); + while (keyIterator.hasNext()) { + WritableStringPair term = (WritableStringPair) keyIterator.next(); + IntWritable prevFreq = (IntWritable) docFreqs[0].get(term); + IntWritable curFreq = (IntWritable) docFreqs[i].get(term); + if (prevFreq != null) { + docFreqs[0].put(term, new IntWritable(prevFreq.get() + curFreq.get())); + } else { + docFreqs[0].put(term, new IntWritable(curFreq.get())); + } + } + } + query.setDocFreqs(docFreqs[0]); + query.setNumDocs(numDocs); + + params = new Object[liveAddresses.length][5]; for (int i = 0; i < params.length; i++) { params[i][0] = query; params[i][1] = new Integer(numHits); @@ -357,6 +402,14 @@ public long getFetchDate(HitDetails hit) throws IOException { return getRemote(hit).getFetchDate(hit); } + + public MapWritable getDocFreqs(Query input) throws IOException { + return null; + } + + public IntWritable getNumDocs() throws IOException { + return null; + } public static void main(String[] args) throws Exception { String usage = "DistributedSearch$Client query ..."; Index: src/java/org/apache/nutch/searcher/IndexSearcher.java =================================================================== --- src/java/org/apache/nutch/searcher/IndexSearcher.java (revision 476852) +++ src/java/org/apache/nutch/searcher/IndexSearcher.java (working copy) @@ -21,12 +21,15 @@ import java.util.ArrayList; import java.util.Enumeration; +import java.util.HashSet; +import java.util.Iterator; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiReader; +import org.apache.lucene.index.Term; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.ScoreDoc; @@ -39,8 +42,11 @@ import org.apache.hadoop.fs.*; import org.apache.hadoop.io.*; import org.apache.hadoop.conf.*; +import org.apache.nutch.crawl.MapWritable; import org.apache.nutch.indexer.*; +import org.apache.nutch.util.WritableStringPair; + /** Implements {@link Searcher} and {@link HitDetailer} for either a single * merged index, or a set of indexes. */ public class IndexSearcher implements Searcher, HitDetailer { @@ -51,6 +57,7 @@ private FileSystem fs; private Configuration conf; private QueryFilters queryFilters; + private IntWritable numDocs; /** Construct given a number of indexes. */ public IndexSearcher(Path[] indexDirs, Configuration conf) throws IOException { @@ -77,6 +84,7 @@ this.luceneSearcher.setSimilarity(new NutchSimilarity()); this.optimizer = new LuceneQueryOptimizer(conf); this.queryFilters = new QueryFilters(conf); + this.numDocs = new IntWritable(this.reader.numDocs()); } private Directory getDirectory(Path file) throws IOException { @@ -127,6 +135,28 @@ results[i] = getDetails(hits[i]); return results; } + + public IntWritable getNumDocs() throws IOException { + return this.numDocs; + } + + public MapWritable getDocFreqs(Query input) throws IOException { + org.apache.lucene.search.BooleanQuery query = + this.queryFilters.filter(input); + HashSet termSet = new HashSet(); + MapWritable docFreqs = new MapWritable(); + + query.extractTerms(termSet); + Iterator termIterator = termSet.iterator(); + while (termIterator.hasNext()) { + Term term = (Term) termIterator.next(); + WritableStringPair pair = new WritableStringPair(term.field(), term.text()); + IntWritable freq = new IntWritable(luceneSearcher.docFreq(term)); + docFreqs.put(pair, freq); + } + + return docFreqs; + } private Hits translateHits(TopDocs topDocs, String dedupField, String sortField) Index: src/java/org/apache/nutch/searcher/Searcher.java =================================================================== --- src/java/org/apache/nutch/searcher/Searcher.java (revision 476852) +++ src/java/org/apache/nutch/searcher/Searcher.java (working copy) @@ -20,6 +20,8 @@ import java.io.IOException; import org.apache.hadoop.io.Closeable; +import org.apache.hadoop.io.IntWritable; +import org.apache.nutch.crawl.MapWritable; /** Service that searches. */ public interface Searcher extends Closeable { @@ -30,4 +32,10 @@ /** Return an HTML-formatted explanation of how a query scored. */ String getExplanation(Query query, Hit hit) throws IOException; + + /** Return the number of documents in an index. */ + IntWritable getNumDocs() throws IOException; + + /** Return a map of pairs for the query. */ + MapWritable getDocFreqs(Query input) throws IOException; } Index: src/java/org/apache/nutch/indexer/NutchSimilarity.java =================================================================== --- src/java/org/apache/nutch/indexer/NutchSimilarity.java (revision 476852) +++ src/java/org/apache/nutch/indexer/NutchSimilarity.java (working copy) @@ -17,11 +17,20 @@ package org.apache.nutch.indexer; +import java.io.IOException; + +import org.apache.lucene.index.Term; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.Searcher; /** Similarity implementatation used by Nutch indexing and search. */ public class NutchSimilarity extends DefaultSimilarity { private static final int MIN_CONTENT_LENGTH = 1000; + + @Override + public float idf(Term term, Searcher searcher) throws IOException { + return 1.0f; + } /** Normalize field by length. Called at index time. */ public float lengthNorm(String fieldName, int numTokens) { Index: src/java/org/apache/nutch/util/WritableStringPair.java =================================================================== --- src/java/org/apache/nutch/util/WritableStringPair.java (revision 0) +++ src/java/org/apache/nutch/util/WritableStringPair.java (revision 0) @@ -0,0 +1,70 @@ +package org.apache.nutch.util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; + +public class WritableStringPair implements Writable { + private String first, second; + + public WritableStringPair() { + set("", ""); + } + + public WritableStringPair(String first, String second) { + set(first, second); + } + + public String getFirst() { + return first; + } + + public void setFirst(String first) { + this.first = first; + } + + public String getSecond() { + return second; + } + + public void setSecond(String second) { + this.second = second; + } + + public void set(String first, String second) { + this.first = first; + this.second = second; + } + + @Override + public int hashCode() { + final int PRIME = 31; + int result = 1; + result = PRIME * result + ((first == null) ? 0 : first.hashCode()); + result = PRIME * result + ((second == null) ? 0 : second.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + + final WritableStringPair other = (WritableStringPair) obj; + return first.equals(other.first) && second.equals(other.second); + } + + public void readFields(DataInput in) throws IOException { + this.first = WritableUtils.readCompressedString(in); + this.second = WritableUtils.readCompressedString(in); + } + + public void write(DataOutput out) throws IOException { + WritableUtils.writeCompressedString(out, first); + WritableUtils.writeCompressedString(out, second); + } +} Index: src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java =================================================================== --- src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java (revision 476852) +++ src/plugin/query-basic/src/java/org/apache/nutch/searcher/basic/BasicQueryFilter.java (working copy) @@ -24,11 +24,16 @@ import org.apache.nutch.analysis.NutchDocumentAnalyzer; import org.apache.nutch.analysis.CommonGrams; +import org.apache.nutch.crawl.MapWritable; +import org.apache.nutch.indexer.NutchSimilarity; import org.apache.nutch.searcher.QueryFilter; import org.apache.nutch.searcher.Query; import org.apache.nutch.searcher.Query.*; +import org.apache.nutch.util.NutchConfiguration; +import org.apache.nutch.util.WritableStringPair; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IntWritable; /** The default query filter. Query terms in the default query field are * expanded to search the url, anchor and content document fields.*/ @@ -48,6 +53,8 @@ { "url", "anchor", "content", "title", "host" }; private float[] FIELD_BOOSTS = new float[5]; + + private static final NutchSimilarity SIMILARITY = new NutchSimilarity(); /** * Set the boost factor for url matches, relative to content and anchor @@ -97,8 +104,8 @@ } out.add(o.isPhrase() - ? exactPhrase(o.getPhrase(), FIELDS[f], FIELD_BOOSTS[f]) - : termQuery(FIELDS[f], o.getTerm(), FIELD_BOOSTS[f]), + ? exactPhrase(input, o.getPhrase(), FIELDS[f], FIELD_BOOSTS[f]) + : termQuery(input, FIELDS[f], o.getTerm(), FIELD_BOOSTS[f]), BooleanClause.Occur.SHOULD); } output.add(out, (c.isProhibited() @@ -144,22 +151,37 @@ private org.apache.lucene.search.Query - termQuery(String field, Term term, float boost) { + termQuery(Query query, String field, Term term, float boost) { TermQuery result = new TermQuery(luceneTerm(field, term)); - result.setBoost(boost); + WritableStringPair pair = new WritableStringPair(field, term.toString()); + MapWritable docFreqs = query.getDocFreqs(); + if (docFreqs != null) { + int docFreq = ((IntWritable) docFreqs.get(pair)).get(); + result.setBoost(boost * SIMILARITY.idf(docFreq, query.getNumDocs())); + } else { + result.setBoost(boost); + } return result; } /** Utility to construct a Lucene exact phrase query for a Nutch phrase. */ private org.apache.lucene.search.Query - exactPhrase(Phrase nutchPhrase, + exactPhrase(Query query, Phrase nutchPhrase, String field, float boost) { Term[] terms = nutchPhrase.getTerms(); PhraseQuery exactPhrase = new PhraseQuery(); + float resultIdf = 0.0f; + MapWritable docFreqs = query.getDocFreqs(); + WritableStringPair pair = new WritableStringPair(); for (int i = 0; i < terms.length; i++) { exactPhrase.add(luceneTerm(field, terms[i])); + if (docFreqs != null) { + pair.set(field, terms[i].toString()); + int docFreq = ((IntWritable) docFreqs.get(pair)).get(); + resultIdf += SIMILARITY.idf(docFreq, query.getNumDocs()); + } } - exactPhrase.setBoost(boost); + exactPhrase.setBoost(boost * ((resultIdf == 0.0) ? 1.0f : resultIdf)); return exactPhrase; }