Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (revision 1080130) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (working copy) @@ -36,6 +36,7 @@ import org.apache.lucene.benchmark.byTask.stats.TaskStats; import org.apache.lucene.benchmark.byTask.tasks.CountingHighlighterTestTask; import org.apache.lucene.benchmark.byTask.tasks.CountingSearchTestTask; +import org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTask; import org.apache.lucene.collation.CollationKeyAnalyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -390,8 +391,13 @@ BufferedReader r = new BufferedReader(new FileReader(lineFile)); int numLines = 0; - while(r.readLine() != null) + String line; + while((line = r.readLine()) != null) { + if (numLines==0 && line.startsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR)) { + continue; // do not count the header line as a doc + } numLines++; + } r.close(); assertEquals("did not see the right number of docs; should be " + NUM_TRY_DOCS + " but was " + numLines, NUM_TRY_DOCS, numLines); Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (revision 1080130) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (working copy) @@ -139,6 +139,8 @@ BufferedReader br = new BufferedReader(new InputStreamReader(in, "utf-8")); try { String line = br.readLine(); + assertHeaderLine(line); + line = br.readLine(); assertNotNull(line); String[] parts = line.split(Character.toString(WriteLineDocTask.SEP)); int numExpParts = expBody == null ? 2 : 3; @@ -153,6 +155,10 @@ br.close(); } } + + private void assertHeaderLine(String line) { + assertTrue("First line should be a header line",line.startsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR)); + } /* Tests WriteLineDocTask with a bzip2 format. */ public void testBZip2() throws Exception { @@ -237,6 +243,8 @@ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), "utf-8")); try { String line = br.readLine(); + assertHeaderLine(line); + line = br.readLine(); assertNull(line); } finally { br.close(); @@ -269,8 +277,10 @@ Set ids = new HashSet(); BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), "utf-8")); try { + String line = br.readLine(); + assertHeaderLine(line); // header line is written once, no matter how many threads there are for (int i = 0; i < threads.length; i++) { - String line = br.readLine(); + line = br.readLine(); String[] parts = line.split(Character.toString(WriteLineDocTask.SEP)); assertEquals(3, parts.length); // check that all thread names written are the same in the same line Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java (revision 1080130) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java (working copy) @@ -20,6 +20,7 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.util.Properties; @@ -44,24 +45,35 @@ private static final CompressorStreamFactory csFactory = new CompressorStreamFactory(); - private void createBZ2LineFile(File file) throws Exception { + private void createBZ2LineFile(File file, boolean addHeader) throws Exception { OutputStream out = new FileOutputStream(file); out = csFactory.createCompressorOutputStream("bzip2", out); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, "utf-8")); + writeDocsToFile(writer, addHeader); + writer.close(); + } + + private void writeDocsToFile(BufferedWriter writer, boolean addHeader) throws IOException { + if (addHeader) { + writer.write(WriteLineDocTask.FIELDS_HEADER_INDICATOR); + writer.write(WriteLineDocTask.SEP); + writer.write(DocMaker.TITLE_FIELD); + writer.write(WriteLineDocTask.SEP); + writer.write(DocMaker.DATE_FIELD); + writer.write(WriteLineDocTask.SEP); + writer.write(DocMaker.BODY_FIELD); + writer.newLine(); + } StringBuilder doc = new StringBuilder(); doc.append("title").append(WriteLineDocTask.SEP).append("date").append(WriteLineDocTask.SEP).append("body"); writer.write(doc.toString()); writer.newLine(); - writer.close(); } - private void createRegularLineFile(File file) throws Exception { + private void createRegularLineFile(File file, boolean addHeader) throws Exception { OutputStream out = new FileOutputStream(file); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, "utf-8")); - StringBuilder doc = new StringBuilder(); - doc.append("title").append(WriteLineDocTask.SEP).append("date").append(WriteLineDocTask.SEP).append("body"); - writer.write(doc.toString()); - writer.newLine(); + writeDocsToFile(writer, addHeader); writer.close(); } @@ -101,22 +113,34 @@ /* Tests LineDocSource with a bzip2 input stream. */ public void testBZip2() throws Exception { File file = new File(getWorkDir(), "one-line.bz2"); - createBZ2LineFile(file); + createBZ2LineFile(file,true); doIndexAndSearchTest(file, true, "true"); } + + public void testBZip2NoHeaderLine() throws Exception { + File file = new File(getWorkDir(), "one-line.bz2"); + createBZ2LineFile(file,false); + doIndexAndSearchTest(file, true, "true"); + } public void testBZip2AutoDetect() throws Exception { File file = new File(getWorkDir(), "one-line.bz2"); - createBZ2LineFile(file); + createBZ2LineFile(file,false); doIndexAndSearchTest(file, false, null); } public void testRegularFile() throws Exception { File file = new File(getWorkDir(), "one-line"); - createRegularLineFile(file); + createRegularLineFile(file,true); doIndexAndSearchTest(file, false, null); } + public void testRegularFileNoHeaderLine() throws Exception { + File file = new File(getWorkDir(), "one-line"); + createRegularLineFile(file,false); + doIndexAndSearchTest(file, false, null); + } + public void testInvalidFormat() throws Exception { String[] testCases = new String[] { "", // empty line Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (revision 1080130) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (working copy) @@ -23,6 +23,8 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.util.Arrays; +import java.util.HashSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,11 +43,17 @@ * to save the IO overhead of opening a file per document to be indexed.
* Supports the following parameters: * * NOTE: this class is not thread-safe and if used by multiple threads the * output is unspecified (as all will write to the same output file in a @@ -53,13 +61,32 @@ */ public class WriteLineDocTask extends PerfTask { + public static final String FIELDS_HEADER_INDICATOR = "FIELDS_HEADER_INDICATOR###"; + public final static char SEP = '\t'; + /** + * Fields to be written by default + */ + public static final String[] DEFAULT_FIELDS = new String[] { + DocMaker.TITLE_FIELD, + DocMaker.DATE_FIELD, + DocMaker.BODY_FIELD, + }; + + /** + * Default fields which at least one of them is required to not skip the doc. + */ + public static final String DEFAULT_SUFFICIENT_FIELDS = DocMaker.TITLE_FIELD +',' + DocMaker.BODY_FIELD; + private int docSize = 0; private PrintWriter lineFileOut = null; private DocMaker docMaker; private ThreadLocal threadBuffer = new ThreadLocal(); private ThreadLocal threadNormalizer = new ThreadLocal(); + private final String[] fieldsToWrite;; + private final boolean[] sufficientFields; + private final boolean checkSufficientFields; public WriteLineDocTask(PerfRunData runData) throws Exception { super(runData); @@ -89,8 +116,46 @@ } lineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, "UTF-8"), 1 << 16)); docMaker = runData.getDocMaker(); + + // init fields + String f2r = config.get("line.fields",null); + fieldsToWrite = f2r!=null ? f2r.split(",") : DEFAULT_FIELDS; + + // init sufficient fields + sufficientFields = new boolean[fieldsToWrite.length]; + String suff = config.get("sufficient.fields",DEFAULT_SUFFICIENT_FIELDS); + if (",".equals(suff)) { + checkSufficientFields = false; + } else { + checkSufficientFields = true; + HashSet sf = new HashSet(Arrays.asList(suff.split(","))); + for (int i=0; i 0 || title.length() > 0) { - - f = doc.getField(DocMaker.DATE_FIELD); - String date = f != null ? matcher.reset(f.stringValue()).replaceAll(" ") : ""; - - StringBuilder sb = threadBuffer.get(); - if (sb == null) { - sb = new StringBuilder(); - threadBuffer.set(sb); - } - sb.setLength(0); - sb.append(title).append(SEP).append(date).append(SEP).append(body); + StringBuilder sb = threadBuffer.get(); + if (sb == null) { + sb = new StringBuilder(); + threadBuffer.set(sb); + } + sb.setLength(0); + + boolean sufficient = !checkSufficientFields; + for (int i=0; i0 && sufficientFields[i]; + } + if (sufficient) { + sb.setLength(sb.length()-1); // remove redundant last separator // lineFileOut is a PrintWriter, which synchronizes internally in println. lineFileOut.println(sb.toString()); } + return 1; } Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocSource.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocSource.java (revision 1080130) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocSource.java (working copy) @@ -22,6 +22,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.Properties; import org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTask; import org.apache.lucene.benchmark.byTask.utils.Config; @@ -40,16 +43,135 @@ *
    *
  • docs.file=<path to the file> *
  • content.source.encoding - default to UTF-8. + *
  • docdata.line.reader - default . *
*/ public class LineDocSource extends ContentSource { - private final static char SEP = WriteLineDocTask.SEP; + /** Reader of a single input line into {@link DocData}. */ + public static abstract class DocDataLineReader { + protected final String[] header; + /** Construct with the header + * @param header header line found in the input file, or null if none + */ + public DocDataLineReader(String[] header) { + this.header = header; + } + /** Fill doc data from input line */ + public abstract void readLine(DocData docData, String line); + } + + /** + * {@link DocDataLineReader} which ignores the header passed to its constructor + * and assumes simply that field names and their order are the same + * as in {@link WriteLineDocTask#DEFAULT_FIELDS} + */ + public static class SimpleDocDataLineReader extends DocDataLineReader { + public SimpleDocDataLineReader(String[] header) { + super(header); + } + public void readLine(DocData docData, String line) { + //System.out.println("SimpleDocDataFiller...."); + int k1 = 0; + int k2 = line.indexOf(WriteLineDocTask.SEP, k1); + if (k2<0) { + throw new RuntimeException("line: [" + line + "] is in an invalid format (missing: separator title::date)!"); + } + docData.setTitle(line.substring(k1,k2)); + k1 = k2+1; + k2 = line.indexOf(WriteLineDocTask.SEP, k1); + if (k2<0) { + throw new RuntimeException("line: [" + line + "] is in an invalid format (missing: separator date::body)!"); + } + docData.setDate(line.substring(k1,k2)); + k1 = k2+1; + k2 = line.indexOf(WriteLineDocTask.SEP, k1); + if (k2>=0) { + throw new RuntimeException("line: [" + line + "] is in an invalid format (too many separators)!"); + } + // last one + docData.setBody(line.substring(k1)); + } + } + + /** + * {@link DocDataLineReader} which sets field names and order by + * the header - any header - of the lines file. + * It is less efficient than {@link SimpleDocDataLineReader} but more powerful. + */ + public static class HeaderDocDataLineReader extends DocDataLineReader { + private enum FieldName { NAME , TITLE , DATE , BODY, PROP } + private final FieldName[] posToF; + public HeaderDocDataLineReader(String[] header) { + super(header); + posToF = new FieldName[header.length]; + for (int i=0; i= 0) { + if (n>=header.length) { + throw new RuntimeException("input line has invalid format: "+(n+1)+" fields instead of "+header.length+" :: [" + line + "]"); + } + sedDocDataField(docData, n, line.substring(k1,k2)); + ++n; + k1 = k2 + 1; + } + if (n!=header.length-1) { + throw new RuntimeException("input line has invalid format: "+(n+1)+" fields instead of "+header.length+" :: [" + line + "]"); + } + // last one + sedDocDataField(docData, n, line.substring(k1)); + } + private void sedDocDataField(DocData docData, int fieldPosition, String text) { + switch(posToF[fieldPosition]) { + case NAME: + docData.setName(text); + break; + case TITLE: + docData.setTitle(text); + break; + case DATE: + docData.setDate(text); + break; + case BODY: + docData.setBody(text); + break; + case PROP: + Properties p = docData.getProps(); + if (p==null) { + p = new Properties(); + docData.setProps(p); + } + p.setProperty(header[fieldPosition], text); + break; + } + } + } + private File file; private BufferedReader reader; private int readCount; + private DocDataLineReader filler = null; + private boolean skipHeaderLine = false; + private synchronized void openFile() { try { if (reader != null) { @@ -57,6 +179,9 @@ } InputStream is = getInputStream(file); reader = new BufferedReader(new InputStreamReader(is, encoding), BUFFER_SIZE); + if (skipHeaderLine) { + reader.readLine(); // skip one line - the header line - already handled that info + } } catch (IOException e) { throw new RuntimeException(e); } @@ -77,7 +202,6 @@ synchronized(this) { line = reader.readLine(); - myID = readCount++; if (line == null) { if (!forever) { throw new NoMoreDataException(); @@ -86,27 +210,54 @@ openFile(); return getNextDocData(docData); } + if (filler == null) { // first line ever, one time initialization, + filler = createDocDataFiller(line); + if (skipHeaderLine) { + return getNextDocData(docData); + } + } + // increment IDS only once... + myID = readCount++; } - // A line must be in the following format. If it's not, fail ! - // title date body - int spot = line.indexOf(SEP); - if (spot == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } - int spot2 = line.indexOf(SEP, 1 + spot); - if (spot2 == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } // The date String was written in the format of DateTools.dateToString. docData.clear(); docData.setID(myID); - docData.setBody(line.substring(1 + spot2, line.length())); - docData.setTitle(line.substring(0, spot)); - docData.setDate(line.substring(1 + spot, spot2)); + filler.readLine(docData, line); return docData; } + private DocDataLineReader createDocDataFiller(String line) { + String[] header; + String headIndicator = WriteLineDocTask.FIELDS_HEADER_INDICATOR + WriteLineDocTask.SEP; + + if (line.startsWith(headIndicator)) { + header = line.substring(headIndicator.length()).split(Character.toString(WriteLineDocTask.SEP)); + skipHeaderLine = true; // mark to skip the header line when input file is reopened + } else { + header = WriteLineDocTask.DEFAULT_FIELDS; + } + + // if a specific DocDataLineReader was configured, must respect it + String docDataLineReaderClassName = getConfig().get("docdata.line.reader", null); + if (docDataLineReaderClassName!=null) { + try { + final Class clazz = + Class.forName(docDataLineReaderClassName).asSubclass(DocDataLineReader.class); + Constructor cnstr = clazz.getConstructor(new Class[]{String[].class}); + return cnstr.newInstance((Object)header); + } catch (Exception e) { + throw new RuntimeException("Failed to instantiate "+docDataLineReaderClassName, e); + } + } + + // if this the simple case, + if (Arrays.deepEquals(header, WriteLineDocTask.DEFAULT_FIELDS)) { + return new SimpleDocDataLineReader(header); + } + return new HeaderDocDataLineReader(header); + } + @Override public void resetInputs() throws IOException { super.resetInputs();