Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (revision 1080130) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (working copy) @@ -36,6 +36,7 @@ import org.apache.lucene.benchmark.byTask.stats.TaskStats; import org.apache.lucene.benchmark.byTask.tasks.CountingHighlighterTestTask; import org.apache.lucene.benchmark.byTask.tasks.CountingSearchTestTask; +import org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTask; import org.apache.lucene.collation.CollationKeyAnalyzer; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; @@ -390,8 +391,13 @@ BufferedReader r = new BufferedReader(new FileReader(lineFile)); int numLines = 0; - while(r.readLine() != null) + String line; + while((line = r.readLine()) != null) { + if (numLines==0 && line.startsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR)) { + continue; // do not count the header line as a doc + } numLines++; + } r.close(); assertEquals("did not see the right number of docs; should be " + NUM_TRY_DOCS + " but was " + numLines, NUM_TRY_DOCS, numLines); Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (revision 1080130) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java (working copy) @@ -139,6 +139,8 @@ BufferedReader br = new BufferedReader(new InputStreamReader(in, "utf-8")); try { String line = br.readLine(); + assertHeaderLine(line); + line = br.readLine(); assertNotNull(line); String[] parts = line.split(Character.toString(WriteLineDocTask.SEP)); int numExpParts = expBody == null ? 2 : 3; @@ -153,6 +155,10 @@ br.close(); } } + + private void assertHeaderLine(String line) { + assertTrue("First line should be a header line",line.startsWith(WriteLineDocTask.FIELDS_HEADER_INDICATOR)); + } /* Tests WriteLineDocTask with a bzip2 format. */ public void testBZip2() throws Exception { @@ -237,6 +243,8 @@ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), "utf-8")); try { String line = br.readLine(); + assertHeaderLine(line); + line = br.readLine(); assertNull(line); } finally { br.close(); @@ -269,8 +277,10 @@ Set ids = new HashSet(); BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), "utf-8")); try { + String line = br.readLine(); + assertHeaderLine(line); // header line is written once, no matter how many threads there are for (int i = 0; i < threads.length; i++) { - String line = br.readLine(); + line = br.readLine(); String[] parts = line.split(Character.toString(WriteLineDocTask.SEP)); assertEquals(3, parts.length); // check that all thread names written are the same in the same line Index: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java =================================================================== --- lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java (revision 1080130) +++ lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java (working copy) @@ -20,6 +20,7 @@ import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.util.Properties; @@ -44,24 +45,35 @@ private static final CompressorStreamFactory csFactory = new CompressorStreamFactory(); - private void createBZ2LineFile(File file) throws Exception { + private void createBZ2LineFile(File file, boolean addHeader) throws Exception { OutputStream out = new FileOutputStream(file); out = csFactory.createCompressorOutputStream("bzip2", out); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, "utf-8")); + writeDocsToFile(writer, addHeader); + writer.close(); + } + + private void writeDocsToFile(BufferedWriter writer, boolean addHeader) throws IOException { + if (addHeader) { + writer.write(WriteLineDocTask.FIELDS_HEADER_INDICATOR); + writer.write(WriteLineDocTask.SEP); + writer.write(DocMaker.TITLE_FIELD); + writer.write(WriteLineDocTask.SEP); + writer.write(DocMaker.DATE_FIELD); + writer.write(WriteLineDocTask.SEP); + writer.write(DocMaker.BODY_FIELD); + writer.newLine(); + } StringBuilder doc = new StringBuilder(); doc.append("title").append(WriteLineDocTask.SEP).append("date").append(WriteLineDocTask.SEP).append("body"); writer.write(doc.toString()); writer.newLine(); - writer.close(); } - private void createRegularLineFile(File file) throws Exception { + private void createRegularLineFile(File file, boolean addHeader) throws Exception { OutputStream out = new FileOutputStream(file); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, "utf-8")); - StringBuilder doc = new StringBuilder(); - doc.append("title").append(WriteLineDocTask.SEP).append("date").append(WriteLineDocTask.SEP).append("body"); - writer.write(doc.toString()); - writer.newLine(); + writeDocsToFile(writer, addHeader); writer.close(); } @@ -101,22 +113,34 @@ /* Tests LineDocSource with a bzip2 input stream. */ public void testBZip2() throws Exception { File file = new File(getWorkDir(), "one-line.bz2"); - createBZ2LineFile(file); + createBZ2LineFile(file,true); doIndexAndSearchTest(file, true, "true"); } + + public void testBZip2NoHeaderLine() throws Exception { + File file = new File(getWorkDir(), "one-line.bz2"); + createBZ2LineFile(file,false); + doIndexAndSearchTest(file, true, "true"); + } public void testBZip2AutoDetect() throws Exception { File file = new File(getWorkDir(), "one-line.bz2"); - createBZ2LineFile(file); + createBZ2LineFile(file,false); doIndexAndSearchTest(file, false, null); } public void testRegularFile() throws Exception { File file = new File(getWorkDir(), "one-line"); - createRegularLineFile(file); + createRegularLineFile(file,true); doIndexAndSearchTest(file, false, null); } + public void testRegularFileNoHeaderLine() throws Exception { + File file = new File(getWorkDir(), "one-line"); + createRegularLineFile(file,false); + doIndexAndSearchTest(file, false, null); + } + public void testInvalidFormat() throws Exception { String[] testCases = new String[] { "", // empty line Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (revision 1080130) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java (working copy) @@ -53,14 +53,44 @@ */ public class WriteLineDocTask extends PerfTask { + public static final String FIELDS_HEADER_INDICATOR = "FIELDS_HEADER_INDICATOR###"; + public final static char SEP = '\t'; + // fields to be written by default + public static final String[] DEFAULT_FIELDS = new String[] { + DocMaker.TITLE_FIELD, + DocMaker.DATE_FIELD, + DocMaker.BODY_FIELD, + }; + private int docSize = 0; private PrintWriter lineFileOut = null; private DocMaker docMaker; private ThreadLocal threadBuffer = new ThreadLocal(); private ThreadLocal threadNormalizer = new ThreadLocal(); + private final String[] fieldsToWrite = fields(); + /** + * Fields that should be written by this task, in required order. + * Override this to write more/less/other fields. + */ + public String[] fields() { + return DEFAULT_FIELDS; + } + + /** + * Check if specified field is sufficient justification to write a line for the input document. + * Default implementation requires non empty fields 0 or 2 (body or title). + * Override this method to require other fields to be non empty. + * For example, to always emit a line also for empty docs, always return true. + * @param idx index of the checked field (as in {@link #fields()}) + * @param text text of the checked field + */ + protected boolean sufficientFields(int idx, String text) { + return (idx==0 || idx==2) && text.length()>0; // by default require non empty body or title + } + public WriteLineDocTask(PerfRunData runData) throws Exception { super(runData); Config config = runData.getConfig(); @@ -89,8 +119,27 @@ } lineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, "UTF-8"), 1 << 16)); docMaker = runData.getDocMaker(); + + writeHeader(); } + /** + * Write a header to the lines file - indicating how to read the file later + */ + private void writeHeader() { + StringBuilder sb = threadBuffer.get(); + if (sb == null) { + sb = new StringBuilder(); + threadBuffer.set(sb); + } + sb.setLength(0); + sb.append(FIELDS_HEADER_INDICATOR); + for (String f : fieldsToWrite) { + sb.append(SEP).append(f); + } + lineFileOut.println(sb.toString()); + } + @Override protected String getLogMessage(int recsCount) { return "Wrote " + recsCount + " line docs"; @@ -106,27 +155,26 @@ threadNormalizer.set(matcher); } - Field f = doc.getField(DocMaker.BODY_FIELD); - String body = f != null ? matcher.reset(f.stringValue()).replaceAll(" ") : ""; - - f = doc.getField(DocMaker.TITLE_FIELD); - String title = f != null ? matcher.reset(f.stringValue()).replaceAll(" ") : ""; - - if (body.length() > 0 || title.length() > 0) { - - f = doc.getField(DocMaker.DATE_FIELD); - String date = f != null ? matcher.reset(f.stringValue()).replaceAll(" ") : ""; - - StringBuilder sb = threadBuffer.get(); - if (sb == null) { - sb = new StringBuilder(); - threadBuffer.set(sb); - } - sb.setLength(0); - sb.append(title).append(SEP).append(date).append(SEP).append(body); + StringBuilder sb = threadBuffer.get(); + if (sb == null) { + sb = new StringBuilder(); + threadBuffer.set(sb); + } + sb.setLength(0); + + boolean sufficient = false; + for (int i=0; i=0) { + throw new RuntimeException("line: [" + line + "] is in an invalid format (too many separators)!"); + } + // last one + docData.setBody(line.substring(k1)); + } + } + + /** Field names and their order defined by header line */ + public static class HeaderDocDataFiller implements DocDataFiller { + private final int[] posToM; + private final String[] header; + public HeaderDocDataFiller(String[] header) { + this.header = header; + posToM = new int[header.length]; + for (int i=0; i= 0) { + if (n>=header.length) { + throw new RuntimeException("input line has invalid format: "+(n+1)+" fields instead of "+header.length+" :: [" + line + "]"); + } + fillByPositon(docData, n, line.substring(k1,k2)); + ++n; + k1 = k2 + 1; + } + if (n!=header.length-1) { + throw new RuntimeException("input line has invalid format: "+(n+1)+" fields instead of "+header.length+" :: [" + line + "]"); + } + // last one + fillByPositon(docData, n, line.substring(k1)); + } + private void fillByPositon(DocData docData, int position, String text) { + switch(posToM[position]) { + case 1: + fillName1(docData, text); + break; + case 2: + fillTitle2(docData, text); + break; + case 3: + fillDate3(docData, text); + break; + case 4: + fillBody4(docData, text); + break; + default: + Properties p = docData.getProps(); + if (p==null) { + p = new Properties(); + docData.setProps(p); + } + p.setProperty(header[position], text); + } + } + + private void fillName1( DocData docData, String name ) { docData.setName(name); } + private void fillTitle2(DocData docData, String title) { docData.setTitle(title); } + private void fillDate3( DocData docData, String date ) { docData.setDate(date); } + private void fillBody4( DocData docData, String body ) { docData.setBody(body); } + } + private File file; private BufferedReader reader; private int readCount; + private DocDataFiller filler = null; + private boolean skipHeaderLine = false; + private synchronized void openFile() { try { if (reader != null) { @@ -57,6 +162,9 @@ } InputStream is = getInputStream(file); reader = new BufferedReader(new InputStreamReader(is, encoding), BUFFER_SIZE); + if (skipHeaderLine) { + reader.readLine(); // skip one line - the header line - already handled that info + } } catch (IOException e) { throw new RuntimeException(e); } @@ -77,7 +185,6 @@ synchronized(this) { line = reader.readLine(); - myID = readCount++; if (line == null) { if (!forever) { throw new NoMoreDataException(); @@ -86,27 +193,43 @@ openFile(); return getNextDocData(docData); } + if (filler == null) { + // first line ever, one time initialization, + String headIndicator = WriteLineDocTask.FIELDS_HEADER_INDICATOR + WriteLineDocTask.SEP; + if (line.startsWith(headIndicator)) { + String[] header = line.substring(headIndicator.length()).split(Character.toString(WriteLineDocTask.SEP)); + filler = createDocDataFiller(header); + skipHeaderLine = true; // mark to skip the header line when input file is reopened + return getNextDocData(docData); + } else { + filler = createDocDataFiller(); + } + } + // increment IDS only once... + myID = readCount++; } - // A line must be in the following format. If it's not, fail ! - // title date body - int spot = line.indexOf(SEP); - if (spot == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } - int spot2 = line.indexOf(SEP, 1 + spot); - if (spot2 == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } // The date String was written in the format of DateTools.dateToString. docData.clear(); docData.setID(myID); - docData.setBody(line.substring(1 + spot2, line.length())); - docData.setTitle(line.substring(0, spot)); - docData.setDate(line.substring(1 + spot, spot2)); + filler.fillDocData(docData, line); return docData; } + /** + * Create a {@link DocDataFiller} for an input file without a header line + */ + protected DocDataFiller createDocDataFiller() { + return new SimpleDocDataFiller(); + } + + /** + * Create a {@link DocDataFiller} for an input file with a header line + */ + protected DocDataFiller createDocDataFiller(String[] header) { + return new HeaderDocDataFiller(header); + } + @Override public void resetInputs() throws IOException { super.resetInputs();