Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java (revision 1065651) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StringBuilderReader.java (working copy) @@ -158,8 +158,10 @@ synchronized (lock) { this.sb = sb; length = sb.length(); + next = mark = 0; } } + @Override public long skip(long ns) throws IOException { synchronized (lock) { Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java (revision 0) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFTParser.java (revision 0) @@ -0,0 +1,72 @@ +package org.apache.lucene.benchmark.byTask.feeds; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Date; + +import org.apache.lucene.benchmark.byTask.feeds.TrecContentSource.TrecDocParser; +import org.apache.lucene.benchmark.byTask.feeds.TrecParserByPath.ParsePathType; + +/** + * Parser for the FT docs in trec disks 4+5 collection format + */ +public class TrecFTParser implements TrecDocParser { + + private static final String DATE = ""; + private static final int DATE_LENGTH = DATE.length(); + private static final String DATE_END = ""; + + private static final String HEADLINE = ""; + private static final int HEADLINE_LENGTH = HEADLINE.length(); + private static final String HEADLINE_END = ""; + + public DocData parse(DocData docData, String name, TrecContentSource trecSrc, + StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException { + int mark = 0; // that much is skipped + + // date... + Date date = null; + String title = null; + int d1 = docBuf.indexOf(DATE); + if (d1>=0) { // found date-start + d1 += DATE_LENGTH; + int d2 = docBuf.indexOf(DATE_END,d1); + if (d2>=0) { // found date-end + date = trecSrc.parseDate(docBuf.substring(d1,d2).trim()); + } + } + + // title... + int t1 = docBuf.indexOf(HEADLINE); + if (t1>=0) { // found title-start + t1 += HEADLINE_LENGTH; + int t2 = docBuf.indexOf(HEADLINE_END,t1); + if (t2>=0) { // found title-end + title = docBuf.substring(t1,t2).trim(); + } + } + docData.clear(); + docData.setName(name); + docData.setDate(date); + docData.setTitle(title); + docData.setBody(TrecContentSource.cleanTags(docBuf, mark).toString()); + return docData; + } + +} Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java (revision 0) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecGov2Parser.java (revision 0) @@ -0,0 +1,64 @@ +package org.apache.lucene.benchmark.byTask.feeds; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.util.Date; + +import org.apache.lucene.benchmark.byTask.feeds.TrecContentSource.TrecDocParser; +import org.apache.lucene.benchmark.byTask.feeds.TrecParserByPath.ParsePathType; + +/** + * Parser for the GOV2 collection format + */ +public class TrecGov2Parser implements TrecDocParser { + + private static final String DATE = "Date: "; + private static final int DATE_LENGTH = DATE.length(); + + private static final String DOCHDR = ""; + private static final String TERMINATING_DOCHDR = ""; + private static final int TERMINATING_DOCHDR_LENGTH = TERMINATING_DOCHDR.length(); + + public DocData parse(DocData docData, String name, TrecContentSource trecSrc, + StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException { + // Set up a (per-thread) reused Reader over the read content, reset it to re-read from docBuf + Reader r = trecSrc.getTrecDocReader(docBuf); + + // skip some of the text, optionally set date + Date date = null; + int h1 = docBuf.indexOf(DOCHDR); + if (h1>=0) { + int h2 = docBuf.indexOf(TERMINATING_DOCHDR,h1); + int d1 = docBuf.indexOf(DATE,h1); + if (d1>=0 && d1=0 && d2docs.dir - specifies the directory where the TREC files reside. * Can be set to a relative path if "work.dir" is also specified * (default=trec). + *
  • trec.doc.parser - specifies the {@link TrecDocParser} class to use for + * parsing the TREC documents content (default=TrecGov2Parser). *
  • html.parser - specifies the {@link HTMLParser} class to use for - * parsing the TREC documents content (default=DemoHTMLParser). + * parsing the HTML parts of the TREC documents content (default=DemoHTMLParser). *
  • content.source.encoding - if not specified, ISO-8859-1 is used. *
  • content.source.excludeIteration - if true, do not append iteration number to docname * */ public class TrecContentSource extends ContentSource { + /** Parser for trec doc content, invoked on doc text excluding and + * which are handled in TrecContentSource. Required to be stateless and hence thread safe. */ + public interface TrecDocParser { + + /** + * parse the text prepared in docBuf into a result DocData, + * no synchronization is required. + * @param docData reusable result + * @param name name that should be set to the result + * @param trecSrc callinng trec content source + * @param docBuf text to parse + * @param parsedFile input file from which content was extracted, as some parsers may + * alter their behavior according to the file path. + */ + DocData parse(DocData docData, String name, TrecContentSource trecSrc, + StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException; + } + private static final class DateFormatInfo { DateFormat[] dfs; ParsePosition pos; } - private static final String DATE = "Date: "; - private static final String DOCHDR = ""; - private static final String TERMINATING_DOCHDR = ""; - private static final String DOCNO = ""; - private static final String TERMINATING_DOCNO = ""; - private static final String DOC = ""; - private static final String TERMINATING_DOC = ""; + public static final String DOCNO = ""; + public static final String TERMINATING_DOCNO = ""; + public static final String DOC = ""; + public static final String TERMINATING_DOC = ""; - private static final String NEW_LINE = System.getProperty("line.separator"); + /** separator between lines in the byffer */ + public static final String NEW_LINE = System.getProperty("line.separator"); private static final String DATE_FORMATS [] = { - "EEE, dd MMM yyyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT - "EEE MMM dd kk:mm:ss yyyy z", // Tue Dec 09 16:45:08 2003 EST - "EEE, dd-MMM-':'y kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT - "EEE, dd-MMM-yyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT - "EEE MMM dd kk:mm:ss yyyy", // Tue Dec 09 16:45:08 2003 + "EEE, dd MMM yyyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT + "EEE MMM dd kk:mm:ss yyyy z", // Tue Dec 09 16:45:08 2003 EST + "EEE, dd-MMM-':'y kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT + "EEE, dd-MMM-yyy kk:mm:ss z", // Tue, 09 Dec 2003 22:39:08 GMT + "EEE MMM dd kk:mm:ss yyyy", // Tue Dec 09 16:45:08 2003 + "dd MMM yyyy", // 1 March 1994 + "MMM dd, yyyy", // February 3, 1994 + "yyMMdd", // 910513 + "hhmm z.z.z. MMM dd, yyyy", // 0901 u.t.c. April 28, 1994 }; private ThreadLocal dateFormats = new ThreadLocal(); @@ -83,7 +105,7 @@ private File dataDir = null; private ArrayList inputFiles = new ArrayList(); private int nextFile = 0; - private int rawDocSize; + private int[] rawDocSize = new int[1]; // Use to synchronize threads on reading from the TREC documents. private Object lock = new Object(); @@ -92,7 +114,10 @@ BufferedReader reader; int iteration = 0; HTMLParser htmlParser; + private boolean excludeDocnameIteration; + private TrecDocParser trecDocParser = new TrecGov2Parser(); // default + private ParsePathType currPathType; private DateFormatInfo getDateFormatInfo() { DateFormatInfo dfi = dateFormats.get(); @@ -118,7 +143,7 @@ return sb; } - private Reader getTrecDocReader(StringBuilder docBuffer) { + Reader getTrecDocReader(StringBuilder docBuffer) { StringBuilderReader r = trecDocReader.get(); if (r == null) { r = new StringBuilderReader(docBuffer); @@ -129,10 +154,21 @@ return r; } - // read until finding a line that starts with the specified prefix, or a terminating tag has been found. - private void read(StringBuilder buf, String prefix, boolean collectMatchLine, - boolean collectAll, String terminatingTag) - throws IOException, NoMoreDataException { + HTMLParser getHtmlParser() { + return htmlParser; + } + + /** + * Read until a line starting with the specified lineStart. + * @param buf buffer for collecting the data if so specified/ + * @param lineStart line start to look for, must not be null. + * @param collectMatchLine whether to collect the matching line into buffer. + * @param collectAll whether to collect all lines into buffer. + * @throws IOException + * @throws NoMoreDataException + */ + private void read(StringBuilder buf, String lineStart, + boolean collectMatchLine, boolean collectAll) throws IOException, NoMoreDataException { String sep = ""; while (true) { String line = reader.readLine(); @@ -142,24 +178,16 @@ continue; } - rawDocSize += line.length(); + rawDocSize[0] += line.length(); - if (line.startsWith(prefix)) { + if (lineStart!=null && line.startsWith(lineStart)) { if (collectMatchLine) { buf.append(sep).append(line); sep = NEW_LINE; } - break; + return; } - if (terminatingTag != null && line.startsWith(terminatingTag)) { - // didn't find the prefix that was asked, but the terminating - // tag was found. set the length to 0 to signal no match was - // found. - buf.setLength(0); - break; - } - if (collectAll) { buf.append(sep).append(line); sep = NEW_LINE; @@ -169,7 +197,7 @@ void openNextFile() throws NoMoreDataException, IOException { close(); - int retries = 0; + currPathType = null; while (true) { if (nextFile >= inputFiles.size()) { // exhausted files, start a new round, unless forever set to false. @@ -184,13 +212,13 @@ System.out.println("opening: " + f + " length: " + f.length()); } try { - GZIPInputStream zis = new GZIPInputStream(new FileInputStream(f), BUFFER_SIZE); - reader = new BufferedReader(new InputStreamReader(zis, encoding), BUFFER_SIZE); + InputStream inputStream = getInputStream(f); // support either gzip, bzip2, or regular text file, by extension + reader = new BufferedReader(new InputStreamReader(inputStream, encoding), BUFFER_SIZE); + currPathType = TrecParserByPath.pathType(f); return; } catch (Exception e) { - retries++; - if (retries < 20 && verbose) { - System.out.println("Skipping 'bad' file " + f.getAbsolutePath() + " #retries=" + retries); + if (verbose) { + System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage()); continue; } throw new NoMoreDataException(); @@ -198,7 +226,7 @@ } } - Date parseDate(String dateStr) { + public Date parseDate(String dateStr) { dateStr = dateStr.trim(); DateFormatInfo dfi = getDateFormatInfo(); for (int i = 0; i < dfi.dfs.length; i++) { @@ -237,70 +265,47 @@ @Override public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException { - String dateStr = null, name = null; - Reader r = null; + String name = null; + StringBuilder docBuf = getDocBuffer(); + ParsePathType parsedPathType; + // protect reading from the TREC files by multiple threads. The rest of the - // method, i.e., parsing the content and returning the DocData can run - // unprotected. + // method, i.e., parsing the content and returning the DocData can run unprotected. synchronized (lock) { if (reader == null) { openNextFile(); } - - StringBuilder docBuf = getDocBuffer(); - // 1. skip until doc start + // 1. skip until doc start - required for all TREC formats docBuf.setLength(0); - read(docBuf, DOC, false, false, null); - - // 2. name + read(docBuf, DOC, false, false); + + // save parsedFile for passing trecDataParser after the sync block, in + // case another thread will open another file in between. + parsedPathType = currPathType; + + // 2. name - required for all TREC formats docBuf.setLength(0); - read(docBuf, DOCNO, true, false, null); + read(docBuf, DOCNO, true, false); name = docBuf.substring(DOCNO.length(), docBuf.indexOf(TERMINATING_DOCNO, - DOCNO.length())); - if (!excludeDocnameIteration) + DOCNO.length())).trim(); + + if (!excludeDocnameIteration) { name = name + "_" + iteration; - - // 3. skip until doc header - docBuf.setLength(0); - read(docBuf, DOCHDR, false, false, null); - - boolean findTerminatingDocHdr = false; - - // 4. date - look for the date only until /DOCHDR - docBuf.setLength(0); - read(docBuf, DATE, true, false, TERMINATING_DOCHDR); - if (docBuf.length() != 0) { - // Date found. - dateStr = docBuf.substring(DATE.length()); - findTerminatingDocHdr = true; } - // 5. skip until end of doc header - if (findTerminatingDocHdr) { - docBuf.setLength(0); - read(docBuf, TERMINATING_DOCHDR, false, false, null); - } - - // 6. collect until end of doc + // 3. read all until end of doc docBuf.setLength(0); - read(docBuf, TERMINATING_DOC, false, true, null); + read(docBuf, TERMINATING_DOC, false, true); + } - // 7. Set up a Reader over the read content - r = getTrecDocReader(docBuf); - // Resetting the thread's reader means it will reuse the instance - // allocated as well as re-read from docBuf. - r.reset(); - - // count char length of parsed html text (larger than the plain doc body text). - addBytes(docBuf.length()); - } + // count char length of text to be parsed (may be larger than the resulted plain doc body text). + addBytes(docBuf.length()); // This code segment relies on HtmlParser being thread safe. When we get // here, everything else is already private to that thread, so we're safe. - Date date = dateStr != null ? parseDate(dateStr) : null; try { - docData = htmlParser.parse(docData, name, date, r, null); + docData = trecDocParser.parse(docData, name, this, docBuf, parsedPathType); addDoc(); } catch (InterruptedException ie) { throw new ThreadInterruptedException(ie); @@ -322,28 +327,96 @@ @Override public void setConfig(Config config) { super.setConfig(config); + // dirs File workDir = new File(config.get("work.dir", "work")); String d = config.get("docs.dir", "trec"); dataDir = new File(d); if (!dataDir.isAbsolute()) { dataDir = new File(workDir, d); } + // files collectFiles(dataDir, inputFiles); if (inputFiles.size() == 0) { throw new IllegalArgumentException("No files in dataDir: " + dataDir); } + // trec doc parser try { - String parserClassName = config.get("html.parser", + String trecDocParserClassName = config.get("trec.doc.parser", "org.apache.lucene.benchmark.byTask.feeds.TrecGov2Parser"); + trecDocParser = Class.forName(trecDocParserClassName).asSubclass(TrecDocParser.class).newInstance(); + } catch (Exception e) { + // Should not get here. Throw runtime exception. + throw new RuntimeException(e); + } + // html parser + try { + String htmlParserClassName = config.get("html.parser", "org.apache.lucene.benchmark.byTask.feeds.DemoHTMLParser"); - htmlParser = Class.forName(parserClassName).asSubclass(HTMLParser.class).newInstance(); + htmlParser = Class.forName(htmlParserClassName).asSubclass(HTMLParser.class).newInstance(); } catch (Exception e) { // Should not get here. Throw runtime exception. throw new RuntimeException(e); } + // encoding if (encoding == null) { encoding = "ISO-8859-1"; } + // iteration exclusion in doc name excludeDocnameIteration = config.get("content.source.excludeIteration", false); } + /** utility for trec parsers - extract all text except tags from input buffer */ + static StringBuilder cleanTags(StringBuilder buf, int start) { + StringBuilder res = new StringBuilder(); + int k1; + int k2 = start; + while (k2 >= 0 && (k1 = buf.indexOf("<",k2)) >= 0) { + res.append(buf.subSequence(k2,k1)); + k2 = buf.indexOf(">",k1); + if (k2 >= 0) { + ++k2; + res.append(' '); // one space for the tag + } + } + if (k2 >= 0) { + res.append(buf.substring(k2)); + } + return res; + } + + /** utility for trec parsers - extract all text except tags from input string */ + static StringBuilder cleanTags(String buf, int start) { + StringBuilder res = new StringBuilder(); + int k1; + int k2 = start; + while (k2 >= 0 && (k1 = buf.indexOf("<",k2)) >= 0) { + res.append(buf.subSequence(k2,k1)); + k2 = buf.indexOf(">",k1); + if (k2 >= 0) { + ++k2; + res.append(' '); // one space for the tag + } + } + if (k2 >= 0) { + res.append(buf.substring(k2)); + } + return res; + } + + /** + * utility for trec parsers - extract text within last occurrence of tag start/end + * @param sb buffer from which text is extracted + * @param tagStart start tag + * @param tagEnd end tag + * @return text between the tags or null if tags not found in sb + */ + static String extractTextWithinTags(StringBuilder sb, String tagStart, String tagEnd) { + int k1 = sb.lastIndexOf(tagStart); + if (k1>=0) { + int k2 = sb.lastIndexOf(tagEnd); + if (k2 > k1) { + return sb.substring(k1+tagStart.length(),k2).trim(); + } + } + return null; + } } Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java (revision 0) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecFBISParser.java (revision 0) @@ -0,0 +1,80 @@ +package org.apache.lucene.benchmark.byTask.feeds; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Date; + +import org.apache.lucene.benchmark.byTask.feeds.TrecContentSource.TrecDocParser; +import org.apache.lucene.benchmark.byTask.feeds.TrecParserByPath.ParsePathType; + +/** + * Parser for the FBIS docs in trec disks 4+5 collection format + */ +public class TrecFBISParser implements TrecDocParser { + + private static final String HEADER = "
    "; + private static final String HEADER_END = "
    "; + private static final int HEADER_END_LENGTH = HEADER_END.length(); + + private static final String DATE1 = ""; + private static final int DATE1_LENGTH = DATE1.length(); + private static final String DATE1_END = ""; + + private static final String TI = ""; + private static final int TI_LENGTH = TI.length(); + private static final String TI_END = ""; + + public DocData parse(DocData docData, String name, TrecContentSource trecSrc, + StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException { + int mark = 0; // that much is skipped + // optionally skip some of the text, set date, title + Date date = null; + String title = null; + int h1 = docBuf.indexOf(HEADER); + if (h1>=0) { + int h2 = docBuf.indexOf(HEADER_END,h1); + mark = h2+HEADER_END_LENGTH; + // date... + int d1 = docBuf.indexOf(DATE1,h1); + if (d1>=0 && d1=0 && d2=0 && t1=0 && t2=0) { + int h2 = docBuf.indexOf(TEXT_END,h1); + mark = h1+TEXT_LENGTH; + // date... + int d1 = docBuf.indexOf(DATE,h1); + if (d1>=0 && d1=0 && d2=0 && d1a pathType2parser = new HashMap(); + static { + pathType2parser.put(ParsePathType.GOV2, new TrecGov2Parser()); + pathType2parser.put(ParsePathType.FBIS, new TrecFBISParser()); + pathType2parser.put(ParsePathType.FR94, new TrecFR94Parser()); + pathType2parser.put(ParsePathType.FT, new TrecFTParser()); + pathType2parser.put(ParsePathType.LATIMES, new TrecLATimesParser()); + } + + private static final int MAX_DEPTH = 10; + + /** + * Compute the path type of a file by inspecting name of file and its parents + */ + public static ParsePathType pathType(File f) { + int depth = 0; + while (f != null && ++depth < MAX_DEPTH) { + if (PATH_FBIS.equalsIgnoreCase(f.getName())) { + return ParsePathType.FBIS; + } + if (PATH_GOV2.equalsIgnoreCase(f.getName())) { + return ParsePathType.GOV2; + } + if (PATH_FR94.equalsIgnoreCase(f.getName())) { + return ParsePathType.FR94; + } + if (PATH_FT.equalsIgnoreCase(f.getName())) { + return ParsePathType.FT; + } + if (PATH_LATimes.equalsIgnoreCase(f.getName())) { + return ParsePathType.LATIMES; + } + f = f.getParentFile(); + } + return DEFAULT_PATH_TYPE; + } + + public DocData parse(DocData docData, String name, TrecContentSource trecSrc, + StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException { + return pathType2parser.get(pathType).parse(docData, name, trecSrc, docBuf, pathType); + } + + +} Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java (revision 1065651) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DemoHTMLParser.java (working copy) @@ -29,11 +29,14 @@ */ public class DemoHTMLParser implements org.apache.lucene.benchmark.byTask.feeds.HTMLParser { - public DocData parse(DocData docData, String name, Date date, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException { + public DocData parse(DocData docData, String name, Date date, String title, Reader reader, DateFormat dateFormat) throws IOException, InterruptedException { org.apache.lucene.demo.html.HTMLParser p = new org.apache.lucene.demo.html.HTMLParser(reader); // title - String title = p.getTitle(); + if (title==null) { + title = p.getTitle(); + } + // properties Properties props = p.getMetaTags(); // body Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java (revision 0) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecLATimesParser.java (revision 0) @@ -0,0 +1,80 @@ +package org.apache.lucene.benchmark.byTask.feeds; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Date; + +import org.apache.lucene.benchmark.byTask.feeds.TrecContentSource.TrecDocParser; +import org.apache.lucene.benchmark.byTask.feeds.TrecParserByPath.ParsePathType; + +/** + * Parser for the FT docs in trec disks 4+5 collection format + */ +public class TrecLATimesParser implements TrecDocParser { + + private static final String DATE = ""; + private static final int DATE_LENGTH = DATE.length(); + private static final String DATE_END = ""; + private static final String DATE_NOISE = "day,"; // anything aftre the ',' + + private static final String SUBJECT = ""; + private static final int SUBJECT_LENGTH = SUBJECT.length(); + private static final String SUBJECT_END = ""; + + public DocData parse(DocData docData, String name, TrecContentSource trecSrc, + StringBuilder docBuf, ParsePathType pathType) throws IOException, InterruptedException { + int mark = 0; // that much is skipped + + // date... + Date date = null; + String title = null; + int d1 = docBuf.indexOf(DATE); + if (d1>=0) { // found date-start + d1 += DATE_LENGTH; + int d2 = docBuf.indexOf(DATE_END,d1); + if (d2>=0) { // found date-end + int d2a = docBuf.indexOf(DATE_NOISE,d1); + if (d2a >= 0 && d2a < d2) { + d2 = d2a +3; // we need the "day" part + } + String rawDateStr = docBuf.substring(d1,d2); // possibly with some tags + String dateStr = TrecContentSource.cleanTags(rawDateStr,0).toString(); + date = trecSrc.parseDate(dateStr.trim()); + } + } + + // title... + int t1 = docBuf.indexOf(SUBJECT); + if (t1>=0) { // found title-start + t1 += SUBJECT_LENGTH; + int t2 = docBuf.indexOf(SUBJECT_END,t1); + if (t2>=0) { // found title-end + title = TrecContentSource.cleanTags(docBuf.substring(t1,t2),0).toString().trim(); + } + } + + docData.clear(); + docData.setName(name); + docData.setDate(date); + docData.setTitle(title); + docData.setBody(TrecContentSource.cleanTags(docBuf, mark).toString()); + return docData; + } + +} Index: lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java =================================================================== --- lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java (revision 1065651) +++ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentSource.java (working copy) @@ -56,11 +56,14 @@ public abstract class ContentSource { private static final int BZIP = 0; - private static final int OTHER = 1; + private static final int GZIP = 1; + private static final int OTHER = 2; private static final Map extensionToType = new HashMap(); static { extensionToType.put(".bz2", Integer.valueOf(BZIP)); extensionToType.put(".bzip", Integer.valueOf(BZIP)); + extensionToType.put(".gz", Integer.valueOf(GZIP)); + extensionToType.put(".gzip", Integer.valueOf(GZIP)); } protected static final int BUFFER_SIZE = 1 << 16; // 64K @@ -78,11 +81,13 @@ private CompressorStreamFactory csFactory = new CompressorStreamFactory(); + /** update count of bytes generated by this source */ protected final synchronized void addBytes(long numBytes) { bytesCount += numBytes; totalBytesCount += numBytes; } + /** update count of documents generated by this source */ protected final synchronized void addDoc() { ++docsCount; ++totalDocsCount; @@ -144,6 +149,15 @@ throw ioe; } break; + case GZIP: + try { + is = csFactory.createCompressorInputStream("gz", is); + } catch (CompressorException e) { + IOException ioe = new IOException(e.getMessage()); + ioe.initCause(e); + throw ioe; + } + break; default: // Do nothing, stay with FileInputStream }