Index: lucene/common-build.xml =================================================================== --- lucene/common-build.xml (revision 1066326) +++ lucene/common-build.xml (working copy) @@ -109,6 +109,7 @@ + @@ -381,7 +382,12 @@ - + + + + + Index: lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -1,124 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.index.Payload; -import org.apache.lucene.util.LuceneTestCase; - -import java.io.IOException; -import java.io.Reader; - -/** - * Analyzer for testing - */ -public final class MockAnalyzer extends Analyzer { - private boolean lowerCase; - private boolean payloads; - public static final int KEYWORD = 0; - public static final int WHITESPACE = 1; - public static final int SIMPLE = 2; - private int tokenizer; - - public MockAnalyzer() { - this(WHITESPACE, true); - } - - public MockAnalyzer(int tokenizer, boolean lowercase) { - this(tokenizer, lowercase, true); - } - - public MockAnalyzer(int tokenizer, boolean lowercase, boolean payloads) { - this.tokenizer = tokenizer; - this.lowerCase = lowercase; - this.payloads = payloads; - } - - @Override - public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result; - if (tokenizer == KEYWORD) - result = new KeywordTokenizer(reader); - else if (tokenizer == SIMPLE) - result = new LetterTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader); - else - result = new WhitespaceTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader); - if (lowerCase) - result = new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, result); - if (payloads) - result = new SimplePayloadFilter(result, fieldName); - return result; - } - - private class SavedStreams { - Tokenizer upstream; - TokenStream filter; - } - - @Override - public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { - SavedStreams saved = (SavedStreams) getPreviousTokenStream(); - if (saved == null){ - saved = new SavedStreams(); - if (tokenizer == KEYWORD) - saved.upstream = new KeywordTokenizer(reader); - else if (tokenizer == SIMPLE) - saved.upstream = new LetterTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader); - else - saved.upstream = new WhitespaceTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader); - saved.filter = saved.upstream; - if (lowerCase) - saved.filter = new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, saved.filter); - if (payloads) - saved.filter = new SimplePayloadFilter(saved.filter, fieldName); - setPreviousTokenStream(saved); - return saved.filter; - } else { - saved.upstream.reset(reader); - saved.filter.reset(); - return saved.filter; - } - } -} - -final class SimplePayloadFilter extends TokenFilter { - String fieldName; - int pos; - final PayloadAttribute payloadAttr; - final CharTermAttribute termAttr; - - public SimplePayloadFilter(TokenStream input, String fieldName) { - super(input); - this.fieldName = fieldName; - pos = 0; - payloadAttr = input.addAttribute(PayloadAttribute.class); - termAttr = input.addAttribute(CharTermAttribute.class); - } - - @Override - public boolean incrementToken() throws IOException { - if (input.incrementToken()) { - payloadAttr.setPayload(new Payload(("pos: " + pos).getBytes())); - pos++; - return true; - } else { - return false; - } - } -} \ No newline at end of file Index: lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -1,219 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.StringReader; -import java.io.IOException; - -import org.apache.lucene.analysis.tokenattributes.*; -import org.apache.lucene.util.Attribute; -import org.apache.lucene.util.AttributeImpl; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Base class for all Lucene unit tests that use TokenStreams. - */ -public abstract class BaseTokenStreamTestCase extends LuceneTestCase { - // some helpers to test Analyzers and TokenStreams: - - public static interface CheckClearAttributesAttribute extends Attribute { - boolean getAndResetClearCalled(); - } - - public static final class CheckClearAttributesAttributeImpl extends AttributeImpl implements CheckClearAttributesAttribute { - private boolean clearCalled = false; - - public boolean getAndResetClearCalled() { - try { - return clearCalled; - } finally { - clearCalled = false; - } - } - - @Override - public void clear() { - clearCalled = true; - } - - @Override - public boolean equals(Object other) { - return ( - other instanceof CheckClearAttributesAttributeImpl && - ((CheckClearAttributesAttributeImpl) other).clearCalled == this.clearCalled - ); - } - - @Override - public int hashCode() { - return 76137213 ^ Boolean.valueOf(clearCalled).hashCode(); - } - - @Override - public void copyTo(AttributeImpl target) { - ((CheckClearAttributesAttributeImpl) target).clear(); - } - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], Integer finalOffset) throws IOException { - assertNotNull(output); - CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class); - - assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class)); - CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class); - - OffsetAttribute offsetAtt = null; - if (startOffsets != null || endOffsets != null || finalOffset != null) { - assertTrue("has no OffsetAttribute", ts.hasAttribute(OffsetAttribute.class)); - offsetAtt = ts.getAttribute(OffsetAttribute.class); - } - - TypeAttribute typeAtt = null; - if (types != null) { - assertTrue("has no TypeAttribute", ts.hasAttribute(TypeAttribute.class)); - typeAtt = ts.getAttribute(TypeAttribute.class); - } - - PositionIncrementAttribute posIncrAtt = null; - if (posIncrements != null) { - assertTrue("has no PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class)); - posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class); - } - - ts.reset(); - for (int i = 0; i < output.length; i++) { - // extra safety to enforce, that the state is not preserved and also assign bogus values - ts.clearAttributes(); - termAtt.setEmpty().append("bogusTerm"); - if (offsetAtt != null) offsetAtt.setOffset(14584724,24683243); - if (typeAtt != null) typeAtt.setType("bogusType"); - if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657); - - checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before - assertTrue("token "+i+" does not exist", ts.incrementToken()); - assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled()); - - assertEquals("term "+i, output[i], termAtt.toString()); - if (startOffsets != null) - assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset()); - if (endOffsets != null) - assertEquals("endOffset "+i, endOffsets[i], offsetAtt.endOffset()); - if (types != null) - assertEquals("type "+i, types[i], typeAtt.type()); - if (posIncrements != null) - assertEquals("posIncrement "+i, posIncrements[i], posIncrAtt.getPositionIncrement()); - } - assertFalse("end of stream", ts.incrementToken()); - ts.end(); - if (finalOffset != null) - assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset()); - ts.close(); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output) throws IOException { - assertTokenStreamContents(ts, output, null, null, null, null, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, String[] types) throws IOException { - assertTokenStreamContents(ts, output, null, null, types, null, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int[] posIncrements) throws IOException { - assertTokenStreamContents(ts, output, null, null, null, posIncrements, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[]) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], Integer finalOffset) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, finalOffset); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements, Integer finalOffset) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, finalOffset); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { - assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length()); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output) throws IOException { - assertAnalyzesTo(a, input, output, null, null, null, null); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, String[] types) throws IOException { - assertAnalyzesTo(a, input, output, null, null, types, null); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int[] posIncrements) throws IOException { - assertAnalyzesTo(a, input, output, null, null, null, posIncrements); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[]) throws IOException { - assertAnalyzesTo(a, input, output, startOffsets, endOffsets, null, null); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { - assertAnalyzesTo(a, input, output, startOffsets, endOffsets, null, posIncrements); - } - - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { - assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length()); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output) throws IOException { - assertAnalyzesToReuse(a, input, output, null, null, null, null); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, String[] types) throws IOException { - assertAnalyzesToReuse(a, input, output, null, null, types, null); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int[] posIncrements) throws IOException { - assertAnalyzesToReuse(a, input, output, null, null, null, posIncrements); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[]) throws IOException { - assertAnalyzesToReuse(a, input, output, startOffsets, endOffsets, null, null); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { - assertAnalyzesToReuse(a, input, output, startOffsets, endOffsets, null, posIncrements); - } - - // simple utility method for testing stemmers - - public static void checkOneTerm(Analyzer a, final String input, final String expected) throws IOException { - assertAnalyzesTo(a, input, new String[]{expected}); - } - - public static void checkOneTermReuse(Analyzer a, final String input, final String expected) throws IOException { - assertAnalyzesToReuse(a, input, new String[]{expected}); - } - -} Index: lucene/src/test/org/apache/lucene/search/CheckHits.java =================================================================== --- lucene/src/test/org/apache/lucene/search/CheckHits.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/search/CheckHits.java (working copy) @@ -1,514 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; -import java.util.TreeSet; -import java.util.Random; - -import junit.framework.Assert; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.store.Directory; - -public class CheckHits { - - /** - * Some explains methods calculate their values though a slightly - * different order of operations from the actual scoring method ... - * this allows for a small amount of variation - */ - public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.0002f; - - /** - * Tests that all documents up to maxDoc which are *not* in the - * expected result set, have an explanation which indicates no match - * (ie: Explanation value of 0.0f) - */ - public static void checkNoMatchExplanations(Query q, String defaultFieldName, - Searcher searcher, int[] results) - throws IOException { - - String d = q.toString(defaultFieldName); - Set ignore = new TreeSet(); - for (int i = 0; i < results.length; i++) { - ignore.add(Integer.valueOf(results[i])); - } - - int maxDoc = searcher.maxDoc(); - for (int doc = 0; doc < maxDoc; doc++) { - if (ignore.contains(Integer.valueOf(doc))) continue; - - Explanation exp = searcher.explain(q, doc); - Assert.assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", - exp); - Assert.assertEquals("Explanation of [["+d+"]] for #"+doc+ - " doesn't indicate non-match: " + exp.toString(), - 0.0f, exp.getValue(), 0.0f); - } - - } - - /** - * Tests that a query matches the an expected set of documents using a - * HitCollector. - * - *

- * Note that when using the HitCollector API, documents will be collected - * if they "match" regardless of what their score is. - *

- * @param query the query to test - * @param searcher the searcher to test the query against - * @param defaultFieldName used for displaying the query in assertion messages - * @param results a list of documentIds that must match the query - * @see Searcher#search(Query,Collector) - * @see #checkHits - */ - public static void checkHitCollector(Random random, Query query, String defaultFieldName, - Searcher searcher, int[] results) - throws IOException { - - QueryUtils.check(random,query,searcher); - - Set correct = new TreeSet(); - for (int i = 0; i < results.length; i++) { - correct.add(Integer.valueOf(results[i])); - } - final Set actual = new TreeSet(); - final Collector c = new SetCollector(actual); - - searcher.search(query, c); - Assert.assertEquals("Simple: " + query.toString(defaultFieldName), - correct, actual); - - for (int i = -1; i < 2; i++) { - actual.clear(); - QueryUtils.wrapSearcher(random, searcher, i).search(query, c); - Assert.assertEquals("Wrap Searcher " + i + ": " + - query.toString(defaultFieldName), - correct, actual); - } - - if ( ! ( searcher instanceof IndexSearcher ) ) return; - - for (int i = -1; i < 2; i++) { - actual.clear(); - QueryUtils.wrapUnderlyingReader - (random, (IndexSearcher)searcher, i).search(query, c); - Assert.assertEquals("Wrap Reader " + i + ": " + - query.toString(defaultFieldName), - correct, actual); - } - } - - public static class SetCollector extends Collector { - final Set bag; - public SetCollector(Set bag) { - this.bag = bag; - } - private int base = 0; - @Override - public void setScorer(Scorer scorer) throws IOException {} - @Override - public void collect(int doc) { - bag.add(Integer.valueOf(doc + base)); - } - @Override - public void setNextReader(IndexReader reader, int docBase) { - base = docBase; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - } - - /** - * Tests that a query matches the an expected set of documents using Hits. - * - *

- * Note that when using the Hits API, documents will only be returned - * if they have a positive normalized score. - *

- * @param query the query to test - * @param searcher the searcher to test the query against - * @param defaultFieldName used for displaing the query in assertion messages - * @param results a list of documentIds that must match the query - * @see Searcher#search(Query, int) - * @see #checkHitCollector - */ - public static void checkHits( - Random random, - Query query, - String defaultFieldName, - Searcher searcher, - int[] results) - throws IOException { - - ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs; - - Set correct = new TreeSet(); - for (int i = 0; i < results.length; i++) { - correct.add(Integer.valueOf(results[i])); - } - - Set actual = new TreeSet(); - for (int i = 0; i < hits.length; i++) { - actual.add(Integer.valueOf(hits[i].doc)); - } - - Assert.assertEquals(query.toString(defaultFieldName), correct, actual); - - QueryUtils.check(random, query,searcher); - } - - /** Tests that a Hits has an expected order of documents */ - public static void checkDocIds(String mes, int[] results, ScoreDoc[] hits) - throws IOException { - Assert.assertEquals(mes + " nr of hits", hits.length, results.length); - for (int i = 0; i < results.length; i++) { - Assert.assertEquals(mes + " doc nrs for hit " + i, results[i], hits[i].doc); - } - } - - /** Tests that two queries have an expected order of documents, - * and that the two queries have the same score values. - */ - public static void checkHitsQuery( - Query query, - ScoreDoc[] hits1, - ScoreDoc[] hits2, - int[] results) - throws IOException { - - checkDocIds("hits1", results, hits1); - checkDocIds("hits2", results, hits2); - checkEqual(query, hits1, hits2); - } - - public static void checkEqual(Query query, ScoreDoc[] hits1, ScoreDoc[] hits2) throws IOException { - final float scoreTolerance = 1.0e-6f; - if (hits1.length != hits2.length) { - Assert.fail("Unequal lengths: hits1="+hits1.length+",hits2="+hits2.length); - } - for (int i = 0; i < hits1.length; i++) { - if (hits1[i].doc != hits2[i].doc) { - Assert.fail("Hit " + i + " docnumbers don't match\n" - + hits2str(hits1, hits2,0,0) - + "for query:" + query.toString()); - } - - if ((hits1[i].doc != hits2[i].doc) - || Math.abs(hits1[i].score - hits2[i].score) > scoreTolerance) - { - Assert.fail("Hit " + i + ", doc nrs " + hits1[i].doc + " and " + hits2[i].doc - + "\nunequal : " + hits1[i].score - + "\n and: " + hits2[i].score - + "\nfor query:" + query.toString()); - } - } - } - - public static String hits2str(ScoreDoc[] hits1, ScoreDoc[] hits2, int start, int end) throws IOException { - StringBuilder sb = new StringBuilder(); - int len1=hits1==null ? 0 : hits1.length; - int len2=hits2==null ? 0 : hits2.length; - if (end<=0) { - end = Math.max(len1,len2); - } - - sb.append("Hits length1=").append(len1).append("\tlength2=").append(len2); - - sb.append('\n'); - for (int i=start; i times others" (where is float). - float x = 0; - String descr = expl.getDescription().toLowerCase(); - boolean productOf = descr.endsWith("product of:"); - boolean sumOf = descr.endsWith("sum of:"); - boolean maxOf = descr.endsWith("max of:"); - boolean maxTimesOthers = false; - if (!(productOf || sumOf || maxOf)) { - // maybe 'max plus x times others' - int k1 = descr.indexOf("max plus "); - if (k1>=0) { - k1 += "max plus ".length(); - int k2 = descr.indexOf(" ",k1); - try { - x = Float.parseFloat(descr.substring(k1,k2).trim()); - if (descr.substring(k2).trim().equals("times others of:")) { - maxTimesOthers = true; - } - } catch (NumberFormatException e) { - } - } - } - Assert.assertTrue( - q+": multi valued explanation description=\""+descr - +"\" must be 'max of plus x times others' or end with 'product of'" - +" or 'sum of:' or 'max of:' - "+expl, - productOf || sumOf || maxOf || maxTimesOthers); - float sum = 0; - float product = 1; - float max = 0; - for (int i=0; i maxDiff - || scorerDiff > maxDiff) { - StringBuilder sbord = new StringBuilder(); - for (int i = 0; i < order.length; i++) - sbord.append(order[i] == skip_op ? " skip()" : " next()"); - throw new RuntimeException("ERROR matching docs:" + "\n\t" - + (doc != scorerDoc ? "--> " : "") + "doc=" + doc + ", scorerDoc=" + scorerDoc - + "\n\t" + (!more ? "--> " : "") + "tscorer.more=" + more - + "\n\t" + (scoreDiff > maxDiff ? "--> " : "") - + "scorerScore=" + scorerScore + " scoreDiff=" + scoreDiff - + " maxDiff=" + maxDiff + "\n\t" - + (scorerDiff > maxDiff ? "--> " : "") + "scorerScore2=" - + scorerScore2 + " scorerDiff=" + scorerDiff - + "\n\thitCollector.doc=" + doc + " score=" + score - + "\n\t Scorer=" + scorer + "\n\t Query=" + q + " " - + q.getClass().getName() + "\n\t Searcher=" + s - + "\n\t Order=" + sbord + "\n\t Op=" - + (op == skip_op ? " skip()" : " next()")); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - if (lastReader[0] != null) { - final IndexReader previousReader = lastReader[0]; - Weight w = q.weight(new IndexSearcher(previousReader)); - Scorer scorer = w.scorer(previousReader, true, false); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - } - this.reader = reader; - this.scorer = null; - lastDoc[0] = -1; - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - - if (lastReader[0] != null) { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - final IndexReader previousReader = lastReader[0]; - Weight w = q.weight(new IndexSearcher(previousReader)); - Scorer scorer = w.scorer(previousReader, true, false); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - } - } - } - - // check that first skip on just created scorers always goes to the right doc - private static void checkFirstSkipTo(final Query q, final IndexSearcher s) throws IOException { - //System.out.println("checkFirstSkipTo: "+q); - final float maxDiff = 1e-3f; - final int lastDoc[] = {-1}; - final IndexReader lastReader[] = {null}; - - s.search(q,new Collector() { - private Scorer scorer; - private IndexReader reader; - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public void collect(int doc) throws IOException { - //System.out.println("doc="+doc); - float score = scorer.score(); - try { - - for (int i=lastDoc[0]+1; i<=doc; i++) { - Weight w = q.weight(s); - Scorer scorer = w.scorer(reader, true, false); - Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS); - Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID()); - float skipToScore = scorer.score(); - Assert.assertEquals("unstable skipTo("+i+") score!",skipToScore,scorer.score(),maxDiff); - Assert.assertEquals("query assigned doc "+doc+" a score of <"+score+"> but skipTo("+i+") has <"+skipToScore+">!",score,skipToScore,maxDiff); - } - lastDoc[0] = doc; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - if (lastReader[0] != null) { - final IndexReader previousReader = lastReader[0]; - Weight w = q.weight(new IndexSearcher(previousReader)); - Scorer scorer = w.scorer(previousReader, true, false); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - } - - this.reader = lastReader[0] = reader; - lastDoc[0] = -1; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return false; - } - }); - - if (lastReader[0] != null) { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - final IndexReader previousReader = lastReader[0]; - Weight w = q.weight(new IndexSearcher(previousReader)); - Scorer scorer = w.scorer(previousReader, true, false); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - } - } -} Index: lucene/src/test/org/apache/lucene/index/MockIndexInput.java =================================================================== --- lucene/src/test/org/apache/lucene/index/MockIndexInput.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/index/MockIndexInput.java (working copy) @@ -1,64 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.store.BufferedIndexInput; - -public class MockIndexInput extends BufferedIndexInput { - private byte[] buffer; - private int pointer = 0; - private long length; - - public MockIndexInput(byte[] bytes) { - buffer = bytes; - length = bytes.length; - } - - @Override - protected void readInternal(byte[] dest, int destOffset, int len) { - int remainder = len; - int start = pointer; - while (remainder != 0) { -// int bufferNumber = start / buffer.length; - int bufferOffset = start % buffer.length; - int bytesInBuffer = buffer.length - bufferOffset; - int bytesToCopy = bytesInBuffer >= remainder ? remainder : bytesInBuffer; - System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy); - destOffset += bytesToCopy; - start += bytesToCopy; - remainder -= bytesToCopy; - } - pointer += len; - } - - @Override - public void close() { - // ignore - } - - @Override - protected void seekInternal(long pos) { - pointer = (int) pos; - } - - @Override - public long length() { - return length; - } - -} Index: lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java (working copy) @@ -1,167 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Closeable; -import java.io.IOException; -import java.io.Reader; -import java.util.Random; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LowerCaseFilter; -import org.apache.lucene.analysis.ReusableAnalyzerBase; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.WhitespaceTokenizer; -import org.apache.lucene.document.Document; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.Version; -import org.apache.lucene.util._TestUtil; - -/** Silly class that randomizes the indexing experience. EG - * it may swap in a different merge policy/scheduler; may - * commit periodically; may or may not optimize in the end, - * may flush by doc count instead of RAM, etc. - */ - -public class RandomIndexWriter implements Closeable { - - public IndexWriter w; - private final Random r; - int docCount; - int flushAt; - private boolean getReaderCalled; - - // Randomly calls Thread.yield so we mixup thread scheduling - private static final class MockIndexWriter extends IndexWriter { - - private final Random r; - - public MockIndexWriter(Random r,Directory dir, IndexWriterConfig conf) throws IOException { - super(dir, conf); - // must make a private random since our methods are - // called from different threads; else test failures may - // not be reproducible from the original seed - this.r = new Random(r.nextInt()); - } - - @Override - boolean testPoint(String name) { - if (r.nextInt(4) == 2) - Thread.yield(); - return true; - } - } - - /** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT and Whitespace+LowercasingAnalyzer */ - public RandomIndexWriter(Random r, Directory dir) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, LuceneTestCase.TEST_VERSION_CURRENT, - new ReusableAnalyzerBase() { - @Override - protected TokenStreamComponents createComponents(String fieldName, - Reader reader) { - Tokenizer tokenizer = new WhitespaceTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader); - return new TokenStreamComponents(tokenizer, - new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, tokenizer)); - } })); - } - - /** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT */ - public RandomIndexWriter(Random r, Directory dir, Analyzer a) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, LuceneTestCase.TEST_VERSION_CURRENT, a)); - } - - /** create a RandomIndexWriter with a random config */ - public RandomIndexWriter(Random r, Directory dir, Version v, Analyzer a) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, v, a)); - } - - /** create a RandomIndexWriter with the provided config */ - public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException { - this.r = r; - w = new MockIndexWriter(r, dir, c); - flushAt = _TestUtil.nextInt(r, 10, 1000); - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW config=" + w.getConfig()); - } - } - - public void addDocument(Document doc) throws IOException { - w.addDocument(doc); - if (docCount++ == flushAt) { - w.commit(); - flushAt += _TestUtil.nextInt(r, 10, 1000); - } - } - - public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { - w.addIndexes(dirs); - } - - public void deleteDocuments(Term term) throws CorruptIndexException, IOException { - w.deleteDocuments(term); - } - - public void commit() throws CorruptIndexException, IOException { - w.commit(); - } - - public int numDocs() throws IOException { - return w.numDocs(); - } - - public int maxDoc() { - return w.maxDoc(); - } - - public void deleteAll() throws IOException { - w.deleteAll(); - } - - public IndexReader getReader() throws IOException { - getReaderCalled = true; - if (r.nextInt(4) == 2) { - w.optimize(); - } - if (r.nextBoolean()) { - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW.getReader: use NRT reader"); - } - return w.getReader(); - } else { - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW.getReader: open new reader"); - } - w.commit(); - return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10)); - } - } - - public void close() throws IOException { - // if someone isn't using getReader() API, we want to be sure to - // maybeOptimize since presumably they might open a reader on the dir. - if (getReaderCalled == false && r.nextInt(4) == 2) { - w.optimize(); - } - w.close(); - } - - public void optimize() throws IOException { - w.optimize(); - } -} Index: lucene/src/test/org/apache/lucene/index/DocHelper.java =================================================================== --- lucene/src/test/org/apache/lucene/index/DocHelper.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/index/DocHelper.java (working copy) @@ -1,249 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.HashMap; -import java.util.Map; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.WhitespaceAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.Fieldable; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.store.Directory; -import static org.apache.lucene.util.LuceneTestCase.TEST_VERSION_CURRENT; - -class DocHelper { - public static final String FIELD_1_TEXT = "field one text"; - public static final String TEXT_FIELD_1_KEY = "textField1"; - public static Field textField1 = new Field(TEXT_FIELD_1_KEY, FIELD_1_TEXT, - Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); - - public static final String FIELD_2_TEXT = "field field field two text"; - //Fields will be lexicographically sorted. So, the order is: field, text, two - public static final int [] FIELD_2_FREQS = {3, 1, 1}; - public static final String TEXT_FIELD_2_KEY = "textField2"; - public static Field textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); - - public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms"; - public static final String TEXT_FIELD_3_KEY = "textField3"; - public static Field textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, Field.Store.YES, Field.Index.ANALYZED); - static { textField3.setOmitNorms(true); } - - public static final String KEYWORD_TEXT = "Keyword"; - public static final String KEYWORD_FIELD_KEY = "keyField"; - public static Field keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT, - Field.Store.YES, Field.Index.NOT_ANALYZED); - - public static final String NO_NORMS_TEXT = "omitNormsText"; - public static final String NO_NORMS_KEY = "omitNorms"; - public static Field noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, - Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - - public static final String NO_TF_TEXT = "analyzed with no tf and positions"; - public static final String NO_TF_KEY = "omitTermFreqAndPositions"; - public static Field noTFField = new Field(NO_TF_KEY, NO_TF_TEXT, - Field.Store.YES, Field.Index.ANALYZED); - static { - noTFField.setOmitTermFreqAndPositions(true); - } - - public static final String UNINDEXED_FIELD_TEXT = "unindexed field text"; - public static final String UNINDEXED_FIELD_KEY = "unIndField"; - public static Field unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, - Field.Store.YES, Field.Index.NO); - - - public static final String UNSTORED_1_FIELD_TEXT = "unstored field text"; - public static final String UNSTORED_FIELD_1_KEY = "unStoredField1"; - public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, - Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO); - - public static final String UNSTORED_2_FIELD_TEXT = "unstored field text"; - public static final String UNSTORED_FIELD_2_KEY = "unStoredField2"; - public static Field unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT, - Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES); - - public static final String LAZY_FIELD_BINARY_KEY = "lazyFieldBinary"; - public static byte [] LAZY_FIELD_BINARY_BYTES; - public static Field lazyFieldBinary; - - public static final String LAZY_FIELD_KEY = "lazyField"; - public static final String LAZY_FIELD_TEXT = "These are some field bytes"; - public static Field lazyField = new Field(LAZY_FIELD_KEY, LAZY_FIELD_TEXT, Field.Store.YES, Field.Index.ANALYZED); - - public static final String LARGE_LAZY_FIELD_KEY = "largeLazyField"; - public static String LARGE_LAZY_FIELD_TEXT; - public static Field largeLazyField; - - //From Issue 509 - public static final String FIELD_UTF1_TEXT = "field one \u4e00text"; - public static final String TEXT_FIELD_UTF1_KEY = "textField1Utf8"; - public static Field textUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT, - Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); - - public static final String FIELD_UTF2_TEXT = "field field field \u4e00two text"; - //Fields will be lexicographically sorted. So, the order is: field, text, two - public static final int [] FIELD_UTF2_FREQS = {3, 1, 1}; - public static final String TEXT_FIELD_UTF2_KEY = "textField2Utf8"; - public static Field textUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT, Field.Store.YES, - Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); - - - - - public static Map nameValues = null; - - // ordered list of all the fields... - // could use LinkedHashMap for this purpose if Java1.4 is OK - public static Field[] fields = new Field[] { - textField1, - textField2, - textField3, - keyField, - noNormsField, - noTFField, - unIndField, - unStoredField1, - unStoredField2, - textUtfField1, - textUtfField2, - lazyField, - lazyFieldBinary,//placeholder for binary field, since this is null. It must be second to last. - largeLazyField//placeholder for large field, since this is null. It must always be last - }; - - public static Map all =new HashMap(); - public static Map indexed =new HashMap(); - public static Map stored =new HashMap(); - public static Map unstored=new HashMap(); - public static Map unindexed=new HashMap(); - public static Map termvector=new HashMap(); - public static Map notermvector=new HashMap(); - public static Map lazy= new HashMap(); - public static Map noNorms=new HashMap(); - public static Map noTf=new HashMap(); - - static { - //Initialize the large Lazy Field - StringBuilder buffer = new StringBuilder(); - for (int i = 0; i < 10000; i++) - { - buffer.append("Lazily loading lengths of language in lieu of laughing "); - } - - try { - LAZY_FIELD_BINARY_BYTES = "These are some binary field bytes".getBytes("UTF8"); - } catch (UnsupportedEncodingException e) { - } - lazyFieldBinary = new Field(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); - fields[fields.length - 2] = lazyFieldBinary; - LARGE_LAZY_FIELD_TEXT = buffer.toString(); - largeLazyField = new Field(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT, Field.Store.YES, Field.Index.ANALYZED); - fields[fields.length - 1] = largeLazyField; - for (int i=0; i map, Fieldable field) { - map.put(field.name(), field); - } - - - static - { - nameValues = new HashMap(); - nameValues.put(TEXT_FIELD_1_KEY, FIELD_1_TEXT); - nameValues.put(TEXT_FIELD_2_KEY, FIELD_2_TEXT); - nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT); - nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT); - nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT); - nameValues.put(NO_TF_KEY, NO_TF_TEXT); - nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT); - nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT); - nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT); - nameValues.put(LAZY_FIELD_KEY, LAZY_FIELD_TEXT); - nameValues.put(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); - nameValues.put(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT); - nameValues.put(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT); - nameValues.put(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT); - } - - /** - * Adds the fields above to a document - * @param doc The document to write - */ - public static void setupDoc(Document doc) { - for (int i=0; i unSyncedFiles; - private Set createdFiles; - Set openFilesForWrite = new HashSet(); - volatile boolean crashed; - - // use this for tracking files for crash. - // additionally: provides debugging information in case you leave one open - Map openFileHandles = Collections.synchronizedMap(new IdentityHashMap()); - - // NOTE: we cannot initialize the Map here due to the - // order in which our constructor actually does this - // member initialization vs when it calls super. It seems - // like super is called, then our members are initialized: - Map openFiles; - - // Only tracked if noDeleteOpenFile is true: if an attempt - // is made to delete an open file, we enroll it here. - Set openFilesDeleted; - - private synchronized void init() { - if (openFiles == null) { - openFiles = new HashMap(); - openFilesDeleted = new HashSet(); - } - - if (createdFiles == null) - createdFiles = new HashSet(); - if (unSyncedFiles == null) - unSyncedFiles = new HashSet(); - } - - public MockDirectoryWrapper(Random random, Directory delegate) { - this.delegate = delegate; - // must make a private random since our methods are - // called from different threads; else test failures may - // not be reproducible from the original seed - this.randomState = new Random(random.nextInt()); - init(); - } - - public void setTrackDiskUsage(boolean v) { - trackDiskUsage = v; - } - - /** If set to true, we throw an IOException if the same - * file is opened by createOutput, ever. */ - public void setPreventDoubleWrite(boolean value) { - preventDoubleWrite = value; - } - - @Deprecated - @Override - public void sync(String name) throws IOException { - maybeYield(); - maybeThrowDeterministicException(); - if (crashed) - throw new IOException("cannot sync after crash"); - unSyncedFiles.remove(name); - delegate.sync(name); - } - - @Override - public synchronized void sync(Collection names) throws IOException { - maybeYield(); - for (String name : names) - maybeThrowDeterministicException(); - if (crashed) - throw new IOException("cannot sync after crash"); - unSyncedFiles.removeAll(names); - delegate.sync(names); - } - - @Override - public String toString() { - maybeYield(); - return "MockDirWrapper(" + delegate + ")"; - } - - public synchronized final long sizeInBytes() throws IOException { - if (delegate instanceof RAMDirectory) - return ((RAMDirectory) delegate).sizeInBytes(); - else { - // hack - long size = 0; - for (String file : delegate.listAll()) - size += delegate.fileLength(file); - return size; - } - } - - /** Simulates a crash of OS or machine by overwriting - * unsynced files. */ - public synchronized void crash() throws IOException { - crashed = true; - openFiles = new HashMap(); - openFilesForWrite = new HashSet(); - openFilesDeleted = new HashSet(); - Iterator it = unSyncedFiles.iterator(); - unSyncedFiles = new HashSet(); - // first force-close all files, so we can corrupt on windows etc. - // clone the file map, as these guys want to remove themselves on close. - Map m = new IdentityHashMap(openFileHandles); - for (Closeable f : m.keySet()) - try { - f.close(); - } catch (Exception ignored) {} - - int count = 0; - while(it.hasNext()) { - String name = it.next(); - if (count % 3 == 0) { - deleteFile(name, true); - } else if (count % 3 == 1) { - // Zero out file entirely - long length = fileLength(name); - byte[] zeroes = new byte[256]; - long upto = 0; - IndexOutput out = delegate.createOutput(name); - while(upto < length) { - final int limit = (int) Math.min(length-upto, zeroes.length); - out.writeBytes(zeroes, 0, limit); - upto += limit; - } - out.close(); - } else if (count % 3 == 2) { - // Truncate the file: - IndexOutput out = delegate.createOutput(name); - out.setLength(fileLength(name)/2); - out.close(); - } - count++; - } - } - - public synchronized void clearCrash() throws IOException { - crashed = false; - } - - public void setMaxSizeInBytes(long maxSize) { - this.maxSize = maxSize; - } - public long getMaxSizeInBytes() { - return this.maxSize; - } - - /** - * Returns the peek actual storage used (bytes) in this - * directory. - */ - public long getMaxUsedSizeInBytes() { - return this.maxUsedSize; - } - public void resetMaxUsedSizeInBytes() throws IOException { - this.maxUsedSize = getRecomputedActualSizeInBytes(); - } - - /** - * Emulate windows whereby deleting an open file is not - * allowed (raise IOException). - */ - public void setNoDeleteOpenFile(boolean value) { - this.noDeleteOpenFile = value; - } - public boolean getNoDeleteOpenFile() { - return noDeleteOpenFile; - } - - /** - * Set whether or not checkindex should be run - * on close - */ - public void setCheckIndexOnClose(boolean value) { - this.checkIndexOnClose = value; - } - - public boolean getCheckIndexOnClose() { - return checkIndexOnClose; - } - /** - * If 0.0, no exceptions will be thrown. Else this should - * be a double 0.0 - 1.0. We will randomly throw an - * IOException on the first write to an OutputStream based - * on this probability. - */ - public void setRandomIOExceptionRate(double rate) { - randomIOExceptionRate = rate; - } - public double getRandomIOExceptionRate() { - return randomIOExceptionRate; - } - - void maybeThrowIOException() throws IOException { - if (randomIOExceptionRate > 0.0) { - int number = Math.abs(randomState.nextInt() % 1000); - if (number < randomIOExceptionRate*1000) { - if (LuceneTestCase.VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception"); - new Throwable().printStackTrace(System.out); - } - throw new IOException("a random IOException"); - } - } - } - - @Override - public synchronized void deleteFile(String name) throws IOException { - maybeYield(); - deleteFile(name, false); - } - - // sets the cause of the incoming ioe to be the stack - // trace when the offending file name was opened - private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) { - for(Map.Entry ent : openFileHandles.entrySet()) { - if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) { - ioe.initCause(ent.getValue()); - break; - } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) { - ioe.initCause(ent.getValue()); - break; - } - } - return ioe; - } - - private void maybeYield() { - if (randomState.nextBoolean()) { - Thread.yield(); - } - } - - private synchronized void deleteFile(String name, boolean forced) throws IOException { - maybeYield(); - - maybeThrowDeterministicException(); - - if (crashed && !forced) - throw new IOException("cannot delete after crash"); - - if (unSyncedFiles.contains(name)) - unSyncedFiles.remove(name); - if (!forced && noDeleteOpenFile) { - if (openFiles.containsKey(name)) { - openFilesDeleted.add(name); - throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); - } else { - openFilesDeleted.remove(name); - } - } - delegate.deleteFile(name); - } - - public synchronized Set getOpenDeletedFiles() { - return new HashSet(openFilesDeleted); - } - - @Override - public synchronized IndexOutput createOutput(String name) throws IOException { - maybeYield(); - if (crashed) - throw new IOException("cannot createOutput after crash"); - init(); - synchronized(this) { - if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) - throw new IOException("file \"" + name + "\" was already written to"); - } - if (noDeleteOpenFile && openFiles.containsKey(name)) - throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); - - if (crashed) - throw new IOException("cannot createOutput after crash"); - unSyncedFiles.add(name); - createdFiles.add(name); - - if (delegate instanceof RAMDirectory) { - RAMDirectory ramdir = (RAMDirectory) delegate; - RAMFile file = new RAMFile(ramdir); - RAMFile existing = ramdir.fileMap.get(name); - - // Enforce write once: - if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) - throw new IOException("file " + name + " already exists"); - else { - if (existing!=null) { - ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes); - existing.directory = null; - } - ramdir.fileMap.put(name, file); - } - } - //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); - IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name); - openFileHandles.put(io, new RuntimeException("unclosed IndexOutput")); - openFilesForWrite.add(name); - return io; - } - - @Override - public synchronized IndexInput openInput(String name) throws IOException { - maybeYield(); - if (!delegate.fileExists(name)) - throw new FileNotFoundException(name); - - // cannot open a file for input if it's still open for - // output, except for segments.gen and segments_N - if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { - throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); - } - - if (openFiles.containsKey(name)) { - Integer v = openFiles.get(name); - v = Integer.valueOf(v.intValue()+1); - openFiles.put(name, v); - } else { - openFiles.put(name, Integer.valueOf(1)); - } - - IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name)); - openFileHandles.put(ii, new RuntimeException("unclosed IndexInput")); - return ii; - } - - /** Provided for testing purposes. Use sizeInBytes() instead. */ - public synchronized final long getRecomputedSizeInBytes() throws IOException { - if (!(delegate instanceof RAMDirectory)) - return sizeInBytes(); - long size = 0; - for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) { - size += file.getSizeInBytes(); - } - return size; - } - - /** Like getRecomputedSizeInBytes(), but, uses actual file - * lengths rather than buffer allocations (which are - * quantized up to nearest - * RAMOutputStream.BUFFER_SIZE (now 1024) bytes. - */ - - public final synchronized long getRecomputedActualSizeInBytes() throws IOException { - if (!(delegate instanceof RAMDirectory)) - return sizeInBytes(); - long size = 0; - for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values()) - size += file.length; - return size; - } - - @Override - public synchronized void close() throws IOException { - maybeYield(); - if (openFiles == null) { - openFiles = new HashMap(); - openFilesDeleted = new HashSet(); - } - if (noDeleteOpenFile && openFiles.size() > 0) { - // print the first one as its very verbose otherwise - Exception cause = null; - Iterator stacktraces = openFileHandles.values().iterator(); - if (stacktraces.hasNext()) - cause = stacktraces.next(); - // RuntimeException instead of IOException because - // super() does not throw IOException currently: - throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause); - } - open = false; - if (checkIndexOnClose && IndexReader.indexExists(this)) { - _TestUtil.checkIndex(this); - } - delegate.close(); - } - - boolean open = true; - - public synchronized boolean isOpen() { - return open; - } - - /** - * Objects that represent fail-able conditions. Objects of a derived - * class are created and registered with the mock directory. After - * register, each object will be invoked once for each first write - * of a file, giving the object a chance to throw an IOException. - */ - public static class Failure { - /** - * eval is called on the first write of every new file. - */ - public void eval(MockDirectoryWrapper dir) throws IOException { } - - /** - * reset should set the state of the failure to its default - * (freshly constructed) state. Reset is convenient for tests - * that want to create one failure object and then reuse it in - * multiple cases. This, combined with the fact that Failure - * subclasses are often anonymous classes makes reset difficult to - * do otherwise. - * - * A typical example of use is - * Failure failure = new Failure() { ... }; - * ... - * mock.failOn(failure.reset()) - */ - public Failure reset() { return this; } - - protected boolean doFail; - - public void setDoFail() { - doFail = true; - } - - public void clearDoFail() { - doFail = false; - } - } - - ArrayList failures; - - /** - * add a Failure object to the list of objects to be evaluated - * at every potential failure point - */ - synchronized public void failOn(Failure fail) { - if (failures == null) { - failures = new ArrayList(); - } - failures.add(fail); - } - - /** - * Iterate through the failures list, giving each object a - * chance to throw an IOE - */ - synchronized void maybeThrowDeterministicException() throws IOException { - if (failures != null) { - for(int i = 0; i < failures.size(); i++) { - failures.get(i).eval(this); - } - } - } - - @Override - public synchronized String[] listAll() throws IOException { - maybeYield(); - return delegate.listAll(); - } - - @Override - public synchronized boolean fileExists(String name) throws IOException { - maybeYield(); - return delegate.fileExists(name); - } - - @Override - public synchronized long fileModified(String name) throws IOException { - maybeYield(); - return delegate.fileModified(name); - } - - @Override - public synchronized void touchFile(String name) throws IOException { - maybeYield(); - delegate.touchFile(name); - } - - @Override - public synchronized long fileLength(String name) throws IOException { - maybeYield(); - return delegate.fileLength(name); - } - - @Override - public synchronized Lock makeLock(String name) { - maybeYield(); - return delegate.makeLock(name); - } - - @Override - public synchronized void clearLock(String name) throws IOException { - maybeYield(); - delegate.clearLock(name); - } - - @Override - public synchronized void setLockFactory(LockFactory lockFactory) throws IOException { - maybeYield(); - delegate.setLockFactory(lockFactory); - } - - @Override - public synchronized LockFactory getLockFactory() { - maybeYield(); - return delegate.getLockFactory(); - } - - @Override - public synchronized String getLockID() { - maybeYield(); - return delegate.getLockID(); - } - - @Override - public synchronized void copy(Directory to, String src, String dest) throws IOException { - maybeYield(); - delegate.copy(to, src, dest); - } -} Index: lucene/src/test/org/apache/lucene/store/MockIndexInputWrapper.java =================================================================== --- lucene/src/test/org/apache/lucene/store/MockIndexInputWrapper.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/store/MockIndexInputWrapper.java (working copy) @@ -1,169 +0,0 @@ -package org.apache.lucene.store; - -import java.io.IOException; -import java.util.Map; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Used by MockDirectoryWrapper to create an input stream that - * keeps track of when it's been closed. - */ - -public class MockIndexInputWrapper extends IndexInput { - private MockDirectoryWrapper dir; - final String name; - private IndexInput delegate; - private boolean isClone; - - /** Construct an empty output buffer. - * @throws IOException */ - public MockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) { - this.name = name; - this.dir = dir; - this.delegate = delegate; - } - - @Override - public void close() throws IOException { - delegate.close(); - // Pending resolution on LUCENE-686 we may want to - // remove the conditional check so we also track that - // all clones get closed: - if (!isClone) { - synchronized(dir) { - Integer v = dir.openFiles.get(name); - // Could be null when MockRAMDirectory.crash() was called - if (v != null) { - if (v.intValue() == 1) { - dir.openFiles.remove(name); - dir.openFilesDeleted.remove(name); - } else { - v = Integer.valueOf(v.intValue()-1); - dir.openFiles.put(name, v); - } - } - dir.openFileHandles.remove(this); - } - } - } - - @Override - public Object clone() { - IndexInput iiclone = (IndexInput) delegate.clone(); - MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, name, iiclone); - clone.isClone = true; - // Pending resolution on LUCENE-686 we may want to - // uncomment this code so that we also track that all - // clones get closed: - /* - synchronized(dir.openFiles) { - if (dir.openFiles.containsKey(name)) { - Integer v = (Integer) dir.openFiles.get(name); - v = Integer.valueOf(v.intValue()+1); - dir.openFiles.put(name, v); - } else { - throw new RuntimeException("BUG: cloned file was not open?"); - } - } - */ - return clone; - } - - @Override - public long getFilePointer() { - return delegate.getFilePointer(); - } - - @Override - public void seek(long pos) throws IOException { - delegate.seek(pos); - } - - @Override - public long length() { - return delegate.length(); - } - - @Override - public byte readByte() throws IOException { - return delegate.readByte(); - } - - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - delegate.readBytes(b, offset, len); - } - - @Override - public void copyBytes(IndexOutput out, long numBytes) throws IOException { - delegate.copyBytes(out, numBytes); - } - - @Override - public void readBytes(byte[] b, int offset, int len, boolean useBuffer) - throws IOException { - delegate.readBytes(b, offset, len, useBuffer); - } - - @Override - public int readInt() throws IOException { - return delegate.readInt(); - } - - @Override - public int readVInt() throws IOException { - return delegate.readVInt(); - } - - @Override - public long readLong() throws IOException { - return delegate.readLong(); - } - - @Override - public long readVLong() throws IOException { - return delegate.readVLong(); - } - - @Override - public String readString() throws IOException { - return delegate.readString(); - } - - @Override - public Map readStringStringMap() throws IOException { - return delegate.readStringStringMap(); - } - - @Override - public void setModifiedUTF8StringsMode() { - delegate.setModifiedUTF8StringsMode(); - } - - @Override - public void readChars(char[] buffer, int start, int length) - throws IOException { - delegate.readChars(buffer, start, length); - } - - @Override - public void skipChars(int length) throws IOException { - delegate.skipChars(length); - } -} Index: lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java =================================================================== --- lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java (working copy) @@ -1,159 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.LuceneTestCase; - -/** - * Used by MockRAMDirectory to create an output stream that - * will throw an IOException on fake disk full, track max - * disk space actually used, and maybe throw random - * IOExceptions. - */ - -public class MockIndexOutputWrapper extends IndexOutput { - private MockDirectoryWrapper dir; - private final IndexOutput delegate; - private boolean first=true; - final String name; - - byte[] singleByte = new byte[1]; - - /** Construct an empty output buffer. */ - public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput delegate, String name) { - this.dir = dir; - this.name = name; - this.delegate = delegate; - } - - @Override - public void close() throws IOException { - dir.maybeThrowDeterministicException(); - delegate.close(); - if (dir.trackDiskUsage) { - // Now compute actual disk usage & track the maxUsedSize - // in the MockDirectoryWrapper: - long size = dir.getRecomputedActualSizeInBytes(); - if (size > dir.maxUsedSize) { - dir.maxUsedSize = size; - } - } - synchronized(dir) { - dir.openFileHandles.remove(this); - dir.openFilesForWrite.remove(name); - } - } - - @Override - public void flush() throws IOException { - dir.maybeThrowDeterministicException(); - delegate.flush(); - } - - @Override - public void writeByte(byte b) throws IOException { - singleByte[0] = b; - writeBytes(singleByte, 0, 1); - } - - @Override - public void writeBytes(byte[] b, int offset, int len) throws IOException { - long freeSpace = dir.maxSize == 0 ? 0 : dir.maxSize - dir.sizeInBytes(); - long realUsage = 0; - - // If MockRAMDir crashed since we were opened, then - // don't write anything: - if (dir.crashed) - throw new IOException("MockRAMDirectory was crashed; cannot write to " + name); - - // Enforce disk full: - if (dir.maxSize != 0 && freeSpace <= len) { - // Compute the real disk free. This will greatly slow - // down our test but makes it more accurate: - realUsage = dir.getRecomputedActualSizeInBytes(); - freeSpace = dir.maxSize - realUsage; - } - - if (dir.maxSize != 0 && freeSpace <= len) { - if (freeSpace > 0) { - realUsage += freeSpace; - delegate.writeBytes(b, offset, (int) freeSpace); - } - if (realUsage > dir.maxUsedSize) { - dir.maxUsedSize = realUsage; - } - String message = "fake disk full at " + dir.getRecomputedActualSizeInBytes() + " bytes when writing " + name + " (file length=" + delegate.length(); - if (freeSpace > 0) { - message += "; wrote " + freeSpace + " of " + len + " bytes"; - } - message += ")"; - if (LuceneTestCase.VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": MDW: now throw fake disk full"); - new Throwable().printStackTrace(System.out); - } - throw new IOException(message); - } else { - if (dir.randomState.nextBoolean()) { - final int half = len/2; - delegate.writeBytes(b, offset, half); - Thread.yield(); - delegate.writeBytes(b, offset+half, len-half); - } else { - delegate.writeBytes(b, offset, len); - } - } - - dir.maybeThrowDeterministicException(); - - if (first) { - // Maybe throw random exception; only do this on first - // write to a new file: - first = false; - dir.maybeThrowIOException(); - } - } - - @Override - public long getFilePointer() { - return delegate.getFilePointer(); - } - - @Override - public void seek(long pos) throws IOException { - delegate.seek(pos); - } - - @Override - public long length() throws IOException { - return delegate.length(); - } - - @Override - public void setLength(long length) throws IOException { - delegate.setLength(length); - } - - @Override - public void copyBytes(IndexInput input, long numBytes) throws IOException { - delegate.copyBytes(input, numBytes); - // TODO: we may need to check disk full here as well - dir.maybeThrowDeterministicException(); - } -} Index: lucene/src/test/org/apache/lucene/store/_TestHelper.java =================================================================== --- lucene/src/test/org/apache/lucene/store/_TestHelper.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/store/_TestHelper.java (working copy) @@ -1,65 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; - -/** This class provides access to package-level features defined in the - * store package. It is used for testing only. - */ -public class _TestHelper { - - /** Returns true if the instance of the provided input stream is actually - * an SimpleFSIndexInput. - */ - public static boolean isSimpleFSIndexInput(IndexInput is) { - return is instanceof SimpleFSIndexInput; - } - - /** Returns true if the provided input stream is an SimpleFSIndexInput and - * is a clone, that is it does not own its underlying file descriptor. - */ - public static boolean isSimpleFSIndexInputClone(IndexInput is) { - if (isSimpleFSIndexInput(is)) { - return ((SimpleFSIndexInput) is).isClone; - } else { - return false; - } - } - - /** Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns - * true if the underlying file descriptor is valid, and false otherwise. - * This can be used to determine if the OS file has been closed. - * The descriptor becomes invalid when the non-clone instance of the - * SimpleFSIndexInput that owns this descriptor is closed. However, the - * descriptor may possibly become invalid in other ways as well. - */ - public static boolean isSimpleFSIndexInputOpen(IndexInput is) - throws IOException - { - if (isSimpleFSIndexInput(is)) { - SimpleFSIndexInput fis = (SimpleFSIndexInput) is; - return fis.isFDValid(); - } else { - return false; - } - } - -} Index: lucene/src/test/org/apache/lucene/util/LineFileDocs.java =================================================================== --- lucene/src/test/org/apache/lucene/util/LineFileDocs.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/util/LineFileDocs.java (working copy) @@ -1,180 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.InputStream; -import java.io.BufferedInputStream; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.GZIPInputStream; -import java.util.Random; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -// Minimal port of contrib/benchmark's LneDocSource + -// DocMaker, so tests can enum docs from a line file created -// by contrib/benchmark's WriteLineDoc task -public class LineFileDocs implements Closeable { - - private BufferedReader reader; - private final static int BUFFER_SIZE = 1 << 16; // 64K - private final AtomicInteger id = new AtomicInteger(); - private final String path; - - // If forever is true, we rewind the file at EOF (repeat - // the docs over and over) - public LineFileDocs(Random random, String path) throws IOException { - this.path = path; - open(random); - } - - public LineFileDocs(Random random) throws IOException { - this(random, LuceneTestCase.TEST_LINE_DOCS_FILE); - } - - public synchronized void close() throws IOException { - if (reader != null) { - reader.close(); - reader = null; - } - } - - private synchronized void open(Random random) throws IOException { - InputStream is = getClass().getResourceAsStream(path); - if (is == null) { - // if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir) - is = new FileInputStream(path); - } - File file = new File(path); - long size; - if (file.exists()) { - size = file.length(); - } else { - size = is.available(); - } - if (path.endsWith(".gz")) { - is = new GZIPInputStream(is); - // guestimate: - size *= 2.8; - } - - final InputStream in = new BufferedInputStream(is, BUFFER_SIZE); - reader = new BufferedReader(new InputStreamReader(in, "UTF-8"), BUFFER_SIZE); - - // Override sizes for currently "known" line files: - if (path.equals("europarl.lines.txt.gz")) { - size = 15129506L; - } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) { - size = 3038178822L; - } - - // Randomly seek to starting point: - if (random != null && size > 3) { - final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3); - if (LuceneTestCase.VERBOSE) { - System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open"); - } - reader.skip(seekTo); - reader.readLine(); - } - } - - public synchronized void reset(Random random) throws IOException { - close(); - open(random); - id.set(0); - } - - private final static char SEP = '\t'; - - private static final class DocState { - final Document doc; - final Field titleTokenized; - final Field title; - final Field body; - final Field id; - final Field date; - - public DocState() { - doc = new Document(); - - title = new Field("title", "", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add(title); - - titleTokenized = new Field("titleTokenized", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); - doc.add(titleTokenized); - - body = new Field("body", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); - doc.add(body); - - id = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add(id); - - date = new Field("date", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); - doc.add(date); - } - } - - private final ThreadLocal threadDocs = new ThreadLocal(); - - // Document instance is re-used per-thread - public Document nextDoc() throws IOException { - String line; - synchronized(this) { - line = reader.readLine(); - if (line == null) { - // Always rewind at end: - if (LuceneTestCase.VERBOSE) { - System.out.println("TEST: LineFileDocs: now rewind file..."); - } - close(); - open(null); - line = reader.readLine(); - } - } - - DocState docState = threadDocs.get(); - if (docState == null) { - docState = new DocState(); - threadDocs.set(docState); - } - - int spot = line.indexOf(SEP); - if (spot == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } - int spot2 = line.indexOf(SEP, 1 + spot); - if (spot2 == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } - - docState.body.setValue(line.substring(1+spot2, line.length())); - final String title = line.substring(0, spot); - docState.title.setValue(title); - docState.titleTokenized.setValue(title); - docState.date.setValue(line.substring(1+spot, spot2)); - docState.id.setValue(Integer.toString(id.getAndIncrement())); - return docState.doc; - } -} Index: lucene/src/test/org/apache/lucene/util/LuceneJUnitDividingSelector.java =================================================================== --- lucene/src/test/org/apache/lucene/util/LuceneJUnitDividingSelector.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/util/LuceneJUnitDividingSelector.java (working copy) @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.lucene.util; -import java.io.File; - -import org.apache.tools.ant.BuildException; -import org.apache.tools.ant.types.Parameter; -import org.apache.tools.ant.types.selectors.BaseExtendSelector; - -/** Divides filesets into equal groups */ -public class LuceneJUnitDividingSelector extends BaseExtendSelector { - private int counter; - /** Number of total parts to split. */ - private int divisor; - /** Current part to accept. */ - private int part; - - @Override - public void setParameters(Parameter[] pParameters) { - super.setParameters(pParameters); - for (int j = 0; j < pParameters.length; j++) { - Parameter p = pParameters[j]; - if ("divisor".equalsIgnoreCase(p.getName())) { - divisor = Integer.parseInt(p.getValue()); - } - else if ("part".equalsIgnoreCase(p.getName())) { - part = Integer.parseInt(p.getValue()); - } - else { - throw new BuildException("unknown " + p.getName()); - } - } - } - - @Override - public void verifySettings() { - super.verifySettings(); - if (divisor <= 0 || part <= 0) { - throw new BuildException("part or divisor not set"); - } - if (part > divisor) { - throw new BuildException("part must be <= divisor"); - } - } - - @Override - public boolean isSelected(File dir, String name, File path) { - counter = counter % divisor + 1; - return counter == part; - } -} Index: lucene/src/test/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -1,1040 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.lang.annotation.Documented; -import java.lang.annotation.Inherited; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.TimeZone; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Field.Index; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.Field.TermVector; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.lucene.index.LogMergePolicy; -import org.apache.lucene.index.SerialMergeScheduler; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.FieldCache.CacheEntry; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestWatchman; -import org.junit.runner.Description; -import org.junit.runner.RunWith; -import org.junit.runner.manipulation.Filter; -import org.junit.runner.manipulation.NoTestsRemainException; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.BlockJUnit4ClassRunner; -import org.junit.runners.model.FrameworkMethod; -import org.junit.runners.model.InitializationError; - -/** - * Base class for all Lucene unit tests, Junit3 or Junit4 variant. - *

- *

- *

- * If you - * override either setUp() or - * tearDown() in your unit test, make sure you - * call super.setUp() and - * super.tearDown() - *

- * - * @After - replaces setup - * @Before - replaces teardown - * @Test - any public method with this annotation is a test case, regardless - * of its name - *

- *

- * See Junit4 documentation for a complete list of features. - *

- * Import from org.junit rather than junit.framework. - *

- * You should be able to use this class anywhere you used LuceneTestCase - * if you annotate your derived class correctly with the annotations above - * @see #assertSaneFieldCaches(String) - */ - -@RunWith(LuceneTestCase.LuceneTestCaseRunner.class) -public abstract class LuceneTestCase extends Assert { - - /** - * true iff tests are run in verbose mode. Note: if it is false, tests are not - * expected to print any messages. - */ - public static final boolean VERBOSE = Boolean.getBoolean("tests.verbose"); - - /** Use this constant when creating Analyzers and any other version-dependent stuff. - *

NOTE: Change this when development starts for new Lucene version: - */ - public static final Version TEST_VERSION_CURRENT = Version.LUCENE_31; - - /** - * If this is set, it is the only method that should run. - */ - static final String TEST_METHOD; - - /** Create indexes in this directory, optimally use a subdir, named after the test */ - public static final File TEMP_DIR; - static { - String method = System.getProperty("testmethod", "").trim(); - TEST_METHOD = method.length() == 0 ? null : method; - String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir")); - if (s == null) - throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'."); - TEMP_DIR = new File(s); - TEMP_DIR.mkdirs(); - } - - // by default we randomly pick a different codec for - // each test case (non-J4 tests) and each test class (J4 - // tests) - /** Gets the locale to run tests with */ - public static final String TEST_LOCALE = System.getProperty("tests.locale", "random"); - /** Gets the timezone to run tests with */ - public static final String TEST_TIMEZONE = System.getProperty("tests.timezone", "random"); - /** Gets the directory to run tests with */ - public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random"); - /** Get the number of times to run tests */ - public static final int TEST_ITER = Integer.parseInt(System.getProperty("tests.iter", "1")); - /** Get the random seed for tests */ - public static final String TEST_SEED = System.getProperty("tests.seed", "random"); - /** whether or not nightly tests should run */ - public static final boolean TEST_NIGHTLY = Boolean.parseBoolean(System.getProperty("tests.nightly", "false")); - /** the line file used by LineFileDocs */ - public static final String TEST_LINE_DOCS_FILE = System.getProperty("tests.linedocsfile", "europarl.lines.txt.gz"); - - /** - * A random multiplier which you should use when writing random tests: - * multiply it by the number of iterations - */ - public static final int RANDOM_MULTIPLIER = Integer.parseInt(System.getProperty("tests.multiplier", "1")); - - private int savedBoolMaxClauseCount; - - private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null; - - /** Used to track if setUp and tearDown are called correctly from subclasses */ - private boolean setup; - - private static class UncaughtExceptionEntry { - public final Thread thread; - public final Throwable exception; - - public UncaughtExceptionEntry(Thread thread, Throwable exception) { - this.thread = thread; - this.exception = exception; - } - } - private List uncaughtExceptions = Collections.synchronizedList(new ArrayList()); - - private static Locale locale; - private static Locale savedLocale; - private static TimeZone timeZone; - private static TimeZone savedTimeZone; - - private static Map stores; - - private static class TwoLongs { - public final long l1, l2; - - public TwoLongs(long l1, long l2) { - this.l1 = l1; - this.l2 = l2; - } - - @Override - public String toString() { - return l1 + ":" + l2; - } - - public static TwoLongs fromString(String s) { - final int i = s.indexOf(':'); - assert i != -1; - return new TwoLongs(Long.parseLong(s.substring(0, i)), - Long.parseLong(s.substring(1+i))); - } - } - - /** @deprecated: until we fix no-fork problems in solr tests */ - @Deprecated - private static List testClassesRun = new ArrayList(); - - @BeforeClass - public static void beforeClassLuceneTestCaseJ4() { - staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : TwoLongs.fromString(TEST_SEED).l1; - random.setSeed(staticSeed); - stores = Collections.synchronizedMap(new IdentityHashMap()); - // this code consumes randoms where 4.0's lucenetestcase would: to make seeds work across both branches. - // TODO: doesn't completely work, because what if we get mockrandom codec?! - if (random.nextInt(4) != 0) { - random.nextInt(); // consume RandomCodecProvider's seed. - } - // end compatibility random-consumption - savedLocale = Locale.getDefault(); - locale = TEST_LOCALE.equals("random") ? randomLocale(random) : localeForName(TEST_LOCALE); - Locale.setDefault(locale); - savedTimeZone = TimeZone.getDefault(); - timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE); - TimeZone.setDefault(timeZone); - testsFailed = false; - } - - @AfterClass - public static void afterClassLuceneTestCaseJ4() { - int rogueThreads = threadCleanup("test class"); - if (rogueThreads > 0) { - // TODO: fail here once the leaks are fixed. - System.err.println("RESOURCE LEAK: test class left " + rogueThreads + " thread(s) running"); - } - Locale.setDefault(savedLocale); - TimeZone.setDefault(savedTimeZone); - System.clearProperty("solr.solr.home"); - System.clearProperty("solr.data.dir"); - // now look for unclosed resources - if (!testsFailed) - for (MockDirectoryWrapper d : stores.keySet()) { - if (d.isOpen()) { - StackTraceElement elements[] = stores.get(d); - // Look for the first class that is not LuceneTestCase that requested - // a Directory. The first two items are of Thread's, so skipping over - // them. - StackTraceElement element = null; - for (int i = 2; i < elements.length; i++) { - StackTraceElement ste = elements[i]; - if (ste.getClassName().indexOf("LuceneTestCase") == -1) { - element = ste; - break; - } - } - fail("directory of test was not closed, opened from: " + element); - } - } - stores = null; - // if verbose or tests failed, report some information back - if (VERBOSE || testsFailed) - System.err.println("NOTE: test params are: " + - "locale=" + locale + - ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID())); - if (testsFailed) { - System.err.println("NOTE: all tests run in this JVM:"); - System.err.println(Arrays.toString(testClassesRun.toArray())); - System.err.println("NOTE: " + System.getProperty("os.name") + " " - + System.getProperty("os.version") + " " - + System.getProperty("os.arch") + "/" - + System.getProperty("java.vendor") + " " - + System.getProperty("java.version") + " " - + (Constants.JRE_IS_64BIT ? "(64-bit)" : "(32-bit)") + "/" - + "cpus=" + Runtime.getRuntime().availableProcessors() + "," - + "threads=" + Thread.activeCount() + "," - + "free=" + Runtime.getRuntime().freeMemory() + "," - + "total=" + Runtime.getRuntime().totalMemory()); - } - } - - private static boolean testsFailed; /* true if any tests failed */ - - // This is how we get control when errors occur. - // Think of this as start/end/success/failed - // events. - @Rule - public final TestWatchman intercept = new TestWatchman() { - - @Override - public void failed(Throwable e, FrameworkMethod method) { - // org.junit.internal.AssumptionViolatedException in older releases - // org.junit.Assume.AssumptionViolatedException in recent ones - if (e.getClass().getName().endsWith("AssumptionViolatedException")) { - if (e.getCause() instanceof TestIgnoredException) - e = e.getCause(); - System.err.print("NOTE: Assume failed in '" + method.getName() + "' (ignored):"); - if (VERBOSE) { - System.err.println(); - e.printStackTrace(System.err); - } else { - System.err.print(" "); - System.err.println(e.getMessage()); - } - } else { - testsFailed = true; - reportAdditionalFailureInfo(); - } - super.failed(e, method); - } - - @Override - public void starting(FrameworkMethod method) { - // set current method name for logging - LuceneTestCase.this.name = method.getName(); - super.starting(method); - } - - }; - - @Before - public void setUp() throws Exception { - seed = "random".equals(TEST_SEED) ? seedRand.nextLong() : TwoLongs.fromString(TEST_SEED).l2; - random.setSeed(seed); - assertFalse("ensure your tearDown() calls super.tearDown()!!!", setup); - setup = true; - savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread t, Throwable e) { - testsFailed = true; - uncaughtExceptions.add(new UncaughtExceptionEntry(t, e)); - if (savedUncaughtExceptionHandler != null) - savedUncaughtExceptionHandler.uncaughtException(t, e); - } - }); - - savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); - } - - - /** - * Forcible purges all cache entries from the FieldCache. - *

- * This method will be called by tearDown to clean up FieldCache.DEFAULT. - * If a (poorly written) test has some expectation that the FieldCache - * will persist across test methods (ie: a static IndexReader) this - * method can be overridden to do nothing. - *

- * - * @see FieldCache#purgeAllCaches() - */ - protected void purgeFieldCache(final FieldCache fc) { - fc.purgeAllCaches(); - } - - protected String getTestLabel() { - return getClass().getName() + "." + getName(); - } - - @After - public void tearDown() throws Exception { - assertTrue("ensure your setUp() calls super.setUp()!!!", setup); - setup = false; - BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount); - if (!getClass().getName().startsWith("org.apache.solr")) { - int rogueThreads = threadCleanup("test method: '" + getName() + "'"); - if (rogueThreads > 0) { - System.err.println("RESOURCE LEAK: test method: '" + getName() - + "' left " + rogueThreads + " thread(s) running"); - // TODO: fail, but print seed for now. - if (!testsFailed && uncaughtExceptions.isEmpty()) { - reportAdditionalFailureInfo(); - } - } - } - Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler); - try { - - if (!uncaughtExceptions.isEmpty()) { - testsFailed = true; - reportAdditionalFailureInfo(); - System.err.println("The following exceptions were thrown by threads:"); - for (UncaughtExceptionEntry entry : uncaughtExceptions) { - System.err.println("*** Thread: " + entry.thread.getName() + " ***"); - entry.exception.printStackTrace(System.err); - } - fail("Some threads threw uncaught exceptions!"); - } - - // calling assertSaneFieldCaches here isn't as useful as having test - // classes call it directly from the scope where the index readers - // are used, because they could be gc'ed just before this tearDown - // method is called. - // - // But it's better then nothing. - // - // If you are testing functionality that you know for a fact - // "violates" FieldCache sanity, then you should either explicitly - // call purgeFieldCache at the end of your test method, or refactor - // your Test class so that the inconsistant FieldCache usages are - // isolated in distinct test methods - assertSaneFieldCaches(getTestLabel()); - - } finally { - purgeFieldCache(FieldCache.DEFAULT); - } - } - - private final static int THREAD_STOP_GRACE_MSEC = 1000; - // jvm-wide list of 'rogue threads' we found, so they only get reported once. - private final static IdentityHashMap rogueThreads = new IdentityHashMap(); - - static { - // just a hack for things like eclipse test-runner threads - for (Thread t : Thread.getAllStackTraces().keySet()) { - rogueThreads.put(t, true); - } - } - - /** - * Looks for leftover running threads, trying to kill them off, - * so they don't fail future tests. - * returns the number of rogue threads that it found. - */ - private static int threadCleanup(String context) { - // educated guess - Thread[] stillRunning = new Thread[Thread.activeCount()+1]; - int threadCount = 0; - int rogueCount = 0; - - if ((threadCount = Thread.enumerate(stillRunning)) > 1) { - while (threadCount == stillRunning.length) { - // truncated response - stillRunning = new Thread[stillRunning.length*2]; - threadCount = Thread.enumerate(stillRunning); - } - - for (int i = 0; i < threadCount; i++) { - Thread t = stillRunning[i]; - - if (t.isAlive() && - !rogueThreads.containsKey(t) && - t != Thread.currentThread() && - // TODO: TimeLimitingCollector starts a thread statically.... WTF?! - !t.getName().equals("TimeLimitedCollector timer thread")) { - System.err.println("WARNING: " + context + " left thread running: " + t); - rogueThreads.put(t, true); - rogueCount++; - // wait on the thread to die of natural causes - try { - t.join(THREAD_STOP_GRACE_MSEC); - } catch (InterruptedException e) { e.printStackTrace(); } - // try to stop the thread: - t.setUncaughtExceptionHandler(null); - Thread.setDefaultUncaughtExceptionHandler(null); - t.interrupt(); - try { - t.join(THREAD_STOP_GRACE_MSEC); - } catch (InterruptedException e) { e.printStackTrace(); } - } - } - } - return rogueCount; - } - - /** - * Asserts that FieldCacheSanityChecker does not detect any - * problems with FieldCache.DEFAULT. - *

- * If any problems are found, they are logged to System.err - * (allong with the msg) when the Assertion is thrown. - *

- *

- * This method is called by tearDown after every test method, - * however IndexReaders scoped inside test methods may be garbage - * collected prior to this method being called, causing errors to - * be overlooked. Tests are encouraged to keep their IndexReaders - * scoped at the class level, or to explicitly call this method - * directly in the same scope as the IndexReader. - *

- * - * @see FieldCacheSanityChecker - */ - protected void assertSaneFieldCaches(final String msg) { - final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries(); - Insanity[] insanity = null; - try { - try { - insanity = FieldCacheSanityChecker.checkSanity(entries); - } catch (RuntimeException e) { - dumpArray(msg + ": FieldCache", entries, System.err); - throw e; - } - - assertEquals(msg + ": Insane FieldCache usage(s) found", - 0, insanity.length); - insanity = null; - } finally { - - // report this in the event of any exception/failure - // if no failure, then insanity will be null anyway - if (null != insanity) { - dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err); - } - - } - } - - // These deprecated methods should be removed soon, when all tests using no Epsilon are fixed: - - @Deprecated - static public void assertEquals(double expected, double actual) { - assertEquals(null, expected, actual); - } - - @Deprecated - static public void assertEquals(String message, double expected, double actual) { - assertEquals(message, Double.valueOf(expected), Double.valueOf(actual)); - } - - @Deprecated - static public void assertEquals(float expected, float actual) { - assertEquals(null, expected, actual); - } - - @Deprecated - static public void assertEquals(String message, float expected, float actual) { - assertEquals(message, Float.valueOf(expected), Float.valueOf(actual)); - } - - // Replacement for Assume jUnit class, so we can add a message with explanation: - - private static final class TestIgnoredException extends RuntimeException { - TestIgnoredException(String msg) { - super(msg); - } - - TestIgnoredException(String msg, Throwable t) { - super(msg, t); - } - - @Override - public String getMessage() { - StringBuilder sb = new StringBuilder(super.getMessage()); - if (getCause() != null) - sb.append(" - ").append(getCause()); - return sb.toString(); - } - - // only this one is called by our code, exception is not used outside this class: - @Override - public void printStackTrace(PrintStream s) { - if (getCause() != null) { - s.println(super.toString() + " - Caused by:"); - getCause().printStackTrace(s); - } else { - super.printStackTrace(s); - } - } - } - - public static void assumeTrue(String msg, boolean b) { - Assume.assumeNoException(b ? null : new TestIgnoredException(msg)); - } - - public static void assumeFalse(String msg, boolean b) { - assumeTrue(msg, !b); - } - - public static void assumeNoException(String msg, Exception e) { - Assume.assumeNoException(e == null ? null : new TestIgnoredException(msg, e)); - } - - /** - * Convinience method for logging an iterator. - * - * @param label String logged before/after the items in the iterator - * @param iter Each next() is toString()ed and logged on it's own line. If iter is null this is logged differnetly then an empty iterator. - * @param stream Stream to log messages to. - */ - public static void dumpIterator(String label, Iterator iter, - PrintStream stream) { - stream.println("*** BEGIN " + label + " ***"); - if (null == iter) { - stream.println(" ... NULL ..."); - } else { - while (iter.hasNext()) { - stream.println(iter.next().toString()); - } - } - stream.println("*** END " + label + " ***"); - } - - /** - * Convinience method for logging an array. Wraps the array in an iterator and delegates - * - * @see #dumpIterator(String,Iterator,PrintStream) - */ - public static void dumpArray(String label, Object[] objs, - PrintStream stream) { - Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator(); - dumpIterator(label, iter, stream); - } - - /** create a new index writer config with random defaults */ - public static IndexWriterConfig newIndexWriterConfig(Version v, Analyzer a) { - return newIndexWriterConfig(random, v, a); - } - - public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) { - IndexWriterConfig c = new IndexWriterConfig(v, a); - if (r.nextBoolean()) { - c.setMergePolicy(new LogDocMergePolicy()); - } - if (r.nextBoolean()) { - c.setMergeScheduler(new SerialMergeScheduler()); - } - if (r.nextBoolean()) { - if (r.nextInt(20) == 17) { - c.setMaxBufferedDocs(2); - } else { - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 1000)); - } - } - if (r.nextBoolean()) { - c.setTermIndexInterval(_TestUtil.nextInt(r, 1, 1000)); - } - if (r.nextBoolean()) { - c.setMaxThreadStates(_TestUtil.nextInt(r, 1, 20)); - } - - if (c.getMergePolicy() instanceof LogMergePolicy) { - LogMergePolicy logmp = (LogMergePolicy) c.getMergePolicy(); - logmp.setUseCompoundFile(r.nextBoolean()); - logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - if (r.nextInt(3) == 2) { - logmp.setMergeFactor(2); - } else { - logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20)); - } - } - - c.setReaderPooling(r.nextBoolean()); - c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4)); - return c; - } - - public static LogMergePolicy newLogMergePolicy() { - return newLogMergePolicy(random); - } - - public static LogMergePolicy newLogMergePolicy(Random r) { - LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); - logmp.setUseCompoundFile(r.nextBoolean()); - logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - if (r.nextInt(3) == 2) { - logmp.setMergeFactor(2); - } else { - logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20)); - } - return logmp; - } - - public static LogMergePolicy newLogMergePolicy(boolean useCFS) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setUseCompoundFile(useCFS); - return logmp; - } - - public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setUseCompoundFile(useCFS); - logmp.setMergeFactor(mergeFactor); - return logmp; - } - - public static LogMergePolicy newLogMergePolicy(int mergeFactor) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setMergeFactor(mergeFactor); - return logmp; - } - - /** - * Returns a new Directory instance. Use this when the test does not - * care about the specific Directory implementation (most tests). - *

- * The Directory is wrapped with {@link MockDirectoryWrapper}. - * By default this means it will be picky, such as ensuring that you - * properly close it and all open files in your test. It will emulate - * some features of Windows, such as not allowing open files to be - * overwritten. - */ - public static MockDirectoryWrapper newDirectory() throws IOException { - return newDirectory(random); - } - - public static MockDirectoryWrapper newDirectory(Random r) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); - MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); - stores.put(dir, Thread.currentThread().getStackTrace()); - return dir; - } - - /** - * Returns a new Directory instance, with contents copied from the - * provided directory. See {@link #newDirectory()} for more - * information. - */ - public static MockDirectoryWrapper newDirectory(Directory d) throws IOException { - return newDirectory(random, d); - } - - /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f) throws IOException { - return newFSDirectory(f, null); - } - - /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf) throws IOException { - String fsdirClass = TEST_DIRECTORY; - if (fsdirClass.equals("random")) { - fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; - } - - if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store - fsdirClass = "org.apache.lucene.store." + fsdirClass; - } - - Class clazz; - try { - try { - clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); - } catch (ClassCastException e) { - // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random - fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; - - if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store - fsdirClass = "org.apache.lucene.store." + fsdirClass; - } - - clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); - } - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f, lf)); - stores.put(dir, Thread.currentThread().getStackTrace()); - return dir; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); - for (String file : d.listAll()) { - d.copy(impl, file, file); - } - MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); - stores.put(dir, Thread.currentThread().getStackTrace()); - return dir; - } - - public static Field newField(String name, String value, Index index) { - return newField(random, name, value, index); - } - - public static Field newField(String name, String value, Store store, Index index) { - return newField(random, name, value, store, index); - } - - public static Field newField(String name, String value, Store store, Index index, TermVector tv) { - return newField(random, name, value, store, index, tv); - } - - public static Field newField(Random random, String name, String value, Index index) { - return newField(random, name, value, Store.NO, index); - } - - public static Field newField(Random random, String name, String value, Store store, Index index) { - return newField(random, name, value, store, index, TermVector.NO); - } - - public static Field newField(Random random, String name, String value, Store store, Index index, TermVector tv) { - if (!index.isIndexed()) - return new Field(name, value, store, index); - - if (!store.isStored() && random.nextBoolean()) - store = Store.YES; // randomly store it - - tv = randomTVSetting(random, tv); - - return new Field(name, value, store, index, tv); - } - - static final TermVector tvSettings[] = { - TermVector.NO, TermVector.YES, TermVector.WITH_OFFSETS, - TermVector.WITH_POSITIONS, TermVector.WITH_POSITIONS_OFFSETS - }; - - private static TermVector randomTVSetting(Random random, TermVector minimum) { - switch(minimum) { - case NO: return tvSettings[_TestUtil.nextInt(random, 0, tvSettings.length-1)]; - case YES: return tvSettings[_TestUtil.nextInt(random, 1, tvSettings.length-1)]; - case WITH_OFFSETS: return random.nextBoolean() ? TermVector.WITH_OFFSETS - : TermVector.WITH_POSITIONS_OFFSETS; - case WITH_POSITIONS: return random.nextBoolean() ? TermVector.WITH_POSITIONS - : TermVector.WITH_POSITIONS_OFFSETS; - default: return TermVector.WITH_POSITIONS_OFFSETS; - } - } - - /** return a random Locale from the available locales on the system */ - public static Locale randomLocale(Random random) { - Locale locales[] = Locale.getAvailableLocales(); - return locales[random.nextInt(locales.length)]; - } - - /** return a random TimeZone from the available timezones on the system */ - public static TimeZone randomTimeZone(Random random) { - String tzIds[] = TimeZone.getAvailableIDs(); - return TimeZone.getTimeZone(tzIds[random.nextInt(tzIds.length)]); - } - - /** return a Locale object equivalent to its programmatic name */ - public static Locale localeForName(String localeName) { - String elements[] = localeName.split("\\_"); - switch(elements.length) { - case 3: return new Locale(elements[0], elements[1], elements[2]); - case 2: return new Locale(elements[0], elements[1]); - case 1: return new Locale(elements[0]); - default: throw new IllegalArgumentException("Invalid Locale: " + localeName); - } - } - - private static final String FS_DIRECTORIES[] = { - "SimpleFSDirectory", - "NIOFSDirectory", - "MMapDirectory" - }; - - private static final String CORE_DIRECTORIES[] = { - "RAMDirectory", - FS_DIRECTORIES[0], FS_DIRECTORIES[1], FS_DIRECTORIES[2] - }; - - public static String randomDirectory(Random random) { - if (random.nextInt(10) == 0) { - return CORE_DIRECTORIES[random.nextInt(CORE_DIRECTORIES.length)]; - } else { - return "RAMDirectory"; - } - } - - private static Directory newFSDirectoryImpl( - Class clazz, File file, LockFactory lockFactory) - throws IOException { - try { - // Assuming every FSDirectory has a ctor(File), but not all may take a - // LockFactory too, so setting it afterwards. - Constructor ctor = clazz.getConstructor(File.class); - FSDirectory d = ctor.newInstance(file); - if (lockFactory != null) { - d.setLockFactory(lockFactory); - } - return d; - } catch (Exception e) { - return FSDirectory.open(file); - } - } - - static Directory newDirectoryImpl(Random random, String clazzName) { - if (clazzName.equals("random")) - clazzName = randomDirectory(random); - if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store - clazzName = "org.apache.lucene.store." + clazzName; - try { - final Class clazz = Class.forName(clazzName).asSubclass(Directory.class); - // If it is a FSDirectory type, try its ctor(File) - if (FSDirectory.class.isAssignableFrom(clazz)) { - final File tmpFile = File.createTempFile("test", "tmp", TEMP_DIR); - tmpFile.delete(); - tmpFile.mkdir(); - return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile, null); - } - - // try empty ctor - return clazz.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - - public String getName() { - return this.name; - } - - /** Gets a resource from the classpath as {@link File}. This method should only be used, - * if a real file is needed. To get a stream, code should prefer - * {@link Class#getResourceAsStream} using {@code this.getClass()}. - */ - - protected File getDataFile(String name) throws IOException { - try { - return new File(this.getClass().getResource(name).toURI()); - } catch (Exception e) { - throw new IOException("Cannot find resource: " + name); - } - } - - // We get here from InterceptTestCaseEvents on the 'failed' event.... - public void reportAdditionalFailureInfo() { - System.err.println("NOTE: reproduce with: ant test -Dtestcase=" + getClass().getSimpleName() - + " -Dtestmethod=" + getName() + " -Dtests.seed=" + new TwoLongs(staticSeed, seed) - + reproduceWithExtraParams()); - } - - // extra params that were overridden needed to reproduce the command - private String reproduceWithExtraParams() { - StringBuilder sb = new StringBuilder(); - if (!TEST_LOCALE.equals("random")) sb.append(" -Dtests.locale=").append(TEST_LOCALE); - if (!TEST_TIMEZONE.equals("random")) sb.append(" -Dtests.timezone=").append(TEST_TIMEZONE); - if (!TEST_DIRECTORY.equals("random")) sb.append(" -Dtests.directory=").append(TEST_DIRECTORY); - if (RANDOM_MULTIPLIER > 1) sb.append(" -Dtests.multiplier=").append(RANDOM_MULTIPLIER); - return sb.toString(); - } - - // recorded seed: for beforeClass - private static long staticSeed; - // seed for individual test methods, changed in @before - private long seed; - - private static final Random seedRand = new Random(); - protected static final Random random = new Random(0); - - private String name = ""; - - /** - * Annotation for tests that should only be run during nightly builds. - */ - @Documented - @Inherited - @Retention(RetentionPolicy.RUNTIME) - public @interface Nightly {} - - /** optionally filters the tests to be run by TEST_METHOD */ - public static class LuceneTestCaseRunner extends BlockJUnit4ClassRunner { - private List testMethods; - - @Override - protected List computeTestMethods() { - if (testMethods != null) - return testMethods; - testClassesRun.add(getTestClass().getJavaClass().getSimpleName()); - testMethods = new ArrayList(); - for (Method m : getTestClass().getJavaClass().getMethods()) { - // check if the current test's class has methods annotated with @Ignore - final Ignore ignored = m.getAnnotation(Ignore.class); - if (ignored != null && !m.getName().equals("alwaysIgnoredTestMethod")) { - System.err.println("NOTE: Ignoring test method '" + m.getName() + "': " + ignored.value()); - } - // add methods starting with "test" - final int mod = m.getModifiers(); - if (m.getAnnotation(Test.class) != null || - (m.getName().startsWith("test") && - !Modifier.isAbstract(mod) && - m.getParameterTypes().length == 0 && - m.getReturnType() == Void.TYPE)) - { - if (Modifier.isStatic(mod)) - throw new RuntimeException("Test methods must not be static."); - testMethods.add(new FrameworkMethod(m)); - } - } - - if (testMethods.isEmpty()) { - throw new RuntimeException("No runnable methods!"); - } - - if (TEST_NIGHTLY == false) { - if (getTestClass().getJavaClass().isAnnotationPresent(Nightly.class)) { - /* the test class is annotated with nightly, remove all methods */ - String className = getTestClass().getJavaClass().getSimpleName(); - System.err.println("NOTE: Ignoring nightly-only test class '" + className + "'"); - testMethods.clear(); - } else { - /* remove all nightly-only methods */ - for (int i = 0; i < testMethods.size(); i++) { - final FrameworkMethod m = testMethods.get(i); - if (m.getAnnotation(Nightly.class) != null) { - System.err.println("NOTE: Ignoring nightly-only test method '" + m.getName() + "'"); - testMethods.remove(i--); - } - } - } - /* dodge a possible "no-runnable methods" exception by adding a fake ignored test */ - if (testMethods.isEmpty()) { - try { - testMethods.add(new FrameworkMethod(LuceneTestCase.class.getMethod("alwaysIgnoredTestMethod"))); - } catch (Exception e) { throw new RuntimeException(e); } - } - } - return testMethods; - } - - @Override - protected void runChild(FrameworkMethod arg0, RunNotifier arg1) { - if (VERBOSE) { - System.out.println("\nNOTE: running test " + arg0.getName()); - } - for (int i = 0; i < TEST_ITER; i++) { - if (VERBOSE && TEST_ITER > 1) { - System.out.println("\nNOTE: running iter=" + (1+i) + " of " + TEST_ITER); - } - super.runChild(arg0, arg1); - } - } - - public LuceneTestCaseRunner(Class clazz) throws InitializationError { - super(clazz); - Filter f = new Filter() { - - @Override - public String describe() { return "filters according to TEST_METHOD"; } - - @Override - public boolean shouldRun(Description d) { - return TEST_METHOD == null || d.getMethodName().equals(TEST_METHOD); - } - }; - - try { - f.apply(this); - } catch (NoTestsRemainException e) { - throw new RuntimeException(e); - } - } - } - - @Ignore("just a hack") - public final void alwaysIgnoredTestMethod() {} -} Index: lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java =================================================================== --- lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/util/LuceneJUnitResultFormatter.java (working copy) @@ -1,293 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.lucene.util; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.text.NumberFormat; -import java.util.logging.LogManager; - -import junit.framework.AssertionFailedError; -import junit.framework.Test; - -import org.apache.lucene.store.LockReleaseFailedException; -import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.tools.ant.taskdefs.optional.junit.JUnitResultFormatter; -import org.apache.tools.ant.taskdefs.optional.junit.JUnitTest; -import org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner; -import org.apache.tools.ant.util.FileUtils; -import org.apache.tools.ant.util.StringUtils; -import org.junit.Ignore; - -/** - * Just like BriefJUnitResultFormatter "brief" bundled with ant, - * except all formatted text is buffered until the test suite is finished. - * At this point, the output is written at once in synchronized fashion. - * This way tests can run in parallel without interleaving output. - */ -public class LuceneJUnitResultFormatter implements JUnitResultFormatter { - private static final double ONE_SECOND = 1000.0; - - private static final NativeFSLockFactory lockFactory; - - /** Where to write the log to. */ - private OutputStream out; - - /** Formatter for timings. */ - private NumberFormat numberFormat = NumberFormat.getInstance(); - - /** Output suite has written to System.out */ - private String systemOutput = null; - - /** Output suite has written to System.err */ - private String systemError = null; - - /** Buffer output until the end of the test */ - private ByteArrayOutputStream sb; // use a BOS for our mostly ascii-output - - private static final org.apache.lucene.store.Lock lock; - - static { - File lockDir = new File(System.getProperty("java.io.tmpdir"), - "lucene_junit_lock"); - lockDir.mkdirs(); - if (!lockDir.exists()) { - throw new RuntimeException("Could not make Lock directory:" + lockDir); - } - try { - lockFactory = new NativeFSLockFactory(lockDir); - lock = lockFactory.makeLock("junit_lock"); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** Constructor for LuceneJUnitResultFormatter. */ - public LuceneJUnitResultFormatter() { - } - - /** - * Sets the stream the formatter is supposed to write its results to. - * @param out the output stream to write to - */ - public void setOutput(OutputStream out) { - this.out = out; - } - - /** - * @see JUnitResultFormatter#setSystemOutput(String) - */ - /** {@inheritDoc}. */ - public void setSystemOutput(String out) { - systemOutput = out; - } - - /** - * @see JUnitResultFormatter#setSystemError(String) - */ - /** {@inheritDoc}. */ - public void setSystemError(String err) { - systemError = err; - } - - - /** - * The whole testsuite started. - * @param suite the test suite - */ - public synchronized void startTestSuite(JUnitTest suite) { - if (out == null) { - return; // Quick return - no output do nothing. - } - sb = new ByteArrayOutputStream(); // don't reuse, so its gc'ed - try { - LogManager.getLogManager().readConfiguration(); - } catch (Exception e) {} - append("Testsuite: "); - append(suite.getName()); - append(StringUtils.LINE_SEP); - } - - /** - * The whole testsuite ended. - * @param suite the test suite - */ - public synchronized void endTestSuite(JUnitTest suite) { - append("Tests run: "); - append(suite.runCount()); - append(", Failures: "); - append(suite.failureCount()); - append(", Errors: "); - append(suite.errorCount()); - append(", Time elapsed: "); - append(numberFormat.format(suite.getRunTime() / ONE_SECOND)); - append(" sec"); - append(StringUtils.LINE_SEP); - append(StringUtils.LINE_SEP); - - // append the err and output streams to the log - if (systemOutput != null && systemOutput.length() > 0) { - append("------------- Standard Output ---------------") - .append(StringUtils.LINE_SEP) - .append(systemOutput) - .append("------------- ---------------- ---------------") - .append(StringUtils.LINE_SEP); - } - - // HACK: junit gives us no way to do this in LuceneTestCase - try { - Class clazz = Class.forName(suite.getName()); - Ignore ignore = clazz.getAnnotation(Ignore.class); - if (ignore != null) { - if (systemError == null) systemError = ""; - systemError += "NOTE: Ignoring test class '" + clazz.getSimpleName() + "': " - + ignore.value() + StringUtils.LINE_SEP; - } - } catch (ClassNotFoundException e) { /* no problem */ } - // END HACK - - if (systemError != null && systemError.length() > 0) { - append("------------- Standard Error -----------------") - .append(StringUtils.LINE_SEP) - .append(systemError) - .append("------------- ---------------- ---------------") - .append(StringUtils.LINE_SEP); - } - - if (out != null) { - try { - lock.obtain(5000); - try { - sb.writeTo(out); - out.flush(); - } finally { - try { - lock.release(); - } catch(LockReleaseFailedException e) { - // well lets pretend its released anyway - } - } - } catch (IOException e) { - throw new RuntimeException("unable to write results", e); - } finally { - if (out != System.out && out != System.err) { - FileUtils.close(out); - } - } - } - } - - /** - * A test started. - * @param test a test - */ - public void startTest(Test test) { - } - - /** - * A test ended. - * @param test a test - */ - public void endTest(Test test) { - } - - /** - * Interface TestListener for JUnit <= 3.4. - * - *

A Test failed. - * @param test a test - * @param t the exception thrown by the test - */ - public void addFailure(Test test, Throwable t) { - formatError("\tFAILED", test, t); - } - - /** - * Interface TestListener for JUnit > 3.4. - * - *

A Test failed. - * @param test a test - * @param t the assertion failed by the test - */ - public void addFailure(Test test, AssertionFailedError t) { - addFailure(test, (Throwable) t); - } - - /** - * A test caused an error. - * @param test a test - * @param error the error thrown by the test - */ - public void addError(Test test, Throwable error) { - formatError("\tCaused an ERROR", test, error); - } - - /** - * Format the test for printing.. - * @param test a test - * @return the formatted testname - */ - protected String formatTest(Test test) { - if (test == null) { - return "Null Test: "; - } else { - return "Testcase: " + test.toString() + ":"; - } - } - - /** - * Format an error and print it. - * @param type the type of error - * @param test the test that failed - * @param error the exception that the test threw - */ - protected synchronized void formatError(String type, Test test, - Throwable error) { - if (test != null) { - endTest(test); - } - - append(formatTest(test) + type); - append(StringUtils.LINE_SEP); - append(error.getMessage()); - append(StringUtils.LINE_SEP); - String strace = JUnitTestRunner.getFilteredTrace(error); - append(strace); - append(StringUtils.LINE_SEP); - append(StringUtils.LINE_SEP); - } - - public LuceneJUnitResultFormatter append(String s) { - if (s == null) - s = "(null)"; - try { - sb.write(s.getBytes()); // intentionally use default charset, its a console. - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - public LuceneJUnitResultFormatter append(long l) { - return append(Long.toString(l)); - } -} - Index: lucene/src/test/org/apache/lucene/util/_TestUtil.java =================================================================== --- lucene/src/test/org/apache/lucene/util/_TestUtil.java (revision 1066326) +++ lucene/src/test/org/apache/lucene/util/_TestUtil.java (working copy) @@ -1,261 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Random; -import java.util.Map; -import java.util.HashMap; - -import org.junit.Assert; - -import org.apache.lucene.index.CheckIndex; -import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.LogMergePolicy; -import org.apache.lucene.index.MergeScheduler; -import org.apache.lucene.store.Directory; - -public class _TestUtil { - - /** Returns temp dir, containing String arg in its name; - * does not create the directory. */ - public static File getTempDir(String desc) { - return new File(LuceneTestCase.TEMP_DIR, desc + "." + new Random().nextLong()); - } - - public static void rmDir(File dir) throws IOException { - if (dir.exists()) { - for (File f : dir.listFiles()) { - if (f.isDirectory()) { - rmDir(f); - } else { - if (!f.delete()) { - throw new IOException("could not delete " + f); - } - } - } - if (!dir.delete()) { - throw new IOException("could not delete " + dir); - } - } - } - - public static void syncConcurrentMerges(IndexWriter writer) { - syncConcurrentMerges(writer.getConfig().getMergeScheduler()); - } - - public static void syncConcurrentMerges(MergeScheduler ms) { - if (ms instanceof ConcurrentMergeScheduler) - ((ConcurrentMergeScheduler) ms).sync(); - } - - /** This runs the CheckIndex tool on the index in. If any - * issues are hit, a RuntimeException is thrown; else, - * true is returned. */ - public static CheckIndex.Status checkIndex(Directory dir) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); - - CheckIndex checker = new CheckIndex(dir); - checker.setInfoStream(new PrintStream(bos)); - CheckIndex.Status indexStatus = checker.checkIndex(); - if (indexStatus == null || indexStatus.clean == false) { - System.out.println("CheckIndex failed"); - System.out.println(bos.toString()); - throw new RuntimeException("CheckIndex failed"); - } else { - return indexStatus; - } - } - - /** Use only for testing. - * @deprecated -- in 3.0 we can use Arrays.toString - * instead */ - @Deprecated - public static String arrayToString(int[] array) { - StringBuilder buf = new StringBuilder(); - buf.append("["); - for(int i=0;i 0) { - buf.append(" "); - } - buf.append(array[i]); - } - buf.append("]"); - return buf.toString(); - } - - /** Use only for testing. - * @deprecated -- in 3.0 we can use Arrays.toString - * instead */ - @Deprecated - public static String arrayToString(Object[] array) { - StringBuilder buf = new StringBuilder(); - buf.append("["); - for(int i=0;i 0) { - buf.append(" "); - } - buf.append(array[i]); - } - buf.append("]"); - return buf.toString(); - } - - /** Returns random string, including full unicode range. */ - public static String randomUnicodeString(Random r) { - return randomUnicodeString(r, 20); - } - - public static String randomUnicodeString(Random r, int maxLength) { - final int end = r.nextInt(maxLength); - if (end == 0) { - // allow 0 length - return ""; - } - final char[] buffer = new char[end]; - for (int i = 0; i < end; i++) { - int t = r.nextInt(5); - - - if (0 == t && i < end - 1) { - // Make a surrogate pair - // High surrogate - buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff); - // Low surrogate - buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff); - } - else if (t <= 1) buffer[i] = (char) r.nextInt(0x80); - else if (2 == t) buffer[i] = (char) nextInt(r, 0x80, 0x800); - else if (3 == t) buffer[i] = (char) nextInt(r, 0x800, 0xd7ff); - else if (4 == t) buffer[i] = (char) nextInt(r, 0xe000, 0xfffe); - } - return new String(buffer, 0, end); - } - - private static final int[] blockStarts = { - 0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400, - 0x0500, 0x0530, 0x0590, 0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800, - 0x0900, 0x0980, 0x0A00, 0x0A80, 0x0B00, 0x0B80, 0x0C00, 0x0C80, 0x0D00, - 0x0D80, 0x0E00, 0x0E80, 0x0F00, 0x1000, 0x10A0, 0x1100, 0x1200, 0x1380, - 0x13A0, 0x1400, 0x1680, 0x16A0, 0x1700, 0x1720, 0x1740, 0x1760, 0x1780, - 0x1800, 0x18B0, 0x1900, 0x1950, 0x1980, 0x19E0, 0x1A00, 0x1A20, 0x1B00, - 0x1B80, 0x1C00, 0x1C50, 0x1CD0, 0x1D00, 0x1D80, 0x1DC0, 0x1E00, 0x1F00, - 0x2000, 0x2070, 0x20A0, 0x20D0, 0x2100, 0x2150, 0x2190, 0x2200, 0x2300, - 0x2400, 0x2440, 0x2460, 0x2500, 0x2580, 0x25A0, 0x2600, 0x2700, 0x27C0, - 0x27F0, 0x2800, 0x2900, 0x2980, 0x2A00, 0x2B00, 0x2C00, 0x2C60, 0x2C80, - 0x2D00, 0x2D30, 0x2D80, 0x2DE0, 0x2E00, 0x2E80, 0x2F00, 0x2FF0, 0x3000, - 0x3040, 0x30A0, 0x3100, 0x3130, 0x3190, 0x31A0, 0x31C0, 0x31F0, 0x3200, - 0x3300, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA490, 0xA4D0, 0xA500, 0xA640, - 0xA6A0, 0xA700, 0xA720, 0xA800, 0xA830, 0xA840, 0xA880, 0xA8E0, 0xA900, - 0xA930, 0xA960, 0xA980, 0xAA00, 0xAA60, 0xAA80, 0xABC0, 0xAC00, 0xD7B0, - 0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10, - 0xFE20, 0xFE30, 0xFE50, 0xFE70, 0xFF00, 0xFFF0, - 0x10000, 0x10080, 0x10100, 0x10140, 0x10190, 0x101D0, 0x10280, 0x102A0, - 0x10300, 0x10330, 0x10380, 0x103A0, 0x10400, 0x10450, 0x10480, 0x10800, - 0x10840, 0x10900, 0x10920, 0x10A00, 0x10A60, 0x10B00, 0x10B40, 0x10B60, - 0x10C00, 0x10E60, 0x11080, 0x12000, 0x12400, 0x13000, 0x1D000, 0x1D100, - 0x1D200, 0x1D300, 0x1D360, 0x1D400, 0x1F000, 0x1F030, 0x1F100, 0x1F200, - 0x20000, 0x2A700, 0x2F800, 0xE0000, 0xE0100, 0xF0000, 0x100000 - }; - - private static final int[] blockEnds = { - 0x007F, 0x00FF, 0x017F, 0x024F, 0x02AF, 0x02FF, 0x036F, 0x03FF, 0x04FF, - 0x052F, 0x058F, 0x05FF, 0x06FF, 0x074F, 0x077F, 0x07BF, 0x07FF, 0x083F, - 0x097F, 0x09FF, 0x0A7F, 0x0AFF, 0x0B7F, 0x0BFF, 0x0C7F, 0x0CFF, 0x0D7F, - 0x0DFF, 0x0E7F, 0x0EFF, 0x0FFF, 0x109F, 0x10FF, 0x11FF, 0x137F, 0x139F, - 0x13FF, 0x167F, 0x169F, 0x16FF, 0x171F, 0x173F, 0x175F, 0x177F, 0x17FF, - 0x18AF, 0x18FF, 0x194F, 0x197F, 0x19DF, 0x19FF, 0x1A1F, 0x1AAF, 0x1B7F, - 0x1BBF, 0x1C4F, 0x1C7F, 0x1CFF, 0x1D7F, 0x1DBF, 0x1DFF, 0x1EFF, 0x1FFF, - 0x206F, 0x209F, 0x20CF, 0x20FF, 0x214F, 0x218F, 0x21FF, 0x22FF, 0x23FF, - 0x243F, 0x245F, 0x24FF, 0x257F, 0x259F, 0x25FF, 0x26FF, 0x27BF, 0x27EF, - 0x27FF, 0x28FF, 0x297F, 0x29FF, 0x2AFF, 0x2BFF, 0x2C5F, 0x2C7F, 0x2CFF, - 0x2D2F, 0x2D7F, 0x2DDF, 0x2DFF, 0x2E7F, 0x2EFF, 0x2FDF, 0x2FFF, 0x303F, - 0x309F, 0x30FF, 0x312F, 0x318F, 0x319F, 0x31BF, 0x31EF, 0x31FF, 0x32FF, - 0x33FF, 0x4DBF, 0x4DFF, 0x9FFF, 0xA48F, 0xA4CF, 0xA4FF, 0xA63F, 0xA69F, - 0xA6FF, 0xA71F, 0xA7FF, 0xA82F, 0xA83F, 0xA87F, 0xA8DF, 0xA8FF, 0xA92F, - 0xA95F, 0xA97F, 0xA9DF, 0xAA5F, 0xAA7F, 0xAADF, 0xABFF, 0xD7AF, 0xD7FF, - 0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F, - 0xFE2F, 0xFE4F, 0xFE6F, 0xFEFF, 0xFFEF, 0xFFFF, - 0x1007F, 0x100FF, 0x1013F, 0x1018F, 0x101CF, 0x101FF, 0x1029F, 0x102DF, - 0x1032F, 0x1034F, 0x1039F, 0x103DF, 0x1044F, 0x1047F, 0x104AF, 0x1083F, - 0x1085F, 0x1091F, 0x1093F, 0x10A5F, 0x10A7F, 0x10B3F, 0x10B5F, 0x10B7F, - 0x10C4F, 0x10E7F, 0x110CF, 0x123FF, 0x1247F, 0x1342F, 0x1D0FF, 0x1D1FF, - 0x1D24F, 0x1D35F, 0x1D37F, 0x1D7FF, 0x1F02F, 0x1F09F, 0x1F1FF, 0x1F2FF, - 0x2A6DF, 0x2B73F, 0x2FA1F, 0xE007F, 0xE01EF, 0xFFFFF, 0x10FFFF - }; - - /** Returns random string, all codepoints within the same unicode block. */ - public static String randomRealisticUnicodeString(Random r) { - return randomRealisticUnicodeString(r, 20); - } - - /** Returns random string, all codepoints within the same unicode block. */ - public static String randomRealisticUnicodeString(Random r, int maxLength) { - final int end = r.nextInt(maxLength); - final int block = r.nextInt(blockStarts.length); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < end; i++) - sb.appendCodePoint(nextInt(r, blockStarts[block], blockEnds[block])); - return sb.toString(); - } - - /** start and end are BOTH inclusive */ - public static int nextInt(Random r, int start, int end) { - return start + r.nextInt(end-start+1); - } - - public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException { - String[] files = dir.listAll(); - if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) { - return true; - } else { - return false; - } - } - - // just tries to configure things to keep the open file - // count lowish - public static void reduceOpenFiles(IndexWriter w) { - // keep number of open files lowish - LogMergePolicy lmp = (LogMergePolicy) w.getMergePolicy(); - lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor())); - - MergeScheduler ms = w.getConfig().getMergeScheduler(); - if (ms instanceof ConcurrentMergeScheduler) { - ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2); - ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3); - } - } - - /** Checks some basic behaviour of an AttributeImpl - * @param reflectedValues contains a map with "AttributeClass#key" as values - */ - public static void assertAttributeReflection(final AttributeImpl att, Map reflectedValues) { - final Map map = new HashMap(); - att.reflectWith(new AttributeReflector() { - public void reflect(Class attClass, String key, Object value) { - map.put(attClass.getName() + '#' + key, value); - } - }); - Assert.assertEquals("Reflection does not produce same map", reflectedValues, map); - } -} Index: lucene/build.xml =================================================================== --- lucene/build.xml (revision 1066326) +++ lucene/build.xml (working copy) @@ -37,12 +37,13 @@ + - + Index: lucene/contrib/db/bdb-je/build.xml =================================================================== --- lucene/contrib/db/bdb-je/build.xml (revision 1066326) +++ lucene/contrib/db/bdb-je/build.xml (working copy) @@ -39,13 +39,6 @@ - - - - - - - - - - - - - - + @@ -49,8 +50,6 @@ - - Index: lucene/contrib/analyzers/smartcn/build.xml =================================================================== --- lucene/contrib/analyzers/smartcn/build.xml (revision 1066326) +++ lucene/contrib/analyzers/smartcn/build.xml (working copy) @@ -28,11 +28,5 @@ - - - - - - - + Index: lucene/contrib/analyzers/common/build.xml =================================================================== --- lucene/contrib/analyzers/common/build.xml (revision 1066326) +++ lucene/contrib/analyzers/common/build.xml (working copy) @@ -29,13 +29,6 @@ - - - - - - - Index: lucene/contrib/analyzers/stempel/build.xml =================================================================== --- lucene/contrib/analyzers/stempel/build.xml (revision 1066326) +++ lucene/contrib/analyzers/stempel/build.xml (working copy) @@ -28,11 +28,4 @@ - - - - - - - Index: dev-tools/idea/lucene/lucene.iml =================================================================== --- dev-tools/idea/lucene/lucene.iml (revision 1066326) +++ dev-tools/idea/lucene/lucene.iml (working copy) @@ -9,6 +9,7 @@ + Index: dev-tools/eclipse/dot.classpath =================================================================== --- dev-tools/eclipse/dot.classpath (revision 1066326) +++ dev-tools/eclipse/dot.classpath (working copy) @@ -1,6 +1,7 @@ +